From a5cf8015862203ba670cc99c3a6efe6c6d28e791 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 15:59:37 +0300 Subject: [PATCH 01/17] Add the Python SDK, simplyworks-serverless Adapters in Python 3.12+ declare settings with sw.expect, mark commands with @sw.command, and run with sw.run, which serves --describe and otherwise the host that started it. Resident adapters add start, stop, status and reset; a call's context publishes events, keeps state and records metrics, and Python logging reaches the host. It has no dependencies. The protocol is one gRPC stream over a Unix socket, so the SDK carries its own protobuf codec for adapter.proto's messages and a small HTTP/2 client with HPACK and flow control in both directions. It vendors into a package as plain files, for any platform, with nothing to fetch. --- .github/workflows/python-sdk.yml | 26 + sdk/python/.gitignore | 4 + sdk/python/README.md | 45 ++ sdk/python/pyproject.toml | 20 + .../src/simplyworks_serverless/__init__.py | 28 + .../src/simplyworks_serverless/_adapter.py | 159 ++++++ .../src/simplyworks_serverless/_hpack.py | 186 +++++++ .../src/simplyworks_serverless/_http2.py | 253 +++++++++ .../src/simplyworks_serverless/_huffman.py | 37 ++ .../src/simplyworks_serverless/_runner.py | 498 ++++++++++++++++++ .../src/simplyworks_serverless/_types.py | 124 +++++ .../src/simplyworks_serverless/_wire.py | 242 +++++++++ sdk/python/tests/test_adapter.py | 108 ++++ sdk/python/tests/test_wire.py | 62 +++ 14 files changed, 1792 insertions(+) create mode 100644 .github/workflows/python-sdk.yml create mode 100644 sdk/python/.gitignore create mode 100644 sdk/python/README.md create mode 100644 sdk/python/pyproject.toml create mode 100644 sdk/python/src/simplyworks_serverless/__init__.py create mode 100644 sdk/python/src/simplyworks_serverless/_adapter.py create mode 100644 sdk/python/src/simplyworks_serverless/_hpack.py create mode 100644 sdk/python/src/simplyworks_serverless/_http2.py create mode 100644 sdk/python/src/simplyworks_serverless/_huffman.py create mode 100644 sdk/python/src/simplyworks_serverless/_runner.py create mode 100644 sdk/python/src/simplyworks_serverless/_types.py create mode 100644 sdk/python/src/simplyworks_serverless/_wire.py create mode 100644 sdk/python/tests/test_adapter.py create mode 100644 sdk/python/tests/test_wire.py diff --git a/.github/workflows/python-sdk.yml b/.github/workflows/python-sdk.yml new file mode 100644 index 0000000..ba0df25 --- /dev/null +++ b/.github/workflows/python-sdk.yml @@ -0,0 +1,26 @@ +name: Python SDK + +on: + push: + branches: [main] + paths: ["sdk/python/**", ".github/workflows/python-sdk.yml"] + pull_request: + paths: ["sdk/python/**", ".github/workflows/python-sdk.yml"] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python: ["3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Unit tests + working-directory: sdk/python + run: PYTHONPATH=src python -m unittest discover -s tests -v diff --git a/sdk/python/.gitignore b/sdk/python/.gitignore new file mode 100644 index 0000000..f47a5b3 --- /dev/null +++ b/sdk/python/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.egg-info/ +build/ +dist/ diff --git a/sdk/python/README.md b/sdk/python/README.md new file mode 100644 index 0000000..b18e9b1 --- /dev/null +++ b/sdk/python/README.md @@ -0,0 +1,45 @@ +# simplyworks-serverless + +Write SW-Serverless adapters in Python 3.12 or later. The package has no dependencies: it speaks the +host's protocol — gRPC over a Unix socket — with the standard library alone, so it vendors into an +adapter package as plain files, for any platform. + +```python +import simplyworks_serverless as sw + + +class Greeter: + def __init__(self): + sw.expect("Greeting", "Hello", description="What to say") + sw.expect("ApiKey", secret=True) + + @sw.command(description="Greets someone") + def greet(self, name: str) -> str: + return f"{sw.value_of('Greeting')}, {name}" + + +if __name__ == "__main__": + sw.run(Greeter) +``` + +- **Settings** are declared with `sw.expect(name, default=None, *, required, secret, description, type)` + and read with `sw.value_of(name)`: the call's own properties, then the values the adapter was + started with, then the default. +- **Commands** are methods marked `@sw.command(name, description=...)`. A command takes at most one + argument. A `str` argument or result is raw text, `bytes` are passed as they are, and anything else + is JSON; a dataclass is read from and written as a JSON object. Commands may be sync or async; sync + ones run on a worker thread. +- **Errors** raised by a command reach the caller with their type and message. Raise + `sw.AdapterError(message, type="Acme.Rejected")` to choose the type. +- **Resident adapters** have a `start` method and run until stopped, with optional `stop`, `status` + (returns `connected`, `state`, `details`…) and `reset(session_id)`. In a command, + `sw.context()` publishes events (`await ctx.publish(...)`), keeps small state + (`get_state`/`set_state`/`delete_state`) and records metrics. +- **Logging** through Python's `logging` reaches the host. +- `python main.py --describe` prints what the adapter is; `serverless build` writes it into the + manifest. + +For Bitween adapters, `simplyworks-bitween` has the four kinds — `Handler`, `Mapper`, `Validator`, +`Receiver` — ready to subclass. `serverless init --lang python --kind handler` starts one. + +Tests: `PYTHONPATH=src python -m unittest discover -s tests`. diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml new file mode 100644 index 0000000..0496666 --- /dev/null +++ b/sdk/python/pyproject.toml @@ -0,0 +1,20 @@ +[build-system] +requires = ["setuptools>=69"] +build-backend = "setuptools.build_meta" + +[project] +name = "simplyworks-serverless" +version = "10.1.0" +description = "Write SW-Serverless adapters in Python: settings, commands, and the host protocol, with no dependencies." +readme = "README.md" +requires-python = ">=3.12" +license = "MIT" +authors = [{ name = "Simplify9" }] +classifiers = ["Programming Language :: Python :: 3", "Operating System :: POSIX"] +dependencies = [] + +[project.urls] +Repository = "https://github.com/simplify9/SW-Serverless" + +[tool.setuptools.packages.find] +where = ["src"] diff --git a/sdk/python/src/simplyworks_serverless/__init__.py b/sdk/python/src/simplyworks_serverless/__init__.py new file mode 100644 index 0000000..104aba4 --- /dev/null +++ b/sdk/python/src/simplyworks_serverless/__init__.py @@ -0,0 +1,28 @@ +"""Write SW-Serverless adapters in Python. + + import simplyworks_serverless as sw + + class Greeter: + def __init__(self): + sw.expect("Greeting", "Hello", description="What to say") + + @sw.command(description="Greets someone") + def greet(self, name: str) -> str: + return f"{sw.value_of('Greeting')}, {name}" + + if __name__ == "__main__": + sw.run(Greeter) + +``python adapter.py --describe`` prints what it is; the host runs it otherwise. An adapter with a +``start`` hook is resident: it runs until stopped, with ``stop``, ``status`` and ``reset`` hooks. +""" + +from ._adapter import command, declared_settings, expect, startup_values, value_of +from ._runner import SDK_VERSION, AdapterError, Context, context, describe, run + +__version__ = SDK_VERSION + +__all__ = [ + "AdapterError", "Context", "SDK_VERSION", "command", "context", "declared_settings", "describe", + "expect", "run", "startup_values", "value_of", +] diff --git a/sdk/python/src/simplyworks_serverless/_adapter.py b/sdk/python/src/simplyworks_serverless/_adapter.py new file mode 100644 index 0000000..3b20bd9 --- /dev/null +++ b/sdk/python/src/simplyworks_serverless/_adapter.py @@ -0,0 +1,159 @@ +"""Declaring an adapter: its settings, its commands, its kinds and contracts. + +Settings are declared once, in code, with :func:`expect` — the Python form of the .NET SDK's +``Runner.Expect`` — and reach the manifest through ``--describe`` and the host through ``Hello``. +""" + +import contextvars +import inspect +import typing + +from . import _types + +SETTING_TYPES = ("text", "multiline", "number", "boolean", "select", "json") + + +class Setting: + def __init__(self, name, default, required, secret, description, type): + self.name = name + self.default = default + self.required = required + self.secret = secret + self.description = description + self.type = type + + def describe(self): + return {"name": self.name, "description": self.description, "type": self.type, + "required": self.required, "secret": self.secret, "default": self.default} + + +_settings = {} + + +def expect(name, default=None, *, required=None, secret=False, description=None, type="text"): + """Declares a setting the adapter reads. + + Required unless it has a default or ``required=False`` says otherwise. A secret is masked + wherever Bitween shows it. Declaring the same name again replaces it. + """ + if not name or not isinstance(name, str): + raise ValueError("a setting needs a name") + if type not in SETTING_TYPES: + raise ValueError(f"setting type must be one of {', '.join(SETTING_TYPES)}") + if required is None: + required = default is None + _settings[name] = Setting(name, None if default is None else str(default), bool(required), bool(secret), + description, type) + return name + + +def declared_settings(): + return list(_settings.values()) + + +# The values a call sees: this invocation's properties over the process's startup values. +_startup_values = {} +_call_values = contextvars.ContextVar("sw_call_values", default=None) + + +def value_of(name, default=None): + """A setting's value for the current call: the call's own properties, then the values the + adapter was started with, then the declared default, then ``default``.""" + call = _call_values.get() + if call and name in call: + return call[name] + if name in _startup_values: + return _startup_values[name] + declared = _settings.get(name) + if declared is not None and declared.default is not None: + return declared.default + return default + + +def startup_values(): + return dict(_startup_values) + + +def command(name=None, *, description=None): + """Marks a method as a command the host can call, under ``name`` or the method's own name.""" + + def mark(fn): + fn.__sw_command__ = {"name": name or fn.__name__, "description": description} + return fn + + if callable(name): # used bare: @command + fn, name = name, None + return mark(fn) + return mark + + +class Command: + def __init__(self, name, method, description): + self.name = name + self.method = method + self.description = description + signature = inspect.signature(method) + parameters = [p for p in signature.parameters.values() + if p.name != "self" and p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD)] + if len(parameters) > 1: + raise TypeError(f"command {name} takes {len(parameters)} arguments; a command takes at most one") + try: + hints = typing.get_type_hints(method) + except Exception: + hints = {} + self.takes_argument = bool(parameters) + self.argument_type = hints.get(parameters[0].name, _types._EMPTY) if parameters else None + returns = hints.get("return", signature.return_annotation) + self.returns_value = returns not in (None, type(None)) + self.result_type = returns + + def info(self): + input_schema = _types.schema(self.argument_type) if self.takes_argument else None + output_schema = _types.schema(self.result_type) if self.returns_value else None + return {"name": self.name, "description": self.description, "takes_argument": self.takes_argument, + "returns_value": self.returns_value, "input_schema": input_schema, "output_schema": output_schema, + "parameter_type": _type_name(self.argument_type) if self.takes_argument else ""} + + +def _type_name(tp): + if tp in (_types._EMPTY, None): + return "object" + return getattr(tp, "__name__", str(tp)) + + +def commands_of(adapter): + """Every command on the adapter's class, by wire name. Kinds' base classes add their own.""" + found = {} + cls = adapter if isinstance(adapter, type) else type(adapter) + for klass in reversed(cls.__mro__): + for attr, value in vars(klass).items(): + marker = getattr(value, "__sw_command__", None) + if marker: + found[marker["name"]] = attr + result = {} + for wire_name, attr in found.items(): + method = getattr(adapter, attr) + marker = getattr(getattr(cls, attr), "__sw_command__", None) or {} + result[wire_name] = Command(wire_name, method, marker.get("description")) + return result + + +def kinds_of(cls): + kinds = [] + for klass in cls.__mro__: + for kind in getattr(klass, "__sw_kinds__", ()) or (): + if kind not in kinds: + kinds.append(kind) + return kinds + + +def contracts_of(cls): + contracts = {} + for klass in reversed(cls.__mro__): + contracts.update(getattr(klass, "__sw_contracts__", {}) or {}) + return contracts + + +def is_resident(cls): + """Resident when it has a start hook: it runs until stopped rather than for one session.""" + return callable(getattr(cls, "start", None)) diff --git a/sdk/python/src/simplyworks_serverless/_hpack.py b/sdk/python/src/simplyworks_serverless/_hpack.py new file mode 100644 index 0000000..f9188ea --- /dev/null +++ b/sdk/python/src/simplyworks_serverless/_hpack.py @@ -0,0 +1,186 @@ +"""HPACK (RFC 7541): enough to talk to the host. + +The adapter sends one set of request headers, encoded as literals without indexing, which every +decoder accepts. What the host sends back is decoded in full: static and dynamic tables, size +updates and Huffman-coded strings, since a server is free to use all of them. +""" + +from ._huffman import CODES + +STATIC_TABLE = ( + (":authority", ""), (":method", "GET"), (":method", "POST"), (":path", "/"), + (":path", "/index.html"), (":scheme", "http"), (":scheme", "https"), (":status", "200"), + (":status", "204"), (":status", "206"), (":status", "304"), (":status", "400"), + (":status", "404"), (":status", "500"), ("accept-charset", ""), ("accept-encoding", "gzip, deflate"), + ("accept-language", ""), ("accept-ranges", ""), ("accept", ""), ("access-control-allow-origin", ""), + ("age", ""), ("allow", ""), ("authorization", ""), ("cache-control", ""), + ("content-disposition", ""), ("content-encoding", ""), ("content-language", ""), ("content-length", ""), + ("content-location", ""), ("content-range", ""), ("content-type", ""), ("cookie", ""), + ("date", ""), ("etag", ""), ("expect", ""), ("expires", ""), + ("from", ""), ("host", ""), ("if-match", ""), ("if-modified-since", ""), + ("if-none-match", ""), ("if-range", ""), ("if-unmodified-since", ""), ("last-modified", ""), + ("link", ""), ("location", ""), ("max-forwards", ""), ("proxy-authenticate", ""), + ("proxy-authorization", ""), ("range", ""), ("referer", ""), ("refresh", ""), + ("retry-after", ""), ("server", ""), ("set-cookie", ""), ("strict-transport-security", ""), + ("transfer-encoding", ""), ("user-agent", ""), ("vary", ""), ("via", ""), + ("www-authenticate", ""), +) + + +class HpackError(Exception): + pass + + +def _huffman_tree(): + # A binary trie over the codes: each node is [child0, child1, symbol]. + root = [None, None, None] + for symbol, (code, length) in enumerate(CODES): + node = root + for bit in range(length - 1, -1, -1): + b = (code >> bit) & 1 + if node[b] is None: + node[b] = [None, None, None] + node = node[b] + node[2] = symbol + return root + + +_TREE = _huffman_tree() + + +def huffman_decode(data): + out = bytearray() + node = _TREE + depth = 0 + for byte in data: + for bit in range(7, -1, -1): + node = node[(byte >> bit) & 1] + depth += 1 + if node is None: + raise HpackError("invalid Huffman code") + if node[2] is not None: + if node[2] == 256: + raise HpackError("EOS symbol in a Huffman string") + out.append(node[2]) + node = _TREE + depth = 0 + # Padding is the most significant bits of EOS (all ones), and shorter than a byte. + if depth > 7: + raise HpackError("Huffman padding longer than 7 bits") + return bytes(out) + + +def _decode_int(data, pos, prefix_bits): + mask = (1 << prefix_bits) - 1 + value = data[pos] & mask + pos += 1 + if value < mask: + return value, pos + shift = 0 + while True: + if pos >= len(data): + raise HpackError("truncated integer") + byte = data[pos] + pos += 1 + value += (byte & 0x7F) << shift + shift += 7 + if not byte & 0x80: + return value, pos + + +def _encode_int(value, prefix_bits, first_byte_flags=0): + mask = (1 << prefix_bits) - 1 + if value < mask: + return bytes([first_byte_flags | value]) + out = bytearray([first_byte_flags | mask]) + value -= mask + while value >= 0x80: + out.append((value & 0x7F) | 0x80) + value >>= 7 + out.append(value) + return bytes(out) + + +def _decode_string(data, pos): + if pos >= len(data): + raise HpackError("truncated string") + huffman = data[pos] & 0x80 + length, pos = _decode_int(data, pos, 7) + raw = data[pos:pos + length] + if len(raw) != length: + raise HpackError("truncated string") + pos += length + return (huffman_decode(raw) if huffman else bytes(raw)).decode("latin-1"), pos + + +def _encode_string(text): + raw = text.encode("latin-1") + return _encode_int(len(raw), 7) + raw + + +def encode(headers): + """Headers as literals without indexing: never touch either side's dynamic table.""" + out = bytearray() + for name, value in headers: + out += b"\x00" + _encode_string(name) + _encode_string(value) + return bytes(out) + + +class Decoder: + def __init__(self, max_size=4096): + self.max_size = max_size + self.size = 0 + self.dynamic = [] # newest first + + @staticmethod + def _entry_size(name, value): + return len(name.encode("latin-1")) + len(value.encode("latin-1")) + 32 + + def _evict(self): + while self.size > self.max_size and self.dynamic: + name, value = self.dynamic.pop() + self.size -= self._entry_size(name, value) + + def _add(self, name, value): + self.dynamic.insert(0, (name, value)) + self.size += self._entry_size(name, value) + self._evict() + + def _lookup(self, index): + if index <= 0: + raise HpackError("index 0") + if index <= len(STATIC_TABLE): + return STATIC_TABLE[index - 1] + index -= len(STATIC_TABLE) + 1 + if index >= len(self.dynamic): + raise HpackError("index past the dynamic table") + return self.dynamic[index] + + def decode(self, data): + headers = [] + pos = 0 + while pos < len(data): + byte = data[pos] + if byte & 0x80: # indexed + index, pos = _decode_int(data, pos, 7) + headers.append(self._lookup(index)) + elif byte & 0x40: # literal, incremental indexing + index, pos = _decode_int(data, pos, 6) + name = self._lookup(index)[0] if index else None + if name is None: + name, pos = _decode_string(data, pos) + value, pos = _decode_string(data, pos) + self._add(name, value) + headers.append((name, value)) + elif byte & 0x20: # dynamic table size update + size, pos = _decode_int(data, pos, 5) + self.max_size = size + self._evict() + else: # literal without indexing (0000) or never indexed (0001) + index, pos = _decode_int(data, pos, 4) + name = self._lookup(index)[0] if index else None + if name is None: + name, pos = _decode_string(data, pos) + value, pos = _decode_string(data, pos) + headers.append((name, value)) + return headers diff --git a/sdk/python/src/simplyworks_serverless/_http2.py b/sdk/python/src/simplyworks_serverless/_http2.py new file mode 100644 index 0000000..1d15ac4 --- /dev/null +++ b/sdk/python/src/simplyworks_serverless/_http2.py @@ -0,0 +1,253 @@ +"""One gRPC bidirectional stream over HTTP/2 cleartext (h2c, prior knowledge), on asyncio. + +The host serves gRPC on a Unix domain socket and the adapter opens exactly one call on it, Attach, +for its whole life. That is all this implements: one client stream, the frames it needs, and flow +control in both directions — the part that matters, since a result can be 64 MB and the default +window is 64 KB. +""" + +import asyncio +import struct + +from . import _hpack + +PREFACE = b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" + +DATA, HEADERS, PRIORITY, RST_STREAM, SETTINGS, PUSH_PROMISE, PING, GOAWAY, WINDOW_UPDATE, CONTINUATION = range(10) +END_STREAM, ACK, END_HEADERS, PADDED, PRIORITY_FLAG = 0x1, 0x1, 0x4, 0x8, 0x20 + +SETTINGS_HEADER_TABLE_SIZE, SETTINGS_INITIAL_WINDOW_SIZE, SETTINGS_MAX_FRAME_SIZE = 0x1, 0x4, 0x5 + +STREAM_ID = 1 +DEFAULT_WINDOW = 65535 +# What the adapter lets the host send before it acknowledges: large, so a big payload is not +# trickled through 64 KB at a time. +RECEIVE_WINDOW = 16 * 1024 * 1024 +MAX_MESSAGE = 64 * 1024 * 1024 + + +class GrpcError(Exception): + def __init__(self, status, message): + super().__init__(f"gRPC status {status}: {message}") + self.status = status + + +class GrpcStream: + """Attach: write messages, read messages, and know when the host has ended the call.""" + + def __init__(self, reader, writer, path): + self._reader = reader + self._writer = writer + self._path = path + self._decoder = _hpack.Decoder() + # Messages go out whole and in order, under the message lock, which may wait for window. + # Every socket write takes only the write lock, briefly, so the read loop can still + # acknowledge frames — and so receive the WINDOW_UPDATE that wait is for. + self._message_lock = asyncio.Lock() + self._write_lock = asyncio.Lock() + self._window_changed = asyncio.Condition() + self._conn_send_window = DEFAULT_WINDOW + self._stream_send_window = DEFAULT_WINDOW + self._peer_initial_window = DEFAULT_WINDOW + self._peer_max_frame = 16384 + self._inbound = asyncio.Queue() + self._buffer = bytearray() + self._header_block = bytearray() + self._header_stream = 0 + self._header_end_stream = False + self._closed = False + self._read_task = None + + @classmethod + async def open(cls, socket_path, path): + reader, writer = await asyncio.open_unix_connection(socket_path, limit=2 ** 20) + stream = cls(reader, writer, path) + await stream._start() + return stream + + # ------------------------------------------------------------------ framing + + def _frame(self, frame_type, flags, stream_id, payload=b""): + return struct.pack(">I", len(payload))[1:] + bytes([frame_type, flags]) + struct.pack(">I", stream_id) + payload + + async def _write(self, data): + async with self._write_lock: + self._writer.write(data) + await self._writer.drain() + + async def _start(self): + settings = struct.pack(">HI", SETTINGS_INITIAL_WINDOW_SIZE, RECEIVE_WINDOW) + # Raise the connection-level window too; it starts at 64 KB whatever SETTINGS says. + grow = struct.pack(">I", RECEIVE_WINDOW - DEFAULT_WINDOW) + headers = _hpack.encode([ + (":method", "POST"), (":scheme", "http"), (":path", self._path), (":authority", "localhost"), + ("content-type", "application/grpc"), ("te", "trailers"), + ("user-agent", "simplyworks-serverless-python"), + ]) + await self._write( + PREFACE + + self._frame(SETTINGS, 0, 0, settings) + + self._frame(WINDOW_UPDATE, 0, 0, grow) + + self._frame(HEADERS, END_HEADERS, STREAM_ID, headers)) + self._read_task = asyncio.get_running_loop().create_task(self._read_loop()) + + # ------------------------------------------------------------------ sending + + async def send(self, message): + if len(message) > MAX_MESSAGE: + raise ValueError(f"a message of {len(message)} bytes is more than the {MAX_MESSAGE} the host accepts") + data = b"\x00" + struct.pack(">I", len(message)) + message + async with self._message_lock: + pos = 0 + while pos < len(data): + async with self._window_changed: + await self._window_changed.wait_for( + lambda: self._closed or (self._conn_send_window > 0 and self._stream_send_window > 0)) + if self._closed: + raise ConnectionError("the host closed the stream") + size = min(len(data) - pos, self._conn_send_window, self._stream_send_window, self._peer_max_frame) + self._conn_send_window -= size + self._stream_send_window -= size + await self._write(self._frame(DATA, 0, STREAM_ID, data[pos:pos + size])) + pos += size + + async def close_send(self): + async with self._message_lock: + if not self._closed: + try: + await self._write(self._frame(DATA, END_STREAM, STREAM_ID)) + except (ConnectionError, OSError): + pass + + # ------------------------------------------------------------------ receiving + + async def receive(self): + """The next message, or None once the host has ended the call.""" + item = await self._inbound.get() + if isinstance(item, Exception): + raise item + return item + + async def _read_loop(self): + try: + while True: + head = await self._reader.readexactly(9) + length = int.from_bytes(head[:3], "big") + frame_type, flags = head[3], head[4] + stream_id = int.from_bytes(head[5:9], "big") & 0x7FFFFFFF + payload = await self._reader.readexactly(length) if length else b"" + if await self._on_frame(frame_type, flags, stream_id, payload): + break + await self._inbound.put(None) + except (asyncio.IncompleteReadError, ConnectionError, OSError): + await self._inbound.put(None) + except Exception as ex: # a protocol error: end the call rather than hang + await self._inbound.put(ex) + finally: + await self._mark_closed() + + async def _mark_closed(self): + self._closed = True + async with self._window_changed: + self._window_changed.notify_all() + + async def _on_frame(self, frame_type, flags, stream_id, payload): + """True when the call is over.""" + if frame_type == SETTINGS: + if not flags & ACK: + await self._apply_settings(payload) + await self._write(self._frame(SETTINGS, ACK, 0)) + elif frame_type == PING: + if not flags & ACK: + await self._write(self._frame(PING, ACK, 0, payload)) + elif frame_type == WINDOW_UPDATE: + increment = int.from_bytes(payload[:4], "big") & 0x7FFFFFFF + async with self._window_changed: + if stream_id == 0: + self._conn_send_window += increment + elif stream_id == STREAM_ID: + self._stream_send_window += increment + self._window_changed.notify_all() + elif frame_type in (HEADERS, CONTINUATION): + if frame_type == HEADERS: + payload = self._strip(flags, payload, headers=True) + self._header_block = bytearray(payload) + self._header_stream = stream_id + self._header_end_stream = bool(flags & END_STREAM) + else: + self._header_block += payload + if flags & END_HEADERS: + headers = self._decoder.decode(bytes(self._header_block)) + if self._header_stream == STREAM_ID: + self._check_headers(headers) + if self._header_end_stream: + return True + elif frame_type == DATA: + data = self._strip(flags, payload) + if length := len(payload): + # Hand the window straight back: messages are consumed as they are parsed. + increment = struct.pack(">I", length) + await self._write(self._frame(WINDOW_UPDATE, 0, 0, increment) + + self._frame(WINDOW_UPDATE, 0, STREAM_ID, increment)) + if stream_id == STREAM_ID: + self._buffer += data + await self._drain_messages() + if flags & END_STREAM: + return True + elif frame_type == RST_STREAM and stream_id == STREAM_ID: + code = int.from_bytes(payload[:4], "big") + raise GrpcError(code, "the host reset the stream") + elif frame_type == GOAWAY: + return True + return False + + async def _apply_settings(self, payload): + for i in range(0, len(payload) - 5, 6): + ident, value = struct.unpack(">HI", payload[i:i + 6]) + if ident == SETTINGS_INITIAL_WINDOW_SIZE: + async with self._window_changed: + self._stream_send_window += value - self._peer_initial_window + self._peer_initial_window = value + self._window_changed.notify_all() + elif ident == SETTINGS_MAX_FRAME_SIZE: + self._peer_max_frame = value + + @staticmethod + def _strip(flags, payload, headers=False): + pad = 0 + if flags & PADDED: + pad = payload[0] + payload = payload[1:] + if headers and flags & PRIORITY_FLAG: + payload = payload[5:] + return payload[:len(payload) - pad] if pad else payload + + def _check_headers(self, headers): + values = dict(headers) + status = values.get(":status") + if status is not None and status != "200": + raise GrpcError(-1, f"HTTP status {status}") + grpc_status = values.get("grpc-status") + if grpc_status is not None and grpc_status != "0": + raise GrpcError(int(grpc_status), values.get("grpc-message", "")) + + async def _drain_messages(self): + while len(self._buffer) >= 5: + if self._buffer[0] != 0: + raise GrpcError(-1, "compressed messages are not supported") + length = int.from_bytes(self._buffer[1:5], "big") + if len(self._buffer) < 5 + length: + return + message = bytes(self._buffer[5:5 + length]) + del self._buffer[:5 + length] + await self._inbound.put(message) + + async def aclose(self): + await self.close_send() + try: + self._writer.close() + await self._writer.wait_closed() + except (ConnectionError, OSError): + pass + if self._read_task: + self._read_task.cancel() diff --git a/sdk/python/src/simplyworks_serverless/_huffman.py b/sdk/python/src/simplyworks_serverless/_huffman.py new file mode 100644 index 0000000..7ed2e0f --- /dev/null +++ b/sdk/python/src/simplyworks_serverless/_huffman.py @@ -0,0 +1,37 @@ +"""The HPACK Huffman code, RFC 7541 Appendix B: (code, bit length) for each symbol 0-256.""" + +CODES = ( + (0x1ff8, 13), (0x7fffd8, 23), (0xfffffe2, 28), (0xfffffe3, 28), (0xfffffe4, 28), (0xfffffe5, 28), (0xfffffe6, 28), (0xfffffe7, 28), + (0xfffffe8, 28), (0xffffea, 24), (0x3ffffffc, 30), (0xfffffe9, 28), (0xfffffea, 28), (0x3ffffffd, 30), (0xfffffeb, 28), (0xfffffec, 28), + (0xfffffed, 28), (0xfffffee, 28), (0xfffffef, 28), (0xffffff0, 28), (0xffffff1, 28), (0xffffff2, 28), (0x3ffffffe, 30), (0xffffff3, 28), + (0xffffff4, 28), (0xffffff5, 28), (0xffffff6, 28), (0xffffff7, 28), (0xffffff8, 28), (0xffffff9, 28), (0xffffffa, 28), (0xffffffb, 28), + (0x14, 6), (0x3f8, 10), (0x3f9, 10), (0xffa, 12), (0x1ff9, 13), (0x15, 6), (0xf8, 8), (0x7fa, 11), + (0x3fa, 10), (0x3fb, 10), (0xf9, 8), (0x7fb, 11), (0xfa, 8), (0x16, 6), (0x17, 6), (0x18, 6), + (0x0, 5), (0x1, 5), (0x2, 5), (0x19, 6), (0x1a, 6), (0x1b, 6), (0x1c, 6), (0x1d, 6), + (0x1e, 6), (0x1f, 6), (0x5c, 7), (0xfb, 8), (0x7ffc, 15), (0x20, 6), (0xffb, 12), (0x3fc, 10), + (0x1ffa, 13), (0x21, 6), (0x5d, 7), (0x5e, 7), (0x5f, 7), (0x60, 7), (0x61, 7), (0x62, 7), + (0x63, 7), (0x64, 7), (0x65, 7), (0x66, 7), (0x67, 7), (0x68, 7), (0x69, 7), (0x6a, 7), + (0x6b, 7), (0x6c, 7), (0x6d, 7), (0x6e, 7), (0x6f, 7), (0x70, 7), (0x71, 7), (0x72, 7), + (0xfc, 8), (0x73, 7), (0xfd, 8), (0x1ffb, 13), (0x7fff0, 19), (0x1ffc, 13), (0x3ffc, 14), (0x22, 6), + (0x7ffd, 15), (0x3, 5), (0x23, 6), (0x4, 5), (0x24, 6), (0x5, 5), (0x25, 6), (0x26, 6), + (0x27, 6), (0x6, 5), (0x74, 7), (0x75, 7), (0x28, 6), (0x29, 6), (0x2a, 6), (0x7, 5), + (0x2b, 6), (0x76, 7), (0x2c, 6), (0x8, 5), (0x9, 5), (0x2d, 6), (0x77, 7), (0x78, 7), + (0x79, 7), (0x7a, 7), (0x7b, 7), (0x7ffe, 15), (0x7fc, 11), (0x3ffd, 14), (0x1ffd, 13), (0xffffffc, 28), + (0xfffe6, 20), (0x3fffd2, 22), (0xfffe7, 20), (0xfffe8, 20), (0x3fffd3, 22), (0x3fffd4, 22), (0x3fffd5, 22), (0x7fffd9, 23), + (0x3fffd6, 22), (0x7fffda, 23), (0x7fffdb, 23), (0x7fffdc, 23), (0x7fffdd, 23), (0x7fffde, 23), (0xffffeb, 24), (0x7fffdf, 23), + (0xffffec, 24), (0xffffed, 24), (0x3fffd7, 22), (0x7fffe0, 23), (0xffffee, 24), (0x7fffe1, 23), (0x7fffe2, 23), (0x7fffe3, 23), + (0x7fffe4, 23), (0x1fffdc, 21), (0x3fffd8, 22), (0x7fffe5, 23), (0x3fffd9, 22), (0x7fffe6, 23), (0x7fffe7, 23), (0xffffef, 24), + (0x3fffda, 22), (0x1fffdd, 21), (0xfffe9, 20), (0x3fffdb, 22), (0x3fffdc, 22), (0x7fffe8, 23), (0x7fffe9, 23), (0x1fffde, 21), + (0x7fffea, 23), (0x3fffdd, 22), (0x3fffde, 22), (0xfffff0, 24), (0x1fffdf, 21), (0x3fffdf, 22), (0x7fffeb, 23), (0x7fffec, 23), + (0x1fffe0, 21), (0x1fffe1, 21), (0x3fffe0, 22), (0x1fffe2, 21), (0x7fffed, 23), (0x3fffe1, 22), (0x7fffee, 23), (0x7fffef, 23), + (0xfffea, 20), (0x3fffe2, 22), (0x3fffe3, 22), (0x3fffe4, 22), (0x7ffff0, 23), (0x3fffe5, 22), (0x3fffe6, 22), (0x7ffff1, 23), + (0x3ffffe0, 26), (0x3ffffe1, 26), (0xfffeb, 20), (0x7fff1, 19), (0x3fffe7, 22), (0x7ffff2, 23), (0x3fffe8, 22), (0x1ffffec, 25), + (0x3ffffe2, 26), (0x3ffffe3, 26), (0x3ffffe4, 26), (0x7ffffde, 27), (0x7ffffdf, 27), (0x3ffffe5, 26), (0xfffff1, 24), (0x1ffffed, 25), + (0x7fff2, 19), (0x1fffe3, 21), (0x3ffffe6, 26), (0x7ffffe0, 27), (0x7ffffe1, 27), (0x3ffffe7, 26), (0x7ffffe2, 27), (0xfffff2, 24), + (0x1fffe4, 21), (0x1fffe5, 21), (0x3ffffe8, 26), (0x3ffffe9, 26), (0xffffffd, 28), (0x7ffffe3, 27), (0x7ffffe4, 27), (0x7ffffe5, 27), + (0xfffec, 20), (0xfffff3, 24), (0xfffed, 20), (0x1fffe6, 21), (0x3fffe9, 22), (0x1fffe7, 21), (0x1fffe8, 21), (0x7ffff3, 23), + (0x3fffea, 22), (0x3fffeb, 22), (0x1ffffee, 25), (0x1ffffef, 25), (0xfffff4, 24), (0xfffff5, 24), (0x3ffffea, 26), (0x7ffff4, 23), + (0x3ffffeb, 26), (0x7ffffe6, 27), (0x3ffffec, 26), (0x3ffffed, 26), (0x7ffffe7, 27), (0x7ffffe8, 27), (0x7ffffe9, 27), (0x7ffffea, 27), + (0x7ffffeb, 27), (0xffffffe, 28), (0x7ffffec, 27), (0x7ffffed, 27), (0x7ffffee, 27), (0x7ffffef, 27), (0x7fffff0, 27), (0x3ffffee, 26), + (0x3fffffff, 30), +) diff --git a/sdk/python/src/simplyworks_serverless/_runner.py b/sdk/python/src/simplyworks_serverless/_runner.py new file mode 100644 index 0000000..215e638 --- /dev/null +++ b/sdk/python/src/simplyworks_serverless/_runner.py @@ -0,0 +1,498 @@ +"""Running an adapter: the stdin handshake, the gRPC stream to the host, and every frame on it. + +The process is started by the host, which writes one JSON line on stdin — where to dial and a +one-time token — and keeps stdin open: when it closes, the host is gone and the adapter exits. +""" + +import asyncio +import contextvars +import inspect +import itertools +import json +import logging +import os +import sys +import threading +import time +import traceback + +from . import _adapter, _types, _wire +from ._http2 import GrpcStream + +SDK_VERSION = "10.1.0" +SDK_LANGUAGE = "python" +PROTOCOL = 2 +ATTACH = "/sw.serverless.v1.AdapterHost/Attach" +DESCRIBE_FLAG = "--describe" +DESCRIBE_VERSION = 1 + +# ILogger levels, which the host's log pipeline reads: Trace 0 .. Critical 5. +_LEVELS = ((logging.CRITICAL, 5), (logging.ERROR, 4), (logging.WARNING, 3), (logging.INFO, 2), (logging.DEBUG, 1)) + +_context = contextvars.ContextVar("sw_context", default=None) + + +class AdapterError(Exception): + """An error with a type of the adapter's choosing, which the host records as given.""" + + def __init__(self, message, *, type=None, detail=None): + super().__init__(message) + self.type = type + self.detail = detail + + +class Context: + """What a call can reach beyond its argument: its session, the host's state and events, logs.""" + + def __init__(self, runner, session_id=None, command=None, cancelled=None): + self._runner = runner + self.session_id = session_id + self.command = command + self.cancelled = cancelled or asyncio.Event() + + @property + def adapter_id(self): + return self._runner.handshake.get("adapterId", "") + + @property + def instance_key(self): + return self._runner.handshake.get("instanceKey", "") + + @property + def stopping(self): + """Set once the host has asked the adapter to stop.""" + return self._runner.stopping + + def value_of(self, name, default=None): + return _adapter.value_of(name, default) + + async def publish(self, payload, *, dedupe_key="", content_type="", headers=None, endpoint=""): + """Hands an event to the host and waits until it is persisted. Acknowledge the source only + after this returns: that is what makes delivery at-least-once. Returns the host's reference.""" + ack = await self._runner.request("event", { + "payload": _types.encode(payload), "dedupe_key": dedupe_key, "content_type": content_type, + "headers": headers or {}, "endpoint": endpoint}, "event_ack") + if not ack.get("accepted"): + error = ack.get("error") or {} + raise AdapterError(error.get("message") or "the host did not accept the event", type=error.get("type")) + return ack.get("reference", "") + + async def get_state(self, name): + result = await self._state(0, name) + return result.get("value", "") if result.get("found") else None + + async def set_state(self, name, value): + await self._state(1, name, value) + + async def delete_state(self, name): + await self._state(2, name) + + async def _state(self, op, name, value=""): + result = await self._runner.request("state", {"op": op, "name": name, "value": value}, "state_result") + if result.get("error"): + raise AdapterError(result["error"].get("message", "state request failed"), type=result["error"].get("type")) + return result + + def metric(self, name, value, tags=None): + self._runner.telemetry("metric", {"name": name, "value": float(value), "tags": tags or {}}) + + +def context(): + """The current call's :class:`Context`, or the adapter's own outside a call.""" + current = _context.get() + if current is None: + raise RuntimeError("there is no adapter context outside a running adapter") + return current + + +class _HostLogHandler(logging.Handler): + """Python logging, forwarded to the host as log frames once the stream is open.""" + + def __init__(self, runner): + super().__init__() + self._runner = runner + + def emit(self, record): + level = next((sw for py, sw in _LEVELS if record.levelno >= py), 0) + if level < self._runner.minimum_level: + return + try: + message = record.getMessage() + except Exception: + message = str(record.msg) + exception = "".join(traceback.format_exception(*record.exc_info)) if record.exc_info else "" + self._runner.telemetry("log", { + "level": level, "message": message, "exception": exception, + "properties": {"logger": record.name}, "timestamp_unix_ms": int(record.created * 1000)}) + + +class Runner: + def __init__(self, adapter): + self.adapter = adapter + self.cls = type(adapter) + self.commands = _adapter.commands_of(adapter) + self.handshake = {} + self.stream = None + self.loop = None + self.stopping = None + self.minimum_level = 1 + self._ids = itertools.count(1) + self._pending = {} + self._running = {} + self._telemetry = None + self._ready = None + + # ------------------------------------------------------------------ description + + def hello(self): + return { + "token": self.handshake.get("token") or "", + "adapter_id": self.handshake.get("adapterId") or "", + "instance_key": self.handshake.get("instanceKey") or "", + "protocol_version": PROTOCOL, + "sdk_version": SDK_VERSION, + "sdk_language": SDK_LANGUAGE, + "capabilities": self.capabilities(), + "commands": [self._command_info(c) for c in self.commands.values()], + "settings": [{"name": s.name, "description": s.description or "", "required": s.required, + "secret": s.secret, "default_value": s.default or "", "type": s.type} + for s in _adapter.declared_settings()], + "kinds": _adapter.kinds_of(self.cls), + "contracts": _adapter.contracts_of(self.cls), + } + + @staticmethod + def _command_info(command): + info = command.info() + return {"name": info["name"], "parameter_type": info["parameter_type"], + "returns_value": info["returns_value"], "description": info["description"] or "", + "input_schema": json.dumps(info["input_schema"]) if info["input_schema"] is not None else "", + "output_schema": json.dumps(info["output_schema"]) if info["output_schema"] is not None else ""} + + def capabilities(self): + caps = [] + if _adapter.is_resident(self.cls): + caps.append("resident") + if callable(getattr(self.adapter, "reset", None)): + caps.append("resettable") + caps.append("cancel") + caps.extend("command:" + name for name in self.commands) + return caps + + # ------------------------------------------------------------------ running + + async def run(self, handshake_line): + self.handshake = json.loads(handshake_line) + if int(self.handshake.get("protocol") or 2) < PROTOCOL: + raise RuntimeError(f"the host speaks protocol {self.handshake.get('protocol')}; this SDK speaks {PROTOCOL}") + socket_path = self.handshake.get("socket") + if not socket_path: + raise RuntimeError("the handshake names no socket; this SDK runs on Linux and macOS hosts") + + self.loop = asyncio.get_running_loop() + self.stopping = asyncio.Event() + self._ready = asyncio.Event() + self._telemetry = asyncio.Queue(maxsize=10000) + _context.set(Context(self)) + + self.stream = await GrpcStream.open(socket_path, ATTACH) + await self.send({"hello": self.hello()}) + + handler = _HostLogHandler(self) + logging.getLogger().addHandler(handler) + if logging.getLogger().level > logging.INFO or logging.getLogger().level == logging.NOTSET: + logging.getLogger().setLevel(logging.INFO) + + watcher = threading.Thread(target=self._watch_parent, daemon=True) + watcher.start() + telemetry = self.loop.create_task(self._pump_telemetry()) + try: + await self._read_loop() + finally: + self.stopping.set() + logging.getLogger().removeHandler(handler) + telemetry.cancel() + await self.stream.aclose() + + def _watch_parent(self): + # Stdin stays open while the host lives; end of file means it's gone. + try: + while sys.stdin.readline(): + pass + except Exception: + pass + if self.loop and not self.loop.is_closed(): + self.loop.call_soon_threadsafe(self.stopping.set) + + async def _read_loop(self): + stop = self.loop.create_task(self.stopping.wait()) + try: + while True: + receive = self.loop.create_task(self.stream.receive()) + done, _ = await asyncio.wait({receive, stop}, return_when=asyncio.FIRST_COMPLETED) + if receive not in done: + receive.cancel() + return + data = receive.result() + if data is None: + return + frame = _wire.decode("HostFrame", data) + if await self._on_frame(frame): + return + finally: + stop.cancel() + + async def _on_frame(self, frame): + """True when the adapter should stop.""" + frame_id = frame.get("id", 0) + if "ready" in frame: + ready = frame["ready"] + _adapter._startup_values.clear() + _adapter._startup_values.update(ready.get("startup_values", {})) + start = getattr(self.adapter, "start", None) + if callable(start): + await _call(start) + self._ready.set() + elif "invoke" in frame: + self._start_invoke(frame_id, frame["invoke"]) + elif "cancel" in frame: + running = self._running.get(frame_id) + if running: + running[1].set() + running[0].cancel() + elif "ping" in frame: + self.loop.create_task(self._pong(frame_id)) + elif "reset" in frame: + self.loop.create_task(self._reset(frame_id, frame["reset"].get("session_id", ""))) + elif "set_log_level" in frame: + self.minimum_level = frame["set_log_level"].get("level", 0) + elif "shutdown" in frame: + await self._shutdown(frame["shutdown"]) + return True + else: + for answer in ("state_result", "event_ack"): + if answer in frame: + waiter = self._pending.pop(frame_id, None) + if waiter and not waiter.done(): + waiter.set_result(frame[answer]) + return False + + # ------------------------------------------------------------------ commands + + def _start_invoke(self, frame_id, invoke): + cancelled = asyncio.Event() + task = self.loop.create_task(self._invoke(frame_id, invoke, cancelled)) + self._running[frame_id] = (task, cancelled) + task.add_done_callback(lambda _: self._running.pop(frame_id, None)) + + async def _invoke(self, frame_id, invoke, cancelled): + name = invoke.get("command", "") + try: + await self._ready.wait() + command = self.commands.get(name) + if command is None: + raise AdapterError(f"the adapter has no command named {name}", type="MissingMethodException") + _adapter._call_values.set(invoke.get("properties") or {}) + _context.set(Context(self, invoke.get("session_id") or str(frame_id), name, cancelled)) + args = [_types.decode(command.argument_type, invoke.get("payload", b""))] if command.takes_argument else [] + coro = _call(command.method, *args) + timeout = invoke.get("timeout_seconds") or 0 + result = await (asyncio.wait_for(coro, timeout) if timeout > 0 else coro) + payload = _types.encode(result) if command.returns_value else b"" + await self.send({"id": frame_id, "invoke_result": {"payload": payload}}) + except asyncio.CancelledError: + await self._fail(frame_id, "OperationCanceledException", "the call was cancelled", "") + except Exception as ex: + await self._fail(frame_id, getattr(ex, "type", None) or _qualified(type(ex)), str(ex), + getattr(ex, "detail", None) or traceback.format_exc()) + + async def _fail(self, frame_id, error_type, message, detail): + try: + await self.send({"id": frame_id, "invoke_result": {"error": { + "type": error_type, "message": message, "detail": detail}}}) + except Exception: + pass + + async def _pong(self, frame_id): + pong = {"connected": True, "state": "Running", "in_flight": len(self._running)} + status = getattr(self.adapter, "status", None) + if callable(status): + try: + reported = await _call(status) or {} + pong.update({k: v for k, v in reported.items() if k in + ("connected", "state", "in_flight", "last_error", "last_message_unix_ms", "details")}) + except Exception as ex: + pong.update(connected=False, state="StatusFailed", last_error=str(ex)) + await self.send({"id": frame_id, "pong": pong}) + + async def _reset(self, frame_id, session_id): + result = {} + reset = getattr(self.adapter, "reset", None) + if callable(reset): + try: + await _call(reset, session_id) + except Exception as ex: + result["error"] = {"type": _qualified(type(ex)), "message": str(ex), "detail": traceback.format_exc()} + await self.send({"id": frame_id, "invoke_result": result}) + + async def _shutdown(self, shutdown): + # Ask it to stop first, let commands already running answer, and only then go: a drain + # promised those answers. + deadline = time.monotonic() + (30 if shutdown.get("drain") else 5) + stop = getattr(self.adapter, "stop", None) + if callable(stop): + try: + await asyncio.wait_for(_call(stop), max(0.1, deadline - time.monotonic())) + except Exception: + logging.getLogger("simplyworks_serverless").warning("stop() failed", exc_info=True) + running = [task for task, _ in list(self._running.values())] + if running: + await asyncio.wait(running, timeout=max(0.1, deadline - time.monotonic())) + await self._flush_telemetry() + + # ------------------------------------------------------------------ outbound + + async def send(self, frame): + await self.stream.send(_wire.encode("AdapterFrame", frame)) + + async def request(self, kind, body, answer): + """Sends a frame of the adapter's own and waits for the host's answer to it.""" + frame_id = next(self._ids) + waiter = self.loop.create_future() + self._pending[frame_id] = waiter + await self.send({"id": frame_id, kind: body}) + stopping = self.loop.create_task(self.stopping.wait()) + try: + done, _ = await asyncio.wait({waiter, stopping}, return_when=asyncio.FIRST_COMPLETED) + if waiter not in done: + raise AdapterError("the adapter is stopping", type="OperationCanceledException") + return waiter.result() + finally: + stopping.cancel() + self._pending.pop(frame_id, None) + + def telemetry(self, kind, body): + """Logs and metrics: queued, and dropped rather than allowed to hold anything up.""" + if self.loop is None or self._telemetry is None: + return + + def put(): + try: + self._telemetry.put_nowait({kind: body}) + except asyncio.QueueFull: + pass + + try: + if _in_loop(self.loop): + put() + else: + self.loop.call_soon_threadsafe(put) + except RuntimeError: + pass # the loop has closed + + async def _pump_telemetry(self): + while True: + frame = await self._telemetry.get() + try: + await self.send(frame) + except Exception: + return + + async def _flush_telemetry(self): + while not self._telemetry.empty(): + try: + await self.send(self._telemetry.get_nowait()) + except Exception: + return + + +def _in_loop(loop): + try: + return asyncio.get_running_loop() is loop + except RuntimeError: + return False + + +def _qualified(cls): + module = cls.__module__ + return cls.__qualname__ if module in ("builtins", "__main__") else f"{module}.{cls.__qualname__}" + + +async def _call(fn, *args): + """Calls a hook or command, sync or async. Sync ones run on a worker thread, with the call's + context, so a blocking adapter doesn't stop the host's pings being answered.""" + if inspect.iscoroutinefunction(fn): + return await fn(*args) + result = await asyncio.to_thread(fn, *args) + if inspect.isawaitable(result): + return await result + return result + + +# ---------------------------------------------------------------------------- entry point + +def describe(adapter): + """What ``--describe`` prints: the adapter's settings, commands, kinds and contracts.""" + cls = adapter if isinstance(adapter, type) else type(adapter) + warnings = [] + instance = adapter + if isinstance(adapter, type): + try: + instance = adapter() + except Exception as ex: + warnings.append(f"the adapter could not be built without its settings ({ex}); " + "settings it declares when built are missing") + instance = adapter + commands = [] + for command in _adapter.commands_of(instance).values(): + info = command.info() + commands.append({"name": info["name"], "description": info["description"], + "inputSchema": info["input_schema"], "outputSchema": info["output_schema"], + "returnsValue": info["returns_value"]}) + return { + "describeVersion": DESCRIBE_VERSION, + "sdkLanguage": SDK_LANGUAGE, + "sdkVersion": SDK_VERSION, + "lifecycle": "resident" if _adapter.is_resident(cls) else "classic", + "protocol": {"min": PROTOCOL, "max": PROTOCOL}, + "settings": [s.describe() for s in _adapter.declared_settings()], + "commands": commands, + "kinds": _adapter.kinds_of(cls), + "contracts": _adapter.contracts_of(cls), + "warnings": warnings, + } + + +def _check_kinds(adapter): + check = getattr(adapter, "__sw_check__", None) + if callable(check): + check() + + +def run(adapter): + """Runs the adapter: describes it for ``--describe``, otherwise serves the host that started it. + + ``adapter`` is an instance, or a class built with no arguments. + """ + if DESCRIBE_FLAG in sys.argv[1:]: + sys.stdout.write(json.dumps(describe(adapter), indent=2) + "\n") + sys.stdout.flush() + return + + instance = adapter() if isinstance(adapter, type) else adapter + _check_kinds(instance) + + line = sys.stdin.readline() + if not line: + print("stdin closed before the handshake arrived; the host is gone", file=sys.stderr) + sys.exit(1) + try: + asyncio.run(Runner(instance).run(line)) + except KeyboardInterrupt: + pass + except Exception: + traceback.print_exc() + sys.exit(1) + # Worker threads running blocking commands must not keep a stopped adapter alive. + sys.stdout.flush() + os._exit(0) diff --git a/sdk/python/src/simplyworks_serverless/_types.py b/sdk/python/src/simplyworks_serverless/_types.py new file mode 100644 index 0000000..2305580 --- /dev/null +++ b/sdk/python/src/simplyworks_serverless/_types.py @@ -0,0 +1,124 @@ +"""How command arguments and results cross the wire, and the JSON Schema that describes them. + +The encoding is the protocol's, the same in every language: a string is its raw UTF-8 text, bytes +are passed as they are, nothing is an empty payload, and anything else is JSON. A class can take +charge of its own JSON with ``to_wire``/``from_wire`` and describe it with ``__sw_schema__`` — the +Bitween types do, to keep the property names the contract fixes. +""" + +import base64 +import dataclasses +import inspect +import json +import typing + +_EMPTY = inspect.Parameter.empty + + +def _origin(tp): + return typing.get_origin(tp) or tp + + +def _strip_optional(tp): + if typing.get_origin(tp) is typing.Union: + args = [a for a in typing.get_args(tp) if a is not type(None)] + if len(args) == 1: + return args[0] + return tp + + +def decode(tp, payload): + """The argument a command receives, from the payload it was sent.""" + tp = _strip_optional(tp) + if tp in (_EMPTY, typing.Any, None): + if not payload: + return None + text = payload.decode("utf-8") + try: + return json.loads(text) + except ValueError: + return text + if tp is bytes: + return bytes(payload) + text = payload.decode("utf-8") + if tp is str: + return text + if not text: + return None + value = json.loads(text) + return from_json(tp, value) + + +def from_json(tp, value): + tp = _strip_optional(tp) + if value is None: + return None + if hasattr(tp, "from_wire"): + return tp.from_wire(value) + if dataclasses.is_dataclass(tp): + hints = typing.get_type_hints(tp) + names = {f.name for f in dataclasses.fields(tp)} + return tp(**{k: from_json(hints.get(k, typing.Any), v) for k, v in value.items() if k in names}) + origin = _origin(tp) + args = typing.get_args(tp) + if origin in (list, tuple, set) and args: + return origin(from_json(args[0], v) for v in value) + if origin is dict and len(args) == 2: + return {k: from_json(args[1], v) for k, v in value.items()} + if tp is bytes and isinstance(value, str): + return base64.b64decode(value) + return value + + +def encode(value): + """The payload a command's result becomes.""" + if value is None: + return b"" + if isinstance(value, (bytes, bytearray, memoryview)): + return bytes(value) + if isinstance(value, str): + return value.encode("utf-8") + return json.dumps(to_json(value), separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def to_json(value): + if hasattr(value, "to_wire"): + return value.to_wire() + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return {f.name: to_json(getattr(value, f.name)) for f in dataclasses.fields(value)} + if isinstance(value, dict): + return {str(k): to_json(v) for k, v in value.items()} + if isinstance(value, (list, tuple, set, frozenset)): + return [to_json(v) for v in value] + if isinstance(value, (bytes, bytearray)): + return base64.b64encode(bytes(value)).decode("ascii") + return value + + +def schema(tp): + """A JSON Schema for a command's argument or result type; None when it has none.""" + tp = _strip_optional(tp) + if tp in (_EMPTY, None, type(None)): + return None + if hasattr(tp, "__sw_schema__"): + return tp.__sw_schema__() + simple = {str: {"type": "string"}, int: {"type": "integer"}, float: {"type": "number"}, + bool: {"type": "boolean"}, bytes: {"type": "string", "contentEncoding": "binary"}} + if tp in simple: + return dict(simple[tp]) + if dataclasses.is_dataclass(tp): + hints = typing.get_type_hints(tp) + properties = {f.name: schema(hints.get(f.name, typing.Any)) or {} for f in dataclasses.fields(tp)} + required = [f.name for f in dataclasses.fields(tp) + if f.default is dataclasses.MISSING and f.default_factory is dataclasses.MISSING] + result = {"type": "object", "properties": properties} + if required: + result["required"] = required + return result + origin = _origin(tp) + args = typing.get_args(tp) + if origin in (list, tuple, set): + return {"type": "array", "items": (schema(args[0]) or {}) if args else {}} + if origin is dict: + return {"type": "object", "additionalProperties": (schema(args[1]) or {}) if len(args) == 2 else {}} + return {} diff --git a/sdk/python/src/simplyworks_serverless/_wire.py b/sdk/python/src/simplyworks_serverless/_wire.py new file mode 100644 index 0000000..29e5761 --- /dev/null +++ b/sdk/python/src/simplyworks_serverless/_wire.py @@ -0,0 +1,242 @@ +"""The protobuf wire format, for the messages in adapter.proto and nothing else. + +Hand-written rather than generated so the SDK needs no packages: messages are dicts keyed by the +proto's field names, and each schema below mirrors adapter.proto field for field. A field the +schema doesn't know is skipped when decoding, as protobuf does, so a newer host stays readable. +""" + +import struct + +VARINT, FIXED64, LENGTH, FIXED32 = 0, 1, 2, 5 + + +class WireError(Exception): + pass + + +# Field kinds: int (int32/int64/enum), bool, string, bytes, double, message, map (string->string), +# map_int (string->int32), and repeated forms written ("repeated", kind) or ("repeated_message", name). +SCHEMAS = { + # host -> adapter + "HostFrame": { + 1: ("id", "int"), 2: ("traceparent", "string"), + 3: ("ready", ("message", "Ready")), 4: ("invoke", ("message", "Invoke")), + 5: ("ping", ("message", "Empty")), 6: ("set_log_level", ("message", "SetLogLevel")), + 7: ("reset", ("message", "Reset")), 8: ("shutdown", ("message", "Shutdown")), + 9: ("event_ack", ("message", "EventAck")), 10: ("state_result", ("message", "StateResult")), + 11: ("cancel", ("message", "Empty")), + }, + "Empty": {}, + "Ready": {1: ("max_in_flight", "int"), 2: ("startup_values", "map"), 3: ("adapter_values", "map")}, + "Invoke": { + 1: ("command", "string"), 2: ("payload", "bytes"), 3: ("timeout_seconds", "int"), + 4: ("session_id", "string"), 5: ("properties", "map"), + }, + "SetLogLevel": {1: ("level", "int")}, + "Reset": {1: ("session_id", "string")}, + "Shutdown": {1: ("reason", "string"), 2: ("drain", "bool")}, + "StateResult": {1: ("found", "bool"), 2: ("value", "string"), 3: ("error", ("message", "Error"))}, + "EventAck": {1: ("accepted", "bool"), 2: ("reference", "string"), 3: ("error", ("message", "Error"))}, + # adapter -> host + "AdapterFrame": { + 1: ("id", "int"), 2: ("traceparent", "string"), + 3: ("hello", ("message", "Hello")), 4: ("invoke_result", ("message", "InvokeResult")), + 5: ("event", ("message", "Event")), 6: ("log", ("message", "LogEntry")), + 7: ("metric", ("message", "Metric")), 8: ("pong", ("message", "Pong")), + 9: ("state", ("message", "StateRequest")), + }, + "Hello": { + 1: ("token", "string"), 2: ("adapter_id", "string"), 3: ("instance_key", "string"), + 4: ("protocol_version", "int"), 5: ("sdk_version", "string"), + 6: ("capabilities", ("repeated", "string")), 7: ("commands", ("repeated_message", "CommandInfo")), + 8: ("sdk_language", "string"), 9: ("settings", ("repeated_message", "SettingInfo")), + 10: ("kinds", ("repeated", "string")), 11: ("contracts", "map_int"), + }, + "SettingInfo": { + 1: ("name", "string"), 2: ("description", "string"), 3: ("required", "bool"), + 4: ("secret", "bool"), 5: ("default_value", "string"), 6: ("type", "string"), + }, + "CommandInfo": { + 1: ("name", "string"), 2: ("parameter_type", "string"), 3: ("parameter_schema", "string"), + 4: ("returns_value", "bool"), 5: ("description", "string"), + 6: ("input_schema", "string"), 7: ("output_schema", "string"), + }, + "InvokeResult": {1: ("payload", "bytes"), 2: ("error", ("message", "Error"))}, + "Error": {1: ("type", "string"), 2: ("message", "string"), 3: ("detail", "string")}, + "Event": { + 1: ("payload", "bytes"), 2: ("dedupe_key", "string"), 3: ("content_type", "string"), + 4: ("headers", "map"), 5: ("endpoint", "string"), + }, + "StateRequest": {1: ("op", "int"), 2: ("name", "string"), 3: ("value", "string")}, + "LogEntry": { + 1: ("level", "int"), 2: ("message", "string"), 3: ("exception", "string"), + 4: ("properties", "map"), 5: ("timestamp_unix_ms", "int"), + }, + "Metric": {1: ("name", "string"), 2: ("value", "double"), 3: ("tags", "map")}, + "Pong": { + 1: ("connected", "bool"), 2: ("state", "string"), 3: ("last_message_unix_ms", "int"), + 4: ("in_flight", "int"), 5: ("last_error", "string"), 6: ("details", "map"), + }, +} + +_BY_NAME = {msg: {name: (number, kind) for number, (name, kind) in fields.items()} for msg, fields in SCHEMAS.items()} + + +# ---------------------------------------------------------------------------- primitives + +def _varint(value): + if value < 0: + value += 1 << 64 # int32/int64 negatives are ten-byte two's complement + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + if value: + out.append(byte | 0x80) + else: + out.append(byte) + return bytes(out) + + +def _read_varint(data, pos): + result = 0 + shift = 0 + while True: + if pos >= len(data): + raise WireError("truncated varint") + byte = data[pos] + pos += 1 + result |= (byte & 0x7F) << shift + if not byte & 0x80: + return result, pos + shift += 7 + if shift > 63: + raise WireError("varint too long") + + +def _signed(value): + return value - (1 << 64) if value >= 1 << 63 else value + + +def _key(number, wire_type): + return _varint((number << 3) | wire_type) + + +def _length_delimited(number, payload): + return _key(number, LENGTH) + _varint(len(payload)) + payload + + +# ---------------------------------------------------------------------------- encode + +def encode(message_name, message): + fields = _BY_NAME[message_name] + out = bytearray() + for name, value in message.items(): + if value is None or name not in fields: + continue + number, kind = fields[name] + out += _encode_field(number, kind, value) + return bytes(out) + + +def _encode_field(number, kind, value): + if isinstance(kind, tuple): + tag, inner = kind + if tag == "message": + return _length_delimited(number, encode(inner, value)) + if tag == "repeated": + return b"".join(_encode_field(number, inner, item) for item in value) + if tag == "repeated_message": + return b"".join(_length_delimited(number, encode(inner, item)) for item in value) + raise WireError(f"unknown kind {kind}") + # proto3: default values are not written + if kind == "int": + return _key(number, VARINT) + _varint(int(value)) if value else b"" + if kind == "bool": + return _key(number, VARINT) + b"\x01" if value else b"" + if kind == "double": + return _key(number, FIXED64) + struct.pack("> 3, key & 7 + if wire_type == VARINT: + raw, pos = _read_varint(data, pos) + elif wire_type == FIXED64: + raw = bytes(data[pos:pos + 8]) + pos += 8 + elif wire_type == FIXED32: + raw = bytes(data[pos:pos + 4]) + pos += 4 + elif wire_type == LENGTH: + length, pos = _read_varint(data, pos) + raw = bytes(data[pos:pos + length]) + if len(raw) != length: + raise WireError("truncated field") + pos += length + else: + raise WireError(f"unsupported wire type {wire_type}") + + if number not in fields: + continue # a field from a newer host + name, kind = fields[number] + _decode_field(message, name, kind, raw) + return message + + +def _decode_field(message, name, kind, raw): + if isinstance(kind, tuple): + tag, inner = kind + if tag == "message": + message[name] = decode(inner, raw) + elif tag == "repeated": + message.setdefault(name, []).append(_scalar(inner, raw)) + elif tag == "repeated_message": + message.setdefault(name, []).append(decode(inner, raw)) + return + if kind in ("map", "map_int"): + entry = decode("_MapEntry" if kind == "map" else "_MapIntEntry", raw) + message.setdefault(name, {})[entry.get("key", "")] = entry.get("value", 0 if kind == "map_int" else "") + return + message[name] = _scalar(kind, raw) + + +def _scalar(kind, raw): + if kind == "int": + return _signed(raw) + if kind == "bool": + return bool(raw) + if kind == "double": + return struct.unpack(" str: + return "sent" + + @sw.command + def ping(self) -> None: + pass + + +class DescribeTests(unittest.TestCase): + def setUp(self): + _adapter._settings.clear() + + def test_describes_settings_commands_and_lifecycle(self): + described = sw.describe(Example) + self.assertEqual("python", described["sdkLanguage"]) + self.assertEqual("classic", described["lifecycle"]) + self.assertEqual({"min": 2, "max": 2}, described["protocol"]) + settings = {s["name"]: s for s in described["settings"]} + self.assertTrue(settings["Url"]["required"]) + self.assertFalse(settings["Retries"]["required"]) + self.assertEqual("3", settings["Retries"]["default"]) + self.assertEqual("number", settings["Retries"]["type"]) + self.assertTrue(settings["Key"]["secret"]) + self.assertFalse(settings["Key"]["required"]) + commands = {c["name"]: c for c in described["commands"]} + self.assertEqual("Sends it", commands["Send"]["description"]) + self.assertEqual("object", commands["Send"]["inputSchema"]["type"]) + self.assertTrue(commands["Send"]["returnsValue"]) + self.assertFalse(commands["ping"]["returnsValue"]) + self.assertIsNone(commands["ping"]["inputSchema"]) + json.dumps(described) + + def test_an_adapter_with_a_start_hook_is_resident(self): + class Listener: + async def start(self): + pass + self.assertEqual("resident", sw.describe(Listener)["lifecycle"]) + + def test_an_adapter_that_cannot_be_built_is_still_described_with_a_warning(self): + class NeedsArgs: + def __init__(self, required): + pass + described = sw.describe(NeedsArgs) + self.assertEqual(1, len(described["warnings"])) + + def test_a_command_takes_at_most_one_argument(self): + class TwoArgs: + @sw.command + def both(self, a: str, b: str) -> str: + return a + b + with self.assertRaises(TypeError): + _adapter.commands_of(TwoArgs()) + + def test_values_come_from_the_call_then_startup_then_the_default(self): + sw.expect("Mode", "fast") + self.assertEqual("fast", sw.value_of("Mode")) + _adapter._startup_values["Mode"] = "slow" + try: + self.assertEqual("slow", sw.value_of("Mode")) + token = _adapter._call_values.set({"Mode": "per-call"}) + self.assertEqual("per-call", sw.value_of("Mode")) + _adapter._call_values.reset(token) + finally: + _adapter._startup_values.clear() + self.assertIsNone(sw.value_of("Missing")) + self.assertEqual("x", sw.value_of("Missing", "x")) + + +if __name__ == "__main__": + unittest.main() diff --git a/sdk/python/tests/test_wire.py b/sdk/python/tests/test_wire.py new file mode 100644 index 0000000..60bb5ee --- /dev/null +++ b/sdk/python/tests/test_wire.py @@ -0,0 +1,62 @@ +import unittest + +from simplyworks_serverless import _hpack, _wire + + +class WireTests(unittest.TestCase): + def test_a_frame_round_trips_with_every_kind_of_field(self): + frame = {"id": 42, "hello": { + "token": "t", "protocol_version": 2, "capabilities": ["cancel", "command:Greet"], + "commands": [{"name": "Greet", "returns_value": True, "input_schema": '{"type":"string"}'}], + "settings": [{"name": "Url", "required": True, "secret": True}], + "kinds": ["handler"], "contracts": {"bitween": 1}}} + decoded = _wire.decode("AdapterFrame", _wire.encode("AdapterFrame", frame)) + self.assertEqual(frame, decoded) + + def test_maps_doubles_bytes_and_negative_numbers(self): + frame = {"id": -7, "metric": {"name": "m", "value": 2.5, "tags": {"a": "b", "c": ""}}} + self.assertEqual(frame, _wire.decode("AdapterFrame", _wire.encode("AdapterFrame", frame))) + result = {"invoke_result": {"payload": b"\x00\xff"}} + self.assertEqual(result, _wire.decode("AdapterFrame", _wire.encode("AdapterFrame", result))) + + def test_an_empty_message_is_present(self): + # Ping and Cancel carry nothing: present is all they say. + data = _wire._key(5, _wire.LENGTH) + b"\x00" + self.assertEqual({"ping": {}}, _wire.decode("HostFrame", data)) + + def test_fields_from_a_newer_host_are_skipped(self): + unknown = _wire._key(99, _wire.LENGTH) + b"\x03abc" + _wire._key(98, _wire.VARINT) + b"\x05" + data = unknown + _wire.encode("HostFrame", {"id": 3}) + self.assertEqual({"id": 3}, _wire.decode("HostFrame", data)) + + def test_default_values_are_not_written(self): + self.assertEqual(b"", _wire.encode("Pong", {"connected": False, "state": "", "in_flight": 0})) + + +class HpackTests(unittest.TestCase): + def test_rfc7541_huffman_examples(self): + # RFC 7541, C.4.1 and C.6.1. + self.assertEqual(b"www.example.com", _hpack.huffman_decode(bytes.fromhex("f1e3c2e5f23a6ba0ab90f4ff"))) + self.assertEqual(b"Mon, 21 Oct 2013 20:13:21 GMT", + _hpack.huffman_decode(bytes.fromhex("d07abe941054d444a8200595040b8166e082a62d1bff"))) + + def test_rfc7541_response_sequence_with_huffman_and_the_dynamic_table(self): + # RFC 7541, C.6: three responses on one decoder, the table evicting as it fills. + decoder = _hpack.Decoder(max_size=256) + first = decoder.decode(bytes.fromhex( + "488264025885aec3771a4b6196d07abe941054d444a8200595040b8166e082a62d1bff" + "6e919d29ad171863c78f0b97c8e9ae82ae43d3")) + self.assertEqual([(":status", "302"), ("cache-control", "private"), + ("date", "Mon, 21 Oct 2013 20:13:21 GMT"), ("location", "https://www.example.com")], first) + second = decoder.decode(bytes.fromhex("4883640effc1c0bf")) + self.assertEqual(":status", second[0][0]) + self.assertEqual("307", second[0][1]) + self.assertEqual(("location", "https://www.example.com"), second[3]) + + def test_literals_encode_and_decode(self): + headers = [(":path", "/sw.serverless.v1.AdapterHost/Attach"), ("te", "trailers")] + self.assertEqual(headers, _hpack.Decoder().decode(_hpack.encode(headers))) + + +if __name__ == "__main__": + unittest.main() From f5eb087779a9d419e4fc36311e24f4bba4de48c7 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 15:59:37 +0300 Subject: [PATCH 02/17] Run Python adapters through the real host in the tests Classic sessions as Bitween runs handlers, a resident instance with status, events, state, reset and drain, payloads well past the HTTP/2 window both ways, errors with their types, a cancelled call, concurrent calls on one instance, --describe, and the Bitween handler, validator and receiver written with simplyworks-bitween. --- SW.Serverless.UnitTests/PythonAdapterTests.cs | 388 ++++++++++++++++++ .../PythonAdapters/bitween_handler.py | 21 + .../PythonAdapters/bitween_receiver.py | 40 ++ .../PythonAdapters/bitween_validator.py | 20 + .../PythonAdapters/classic.py | 67 +++ .../PythonAdapters/resident.py | 48 +++ 6 files changed, 584 insertions(+) create mode 100644 SW.Serverless.UnitTests/PythonAdapterTests.cs create mode 100644 SW.Serverless.UnitTests/PythonAdapters/bitween_handler.py create mode 100644 SW.Serverless.UnitTests/PythonAdapters/bitween_receiver.py create mode 100644 SW.Serverless.UnitTests/PythonAdapters/bitween_validator.py create mode 100644 SW.Serverless.UnitTests/PythonAdapters/classic.py create mode 100644 SW.Serverless.UnitTests/PythonAdapters/resident.py diff --git a/SW.Serverless.UnitTests/PythonAdapterTests.cs b/SW.Serverless.UnitTests/PythonAdapterTests.cs new file mode 100644 index 0000000..96a7b31 --- /dev/null +++ b/SW.Serverless.UnitTests/PythonAdapterTests.cs @@ -0,0 +1,388 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json.Linq; +using SW.CloudFiles.Extensions; +using SW.PrimitiveTypes; +using SW.Serverless.Resident; +using SW.Serverless.UnitTests.Fixtures; +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace SW.Serverless.UnitTests +{ + /// + /// Adapters written in Python with simplyworks-serverless, run by the real host: classic + /// sessions as Bitween runs handlers, resident instances, and the Bitween kinds written with + /// simplyworks-bitween. The SDK has no dependencies, so a package is the adapter's files and + /// the SDK's, vendored beside them — what serverless build makes. + /// + [TestClass] + public class PythonAdapterTests + { + static IHost host; + static string workDirectory; + + static readonly (string Id, string Script, string Lifecycle)[] Adapters = + { + ("test.python.classic", "classic.py", "classic"), + ("test.python.resident", "resident.py", "resident"), + ("test.python.handler", "bitween_handler.py", "classic"), + ("test.python.receiver", "bitween_receiver.py", "classic"), + ("test.python.validator", "bitween_validator.py", "classic"), + }; + + [ClassInitialize] + public static async Task ClassInitialize(TestContext context) + { + workDirectory = Path.Combine(Path.GetTempPath(), "swsl-python", Guid.NewGuid().ToString("N")); + host = Host.CreateDefaultBuilder() + .ConfigureLogging(l => l.ClearProviders()) + .ConfigureServices(s => + { + s.AddLocalTestsCloudFiles(o => o.BucketName = TestStore.BucketName + "-python"); + s.AddServerless(o => + { + o.AdapterRemotePath = "adapters"; + o.AdapterLocalPath = Path.Combine(workDirectory, "installed"); + o.AdapterMetadataCacheDuration = 1; + o.CommandTimeout = 30; + }); + s.AddSingleton(); + s.AddSingleton(sp => sp.GetRequiredService()); + s.AddResidentAdapters(o => + { + o.SocketPath = $"/tmp/swsl-py{Environment.ProcessId}.sock"; + o.PipeName = $"swsl-py{Environment.ProcessId}"; + o.HeartbeatInterval = TimeSpan.FromSeconds(2); + o.HandshakeTimeout = TimeSpan.FromSeconds(60); + }); + }) + .Build(); + await host.StartAsync(); + + var files = host.Services.GetRequiredService(); + foreach (var (id, script, lifecycle) in Adapters) + { + using var package = PythonPackage.Build(id, script, lifecycle); + await files.WriteAsync(package, new WriteFileSettings + { + Key = $"adapters-versions/{id}/1.0.0", + ContentType = "application/zip", + Metadata = new Dictionary + { + ["EntryAssembly"] = PythonPackage.Entry, + ["Hash"] = "py-" + Guid.NewGuid().ToString("N")[..12], + ["Protocol"] = "2", + ["Lifecycle"] = lifecycle, + } + }); + } + } + + [ClassCleanup] + public static async Task ClassCleanup() + { + if (host != null) await host.StopAsync(); + host?.Dispose(); + try { Directory.Delete(workDirectory, true); } catch { } + } + + static IServerlessService Service() => host.Services.GetRequiredService(); + static IResidentAdapterHost Residents() => host.Services.GetRequiredService(); + + static async Task InSession(string adapterId, IDictionary values, Func> call) + { + var service = Service(); + await service.StartAsync($"{adapterId}/1.0.0", "corr-py", values ?? new Dictionary()); + try { return await call(service); } + finally { ((IDisposable)service).Dispose(); } + } + + [TestMethod] + public async Task A_classic_python_adapter_answers_with_the_values_it_was_started_with() + { + await InSession("test.python.classic", new Dictionary { ["Prefix"] = "hi " }, async s => + { + Assert.AreEqual("hi world", await s.InvokeAsync("Greet", "world")); + Assert.AreEqual("corr-py", await s.InvokeAsync("Correlation", null)); + Assert.AreEqual(5, await s.InvokeAsync("Add", new { A = 2, B = 3 })); + // Unicode survives both ways. + Assert.AreEqual("hi مرحبا ✓", await s.InvokeAsync("Greet", "مرحبا ✓")); + return 0; + }); + } + + [TestMethod] + public async Task Payloads_past_the_http2_window_cross_both_ways() + { + // 64 KB is HTTP/2's default window: these only arrive if flow control works. + await InSession("test.python.classic", null, async s => + { + var big = await s.InvokeAsync("Big", 3 * 1024 * 1024); + Assert.AreEqual(3 * 1024 * 1024, big.Length); + Assert.AreEqual(2 * 1024 * 1024 + 7, await s.InvokeAsync("Length", new string('y', 2 * 1024 * 1024 + 7))); + return 0; + }); + } + + [TestMethod] + public async Task An_error_reaches_the_caller_with_its_type_and_message() + { + await InSession("test.python.classic", null, async s => + { + // Raised as the adapter raised it: its type, as the adapter named it, and its message. + var rejected = await Assert.ThrowsExceptionAsync(() => s.InvokeAsync("Fail", "no stock")); + Assert.AreEqual("Acme.Rejected", rejected.AdapterExceptionType); + StringAssert.Contains(rejected.Message, "no stock"); + + var crashed = await Assert.ThrowsExceptionAsync(() => s.InvokeAsync("Crash", null)); + Assert.AreEqual("KeyError", crashed.AdapterExceptionType); + StringAssert.Contains(crashed.Detail, "Traceback"); + + var unknown = await Assert.ThrowsExceptionAsync(() => s.InvokeAsync("Teleport", null)); + StringAssert.Contains(unknown.Message, "Teleport"); + + // The session is still good after all three. + Assert.AreEqual("x", await s.InvokeAsync("Big", 1)); + return 0; + }); + } + + [TestMethod] + public async Task A_call_that_times_out_is_cancelled_and_the_adapter_keeps_answering() + { + var residents = Residents(); + var instance = await residents.StartExclusiveAsync(new AdapterSpec + { + AdapterId = "test.python.classic/1.0.0", + InstanceKey = "py-cancel", + StartupValues = new Dictionary { ["Prefix"] = "> " }, + }); + try + { + var started = DateTime.UtcNow; + await Assert.ThrowsExceptionAsync(() => instance.InvokeAsync("Slow", 30.0, timeoutSeconds: 1)); + Assert.IsTrue(DateTime.UtcNow - started < TimeSpan.FromSeconds(10)); + + // Several at once on one instance, while the cancelled one is gone. + var answers = await Task.WhenAll(Enumerable.Range(0, 20) + .Select(i => instance.InvokeAsync("Greet", "n" + i, timeoutSeconds: 15))); + CollectionAssert.AreEqual(Enumerable.Range(0, 20).Select(i => "> n" + i).ToArray(), answers); + } + finally + { + await residents.StopAsync("test.python.classic/1.0.0", "py-cancel", drain: false); + } + } + + [TestMethod] + public async Task A_command_with_no_result_completes() + { + await InSession("test.python.classic", null, async s => + { + await s.InvokeAsync("Nothing", null); + return 0; + }); + } + + [TestMethod] + public async Task A_resident_python_adapter_starts_reports_status_publishes_keeps_state_resets_and_stops() + { + var residents = Residents(); + var sink = host.Services.GetRequiredService(); + var instance = await residents.StartExclusiveAsync(new AdapterSpec + { + AdapterId = "test.python.resident/1.0.0", + InstanceKey = "py-resident", + }); + try + { + Assert.AreEqual(InstanceState.Ready, instance.State); + Assert.IsTrue(await instance.InvokeAsync("Started", timeoutSeconds: 15)); + + var described = residents.Describe().Single(d => d.InstanceKey == "py-resident"); + Assert.AreEqual("python", described.SdkLanguage); + + var pong = await instance.PingAsync(TimeSpan.FromSeconds(10)); + Assert.AreEqual("Listening", pong.State); + Assert.IsTrue(pong.Connected); + + var reference = await instance.InvokeAsync("Publish", "order-1", timeoutSeconds: 15); + StringAssert.StartsWith(reference, "ref-"); + Assert.IsTrue(sink.Delivered.Any(d => d.Body == "order-1" && d.DedupeKey == "k-order-1" && d.Endpoint == "tests")); + + // An empty string is an empty payload, which reads back as null — as from a .NET adapter. + Assert.IsTrue(string.IsNullOrEmpty(await instance.InvokeAsync("Remember", "first", timeoutSeconds: 15))); + Assert.AreEqual("first", await instance.InvokeAsync("Remember", "second", timeoutSeconds: 15)); + await instance.InvokeAsync("Forget", timeoutSeconds: 15); + Assert.IsTrue(string.IsNullOrEmpty(await instance.InvokeAsync("Remember", "third", timeoutSeconds: 15))); + + await instance.ResetAsync("session-9"); + CollectionAssert.AreEqual(new[] { "session-9" }, await instance.InvokeAsync("Resets", timeoutSeconds: 15)); + } + finally + { + await residents.StopAsync("test.python.resident/1.0.0", "py-resident", drain: true); + } + Assert.IsFalse(residents.Describe().Any(d => d.InstanceKey == "py-resident")); + } + + [TestMethod] + public async Task A_bitween_handler_in_python_takes_and_returns_exchange_files_as_dotnet_ones() + { + var answer = await InSession("test.python.handler", new Dictionary { ["Partner"] = "acme" }, s => + s.InvokeAsync("Handle", new { Data = "{\"orderId\":\"SO-1\"}", Filename = "order.json", BadData = false })); + + Assert.AreEqual("answer.json", (string)answer["Filename"]); + Assert.IsFalse((bool)answer["BadData"]); + var data = JObject.Parse((string)answer["Data"]); + Assert.AreEqual("acme", (string)data["to"]); + Assert.AreEqual("SO-1", (string)data["orderId"]); + Assert.AreEqual("order.json", (string)data["from"]); + // Hash is what .NET's ExchangeFile computes: SHA-1 of Data, lower-case hex. + using var sha1 = System.Security.Cryptography.SHA1.Create(); + Assert.AreEqual(Convert.ToHexString(sha1.ComputeHash(Encoding.UTF8.GetBytes((string)answer["Data"]))).ToLowerInvariant(), + (string)answer["Hash"]); + + var rejected = await InSession("test.python.handler", new Dictionary { ["Partner"] = "acme" }, s => + s.InvokeAsync("Handle", new { Data = "{\"reject\":true}" })); + Assert.IsTrue((bool)rejected["BadData"], "a rejected delivery is returned, not raised"); + } + + [TestMethod] + public async Task A_bitween_validator_in_python_reports_each_failure() + { + var result = await InSession("test.python.validator", null, s => + s.InvokeAsync("Validate", new { Data = "{\"lines\":[]}" })); + + Assert.IsFalse((bool)result["Success"]); + CollectionAssert.AreEqual(new[] { "orderId", "lines" }, + result["Validations"]!.Select(v => (string)v["Key"]).ToArray()); + + var valid = await InSession("test.python.validator", null, s => + s.InvokeAsync("Validate", new { Data = "{\"orderId\":\"SO-1\",\"lines\":[1]}" })); + Assert.IsTrue((bool)valid["Success"]); + } + + [TestMethod] + public async Task A_bitween_receiver_in_python_runs_a_session_in_the_contract_s_order() + { + var root = Path.Combine(workDirectory, "receiver"); + var folder = Path.Combine(root, "inbox"); + Directory.CreateDirectory(folder); + File.WriteAllText(Path.Combine(folder, "a.json"), "{\"n\":1}"); + File.WriteAllText(Path.Combine(folder, "b.json"), "{\"n\":2}"); + + var taken = await InSession("test.python.receiver", new Dictionary { ["Folder"] = folder }, async s => + { + var got = new List(); + await s.InvokeAsync("Initialize", null); + foreach (var id in await s.InvokeAsync("ListFiles", null)) + { + var file = await s.InvokeAsync("GetFile", id); + got.Add((string)file["Filename"] + "=" + (string)file["Data"]); + await s.InvokeAsync("DeleteFile", id); + } + await s.InvokeAsync("Finalize", null); + return got; + }); + + CollectionAssert.AreEqual(new[] { "a.json={\"n\":1}", "b.json={\"n\":2}" }, taken); + Assert.AreEqual(0, Directory.GetFiles(folder).Length); + Assert.AreEqual("Initialize,ListFiles,GetFile,DeleteFile,GetFile,DeleteFile,Finalize", + File.ReadAllText(Path.Combine(root, "calls.txt"))); + } + + [TestMethod] + public async Task A_python_adapter_describes_itself_for_the_manifest() + { + var dir = Path.Combine(workDirectory, "describe"); + using (var package = PythonPackage.Build("test.python.handler", "bitween_handler.py", "classic")) + using (var archive = new ZipArchive(package)) + archive.ExtractToDirectory(dir); + + var (description, problem) = await Tooling.LocalAdapterHost.DescribeAsync( + Path.Combine(dir, PythonPackage.Entry), "python"); + + Assert.IsNotNull(description, problem); + Assert.AreEqual("python", description.SdkLanguage); + Assert.AreEqual("classic", description.Lifecycle); + Assert.AreEqual(2, description.Protocol.Min); + CollectionAssert.AreEqual(new[] { "handler" }, description.Kinds); + Assert.AreEqual(1, description.Contracts["bitween"]); + var partner = description.Settings.Single(); + Assert.AreEqual("Partner", partner.Name); + Assert.IsTrue(partner.Required); + var handle = description.Commands.Single(); + Assert.AreEqual("Handle", handle.Name); + Assert.AreEqual("ExchangeFile", handle.InputSchema!.Value.GetProperty("title").GetString()); + } + } + + /// + /// A Python adapter packaged as serverless build packages one: the script as main.py, the SDKs + /// under _vendor/, and an entry that puts _vendor on the path before running main.py. + /// + static class PythonPackage + { + public const string Entry = "_serverless_entry.py"; + + const string Bootstrap = """ + import os, runpy, sys + here = os.path.dirname(os.path.abspath(__file__)) + sys.path.insert(0, os.path.join(here, "_vendor")) + sys.path.insert(0, here) + runpy.run_path(os.path.join(here, "main.py"), run_name="__main__") + """; + + public static MemoryStream Build(string id, string script, string lifecycle) + { + var root = RepositoryRoot(); + var buffer = new MemoryStream(); + using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) + { + void AddText(string name, string text) + { + using var writer = new StreamWriter(archive.CreateEntry(name).Open()); + writer.Write(text); + } + + void AddFolder(string folder, string under) + { + foreach (var file in Directory.EnumerateFiles(folder, "*.py", SearchOption.AllDirectories)) + archive.CreateEntryFromFile(file, under + "/" + Path.GetRelativePath(folder, file).Replace('\\', '/')); + } + + archive.CreateEntryFromFile(Path.Combine(root, "SW.Serverless.UnitTests", "PythonAdapters", script), "main.py"); + AddText(Entry, Bootstrap); + AddFolder(Path.Combine(root, "sdk", "python", "src", "simplyworks_serverless"), "_vendor/simplyworks_serverless"); + AddFolder(Path.Combine(root, "..", "Bitween-api", "sdk", "python", "src", "simplyworks_bitween"), "_vendor/simplyworks_bitween"); + AddText("adapter.json", new JObject + { + ["id"] = id, + ["version"] = "1.0.0", + ["runtime"] = "python", + ["lifecycle"] = lifecycle, + ["protocol"] = new JObject { ["min"] = 2, ["max"] = 2 }, + ["entry"] = Entry, + }.ToString()); + } + buffer.Position = 0; + return buffer; + } + + static string RepositoryRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir != null && !File.Exists(Path.Combine(dir.FullName, "SW.Serverless.sln"))) dir = dir.Parent; + return dir!.FullName; + } + } +} diff --git a/SW.Serverless.UnitTests/PythonAdapters/bitween_handler.py b/SW.Serverless.UnitTests/PythonAdapters/bitween_handler.py new file mode 100644 index 0000000..5187d45 --- /dev/null +++ b/SW.Serverless.UnitTests/PythonAdapters/bitween_handler.py @@ -0,0 +1,21 @@ +"""A Bitween handler, mapper-free: the contract's Handle, written with simplyworks_bitween.""" +import json + +import simplyworks_serverless as sw +from simplyworks_bitween import ExchangeFile, Handler + + +class Orders(Handler): + def __init__(self): + sw.expect("Partner", description="Who receives the orders") + + def handle(self, file: ExchangeFile) -> ExchangeFile: + order = json.loads(file.data) + if order.get("reject"): + return ExchangeFile(data='{"error":"rejected"}', bad_data=True, content_type="application/json") + answer = {"to": sw.value_of("Partner"), "orderId": order["orderId"], "from": file.filename} + return ExchangeFile(data=json.dumps(answer), filename="answer.json", content_type="application/json") + + +if __name__ == "__main__": + sw.run(Orders) diff --git a/SW.Serverless.UnitTests/PythonAdapters/bitween_receiver.py b/SW.Serverless.UnitTests/PythonAdapters/bitween_receiver.py new file mode 100644 index 0000000..d9c042a --- /dev/null +++ b/SW.Serverless.UnitTests/PythonAdapters/bitween_receiver.py @@ -0,0 +1,40 @@ +"""A Bitween receiver over a folder, written with simplyworks_bitween.""" +import os + +import simplyworks_serverless as sw +from simplyworks_bitween import ExchangeFile, Receiver + + +class Folder(Receiver): + def __init__(self): + sw.expect("Folder") + self.calls = [] + + @property + def folder(self): + return sw.value_of("Folder") + + def initialize(self): + self.calls.append("Initialize") + + def list_files(self): + self.calls.append("ListFiles") + return sorted(os.listdir(self.folder)) + + def get_file(self, file_id): + self.calls.append("GetFile") + with open(os.path.join(self.folder, file_id), encoding="utf-8") as f: + return ExchangeFile(data=f.read(), filename=file_id) + + def delete_file(self, file_id): + self.calls.append("DeleteFile") + os.remove(os.path.join(self.folder, file_id)) + + def finalize(self): + self.calls.append("Finalize") + with open(os.path.join(self.folder, "..", "calls.txt"), "w", encoding="utf-8") as f: + f.write(",".join(self.calls)) + + +if __name__ == "__main__": + sw.run(Folder) diff --git a/SW.Serverless.UnitTests/PythonAdapters/bitween_validator.py b/SW.Serverless.UnitTests/PythonAdapters/bitween_validator.py new file mode 100644 index 0000000..acbc60d --- /dev/null +++ b/SW.Serverless.UnitTests/PythonAdapters/bitween_validator.py @@ -0,0 +1,20 @@ +"""A Bitween validator: an order needs an id and at least one line.""" +import json + +import simplyworks_serverless as sw +from simplyworks_bitween import ExchangeFile, ValidationResult, Validator + + +class Orders(Validator): + def validate(self, file: ExchangeFile) -> ValidationResult: + order = json.loads(file.data) + result = ValidationResult() + if not order.get("orderId"): + result.add("orderId", "An order needs an id.") + if not order.get("lines"): + result.add("lines", "An order needs at least one line.") + return result + + +if __name__ == "__main__": + sw.run(Orders) diff --git a/SW.Serverless.UnitTests/PythonAdapters/classic.py b/SW.Serverless.UnitTests/PythonAdapters/classic.py new file mode 100644 index 0000000..5f7deea --- /dev/null +++ b/SW.Serverless.UnitTests/PythonAdapters/classic.py @@ -0,0 +1,67 @@ +"""A classic adapter in Python, called as Bitween calls handlers: one session, one call at a time.""" +import asyncio +import logging +from dataclasses import dataclass + +import simplyworks_serverless as sw + + +@dataclass +class Sum: + A: int + B: int + + +class Classic: + def __init__(self): + sw.expect("Prefix", description="Put before every greeting") + sw.expect("Secret", "s3cret", secret=True) + + @sw.command("Greet", description="Greets someone") + def greet(self, name: str) -> str: + return sw.value_of("Prefix") + name + + @sw.command("Correlation") + def correlation(self) -> str: + return sw.value_of("CorrelationId") + + @sw.command("Add") + async def add(self, numbers: Sum) -> int: + return numbers.A + numbers.B + + @sw.command("Fail") + def fail(self, message: str) -> str: + raise sw.AdapterError(message, type="Acme.Rejected") + + @sw.command("Crash") + def crash(self) -> str: + return {}["missing"] + + @sw.command("Big") + def big(self, size: int) -> str: + return "x" * size + + @sw.command("Length") + def length(self, text: str) -> int: + return len(text) + + @sw.command("Bytes") + def bytes_(self, data: bytes) -> bytes: + return bytes(reversed(data)) + + @sw.command("Nothing") + def nothing(self) -> None: + logging.getLogger("classic").info("did nothing, as asked") + + @sw.command("Slow") + async def slow(self, seconds: float) -> str: + await asyncio.sleep(seconds) + return "finished" + + @sw.command("Property") + def property_(self, name: str) -> str: + return sw.value_of(name, "") + + +if __name__ == "__main__": + sw.run(Classic) diff --git a/SW.Serverless.UnitTests/PythonAdapters/resident.py b/SW.Serverless.UnitTests/PythonAdapters/resident.py new file mode 100644 index 0000000..29731e2 --- /dev/null +++ b/SW.Serverless.UnitTests/PythonAdapters/resident.py @@ -0,0 +1,48 @@ +"""A resident adapter in Python: started, kept running, asked for its status, reset and stopped.""" +import simplyworks_serverless as sw + + +class Resident: + def __init__(self): + self.started = False + self.resets = [] + sw.expect("Name", "resident") + + async def start(self): + self.started = True + + async def stop(self): + self.started = False + + def status(self): + return {"connected": self.started, "state": "Listening", "details": {"resets": str(len(self.resets))}} + + def reset(self, session_id): + self.resets.append(session_id) + + @sw.command("Started") + def is_started(self) -> bool: + return self.started + + @sw.command("Publish") + async def publish(self, text: str) -> str: + return await sw.context().publish(text, dedupe_key="k-" + text, content_type="text/plain", endpoint="tests") + + @sw.command("Remember") + async def remember(self, value: str) -> str: + ctx = sw.context() + before = await ctx.get_state("memory") + await ctx.set_state("memory", value) + return before or "" + + @sw.command("Forget") + async def forget(self) -> None: + await sw.context().delete_state("memory") + + @sw.command("Resets") + def resets_(self) -> list[str]: + return self.resets + + +if __name__ == "__main__": + sw.run(Resident) From 7560d2b7d39b243f8d1100f1f492d0c932e3d4ce Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 15:59:37 +0300 Subject: [PATCH 03/17] Build, scaffold, test and run Python adapters with the CLI serverless init --lang python writes a working adapter of any Bitween kind. serverless build packages a Python adapter: its files as they are, the SDK and the Bitween kinds vendored from copies the CLI carries, requirements.txt vendored with pip, and an entry that puts them on the path. Pure-Python requirements are vendored once and the package runs anywhere; native ones are vendored per target platform, linux-x64 and linux-arm64 unless adapter.json names others, and the manifest lists them. The adapter describes itself on the build machine, with its requirements installed there only for that when it isn't a target. test and run build a Python project folder first, as a .NET one. Scaffolded files end in a newline. --- .../CliCommandTests.cs | 62 +++- SW.Serverless.Installer/AdapterCommands.cs | 20 +- .../Building/PackageBuilder.cs | 20 +- SW.Serverless.Tooling/Building/PythonBuild.cs | 309 ++++++++++++++++++ .../python/simplyworks_bitween/__init__.py | 236 +++++++++++++ .../SW.Serverless.Tooling.csproj | 8 + .../Scaffolding/Scaffolder.cs | 155 ++++++++- 7 files changed, 797 insertions(+), 13 deletions(-) create mode 100644 SW.Serverless.Tooling/Building/PythonBuild.cs create mode 100644 SW.Serverless.Tooling/Contracts/bitween/python/simplyworks_bitween/__init__.py diff --git a/SW.Serverless.Installer.UnitTests/CliCommandTests.cs b/SW.Serverless.Installer.UnitTests/CliCommandTests.cs index f894bb2..b5394b5 100644 --- a/SW.Serverless.Installer.UnitTests/CliCommandTests.cs +++ b/SW.Serverless.Installer.UnitTests/CliCommandTests.cs @@ -67,9 +67,9 @@ public async Task Init_makes_a_project_and_refuses_what_it_can_t_make() StringAssert.Contains(File.ReadAllText(Path.Combine(project, "Program.cs")), "IBitweenValidator"); Assert.AreEqual(Program.Failure, (await Cli("init", "AcmeOrders", "--dir", work)).Exit, "an existing project isn't overwritten"); - var (pythonExit, pythonOutput) = await Cli("init", "PyOrders", "--lang", "python", "--dir", work); - Assert.AreEqual(Program.Failure, pythonExit); - StringAssert.Contains(pythonOutput, "arrives with that language's SDK"); + var (nodeExit, nodeOutput) = await Cli("init", "NodeOrders", "--lang", "node", "--dir", work); + Assert.AreEqual(Program.Failure, nodeExit); + StringAssert.Contains(nodeOutput, "arrives with that language's SDK"); } [TestMethod] @@ -186,4 +186,60 @@ public async Task What_init_writes_builds_and_conforms(string kind) var test = await Cli("test", Path.Combine(project, "bin", "serverless", "package"), "--settings", settings); Assert.AreEqual(Program.Success, test.Exit, test.Output); } + + /// + /// A Python adapter, from init to a package that conforms: built with the SDK and the Bitween + /// kinds vendored from the copies the CLI carries, so no PyPI and no network are needed. + /// + [DataTestMethod] + [DataRow("handler")] + [DataRow("mapper")] + [DataRow("receiver")] + [DataRow("validator")] + public async Task What_init_writes_in_python_builds_and_conforms(string kind) + { + var work = WorkFolder(); + var name = "Py" + char.ToUpper(kind[0]) + kind[1..]; + Assert.AreEqual(Program.Success, (await Cli("init", name, "--lang", "python", "--kind", kind, "--dir", work)).Exit); + var project = Path.Combine(work, name); + Assert.IsTrue(File.Exists(Path.Combine(project, "main.py"))); + + var build = await Cli("build", project); + Assert.AreEqual(Program.Success, build.Exit, build.Output); + + var package = Path.Combine(project, "bin", "serverless", "package"); + var manifest = SW.Serverless.Contract.Catalog.AdapterManifest.Parse(File.ReadAllText(Path.Combine(package, "adapter.json"))); + Assert.AreEqual("python", manifest.Runtime); + Assert.AreEqual(Tooling.Building.PythonBuild.EntryScript, manifest.Entry); + Assert.AreEqual("classic", manifest.Lifecycle); + Assert.AreEqual(2, manifest.Protocol.Min); + CollectionAssert.AreEqual(new[] { kind }, manifest.Kinds); + Assert.AreEqual(1, manifest.Contracts["bitween"]); + Assert.IsNull(manifest.Platforms, "nothing native: it runs anywhere"); + Assert.IsTrue(File.Exists(Path.Combine(package, "_vendor", "simplyworks_serverless", "__init__.py"))); + Assert.IsTrue(File.Exists(Path.Combine(package, "_vendor", "simplyworks_bitween", "__init__.py"))); + Assert.IsTrue(manifest.Source.Files.ContainsKey("main.py")); + Assert.IsTrue(File.Exists(Path.Combine(package, "source", "main.py"))); + + var settings = Path.Combine(work, "settings.json"); + File.WriteAllText(settings, """{ "ApiKey": "k" }"""); + // The project folder: built first, then checked. + var test = await Cli("test", project, "--settings", settings); + Assert.AreEqual(Program.Success, test.Exit, test.Output); + } + + /// + /// The CLI vendors its own copy of the Bitween kinds for Python, as it carries its own copy of + /// the contract; it must be the one Bitween-api maintains. + /// + [TestMethod] + public void The_python_bitween_kinds_the_cli_carries_are_bitween_s() + { + var root = RepositoryRoot(); + var original = Path.GetFullPath(Path.Combine(root, "..", "Bitween-api", "sdk", "python", "src", "simplyworks_bitween", "__init__.py")); + if (!File.Exists(original)) Assert.Inconclusive($"Bitween-api isn't beside this repository ({original})"); + var copy = Path.Combine(root, "SW.Serverless.Tooling", "Contracts", "bitween", "python", "simplyworks_bitween", "__init__.py"); + Assert.AreEqual(File.ReadAllText(original), File.ReadAllText(copy), + "copy Bitween-api/sdk/python/src/simplyworks_bitween into SW.Serverless.Tooling/Contracts/bitween/python"); + } } diff --git a/SW.Serverless.Installer/AdapterCommands.cs b/SW.Serverless.Installer/AdapterCommands.cs index b2be99f..40a063f 100644 --- a/SW.Serverless.Installer/AdapterCommands.cs +++ b/SW.Serverless.Installer/AdapterCommands.cs @@ -21,7 +21,7 @@ public class InitCliOptions [Value(0, Required = true, MetaName = "name", HelpText = "The adapter's name, e.g. AcmeOrders: its folder, project and class.")] public string Name { get; set; } - [Option("lang", Default = "dotnet", HelpText = "Its language: dotnet. Python, Node and Go arrive with their SDKs.")] + [Option("lang", Default = "dotnet", HelpText = "Its language: dotnet or python. Node and Go arrive with their SDKs.")] public string Language { get; set; } [Option("kind", Default = "handler", HelpText = "handler, mapper, validator or receiver.")] @@ -285,7 +285,7 @@ public static Task Publish(PublishPackageCliOptions opts, Func TryDelete(folder)); } - if (Directory.Exists(path) && Directory.GetFiles(path, "*.*proj").Length > 0) + if (Directory.Exists(path) && (Directory.GetFiles(path, "*.*proj").Length > 0 || IsUnbuiltPython(path))) { var output = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "swsl-cli", Guid.NewGuid().ToString("N")); var built = await PackageBuilder.BuildAsync(new BuildRequest { ProjectDirectory = path, OutputDirectory = output, Log = Console.WriteLine }); @@ -300,6 +300,22 @@ public static Task Publish(PublishPackageCliOptions opts, Func { }); } + /// A Python project rather than a built package: its manifest names Python, and no build has written the entry. + static bool IsUnbuiltPython(string folder) + { + var manifestPath = System.IO.Path.Combine(folder, AdapterManifest.FileName); + if (!File.Exists(manifestPath) || File.Exists(System.IO.Path.Combine(folder, PythonBuild.EntryScript))) return false; + try + { + return string.Equals(AdapterManifest.Parse(File.ReadAllText(manifestPath)).Runtime, AdapterManifest.PythonRuntime, + StringComparison.OrdinalIgnoreCase); + } + catch (JsonException) + { + return false; + } + } + static IDictionary ReadSettings(string path) { if (string.IsNullOrWhiteSpace(path)) return new Dictionary(); diff --git a/SW.Serverless.Tooling/Building/PackageBuilder.cs b/SW.Serverless.Tooling/Building/PackageBuilder.cs index 01c7ce5..c4d60f4 100644 --- a/SW.Serverless.Tooling/Building/PackageBuilder.cs +++ b/SW.Serverless.Tooling/Building/PackageBuilder.cs @@ -79,9 +79,14 @@ public static async Task BuildAsync(BuildRequest request) } var runtime = string.IsNullOrWhiteSpace(author.Runtime) ? AdapterManifest.DotnetRuntime : author.Runtime; + if (string.Equals(runtime, AdapterManifest.PythonRuntime, StringComparison.OrdinalIgnoreCase)) + { + await PythonBuild.BuildAsync(request, project, author, result); + return result; + } if (!string.Equals(runtime, AdapterManifest.DotnetRuntime, StringComparison.OrdinalIgnoreCase)) { - result.Problems.Add($"building a '{runtime}' adapter arrives with that language's SDK; this build does .NET"); + result.Problems.Add($"building a '{runtime}' adapter arrives with that language's SDK; this build does .NET and Python"); return result; } @@ -137,7 +142,7 @@ public static async Task BuildAsync(BuildRequest request) manifest.Source = new AdapterSource { BuildCommand = "dotnet publish -c Release", - Lockfiles = source.Keys.Where(k => Lockfiles.Contains(Path.GetFileName(k), StringComparer.OrdinalIgnoreCase)).OrderBy(k => k).ToList(), + Lockfiles = source.Keys.Where(IsLockfile).OrderBy(k => k).ToList(), }; foreach (var (relative, absolute) in source.OrderBy(s => s.Key, StringComparer.Ordinal)) { @@ -216,6 +221,9 @@ internal static AdapterManifest ManifestFrom(AdapterManifest author, bool author return manifest; } + internal static bool IsLockfile(string path) => + Lockfiles.Contains(Path.GetFileName(path), StringComparer.OrdinalIgnoreCase); + /// /// Whether the author's own adapter.json says "lifecycle": "classic" — read from the file, /// since the model fills in classic when nothing is said. @@ -235,11 +243,13 @@ internal static bool AuthorChoseClassic(string authorJson) /// The source to carry, keyed by its path in the package's source folder: the project and /// the local projects it references, under the ignore rules, scanned for secrets. /// - static Dictionary CollectSource(string project, string projectFile, BuildRequest request, BuildResult result) + internal static Dictionary CollectSource(string project, string projectFile, BuildRequest request, BuildResult result) { var roots = new List { project }; - foreach (var referenced in LocalReferences(projectFile, new HashSet(StringComparer.Ordinal))) - if (!roots.Contains(referenced)) roots.Add(referenced); + // A .NET project's local project references come along; other languages have none to follow. + if (projectFile != null) + foreach (var referenced in LocalReferences(projectFile, new HashSet(StringComparer.Ordinal))) + if (!roots.Contains(referenced)) roots.Add(referenced); var common = CommonAncestor(roots); var repository = RepositoryRoot(project); diff --git a/SW.Serverless.Tooling/Building/PythonBuild.cs b/SW.Serverless.Tooling/Building/PythonBuild.cs new file mode 100644 index 0000000..d3ff6b5 --- /dev/null +++ b/SW.Serverless.Tooling/Building/PythonBuild.cs @@ -0,0 +1,309 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Reflection; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using SW.Serverless.Contract.Catalog; + +namespace SW.Serverless.Tooling.Building +{ + /// + /// serverless build for a Python adapter. Python runs from its source, so the package is the + /// adapter's files as they are, with what they import vendored under : + /// the SDK and the Bitween kinds from copies this tool carries — no PyPI, no network — and + /// whatever requirements.txt names, through pip. A small entry script puts the vendored code on + /// the path and runs the adapter's own entry. + /// + /// + /// Pure-Python requirements are vendored once and the package runs anywhere. A requirement that + /// has native code is vendored once per target platform, under its own folder, and the manifest + /// lists those platforms, so a host on any other refuses it at install rather than failing on + /// import. + /// + public static class PythonBuild + { + public const string VendorFolder = "_vendor"; + public const string EntryScript = "_serverless_entry.py"; + public const string DefaultEntry = "main.py"; + public const string DefaultRuntimeVersion = ">=3.12"; + + /// The platforms a package with native requirements is built for, unless adapter.json names others. + public static readonly IReadOnlyList DefaultNativePlatforms = new[] { "linux-x64", "linux-arm64" }; + + /// The SDKs the build vendors itself; requirements.txt naming them is not sent to pip. + static readonly string[] OwnPackages = { "simplyworks-serverless", "simplyworks_serverless", "simplyworks-bitween", "simplyworks_bitween" }; + + // pip's platform tags for each platform a manifest can name. + static readonly Dictionary PipPlatforms = new(StringComparer.OrdinalIgnoreCase) + { + ["linux-x64"] = new[] { "manylinux2014_x86_64", "manylinux_2_28_x86_64", "manylinux_2_17_x86_64" }, + ["linux-arm64"] = new[] { "manylinux2014_aarch64", "manylinux_2_28_aarch64", "manylinux_2_17_aarch64" }, + ["linux-musl-x64"] = new[] { "musllinux_1_2_x86_64" }, + ["linux-musl-arm64"] = new[] { "musllinux_1_2_aarch64" }, + ["osx-arm64"] = new[] { "macosx_11_0_arm64" }, + ["osx-x64"] = new[] { "macosx_10_9_x86_64" }, + ["win-x64"] = new[] { "win_amd64" }, + }; + + /// The Python the bootstrap and the vendored code are written for, as pip's --python-version takes it. + const string TargetPythonVersion = "3.12"; + + internal static async Task BuildAsync(BuildRequest request, string project, AdapterManifest author, BuildResult result) + { + var entry = string.IsNullOrWhiteSpace(author.Entry) ? DefaultEntry : author.Entry.Replace('\\', '/'); + if (!File.Exists(Path.Combine(project, entry))) + { + result.Problems.Add($"there is no {entry} in {project}; set \"entry\" in {AdapterManifest.FileName} to the script that runs the adapter"); + return; + } + + // Source first, as for .NET: a secret found here stops the build before anything is produced. + var source = PackageBuilder.CollectSource(project, null, request, result); + if (!result.Succeeded || request.DryRun) return; + + var output = Path.GetFullPath(request.OutputDirectory ?? Path.Combine(project, "bin", "serverless")); + var packageDirectory = Path.Combine(output, "package"); + if (Directory.Exists(packageDirectory)) Directory.Delete(packageDirectory, true); + Directory.CreateDirectory(packageDirectory); + + // The adapter's own files run as they are: everything the source rules keep, which leaves + // out caches, virtual environments and secrets. + foreach (var (relative, absolute) in source) + { + var target = Path.Combine(packageDirectory, relative); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.Copy(absolute, target); + } + + var vendor = Path.Combine(packageDirectory, VendorFolder); + WriteOwnPackages(vendor); + + var (platforms, describeOnly) = await VendorRequirementsAsync(request, project, author, vendor, result); + if (!result.Succeeded) return; + + await File.WriteAllTextAsync(Path.Combine(packageDirectory, EntryScript), Bootstrap(entry)); + + request.Log("Asking the adapter to describe itself..."); + var (description, problem) = await LocalAdapterHost.DescribeAsync(Path.Combine(packageDirectory, EntryScript), + AdapterManifest.PythonRuntime, request.Runtimes); + if (description == null) + { + result.Problems.Add($"{problem}. A Python adapter describes itself through simplyworks_serverless.run(); make sure {entry} calls it"); + return; + } + result.Warnings.AddRange(description.Warnings); + // Vendored for this machine only so it could describe itself; not one of its platforms. + if (describeOnly != null) Directory.Delete(describeOnly, true); + + var manifest = PackageBuilder.ManifestFrom(author, false, description, EntryScript, "python"); + manifest.Runtime = AdapterManifest.PythonRuntime; + manifest.RuntimeVersion ??= DefaultRuntimeVersion; + manifest.Platforms = platforms; + // Every Python adapter speaks gRPC, whichever way it runs. + manifest.Lifecycle = description.Lifecycle == AdapterManifest.ResidentLifecycle + ? AdapterManifest.ResidentLifecycle + : AdapterManifest.ClassicLifecycle; + manifest.Protocol = new AdapterProtocolRange { Min = 2, Max = 2 }; + + if (request.IncludeSource) + { + manifest.Source = new AdapterSource + { + BuildCommand = "serverless build", + Lockfiles = source.Keys.Where(k => PackageBuilder.IsLockfile(k)).OrderBy(k => k).ToList(), + }; + var sourceDirectory = Path.Combine(packageDirectory, AdapterSource.DefaultPath); + foreach (var (relative, absolute) in source.OrderBy(s => s.Key, StringComparer.Ordinal)) + { + var target = Path.Combine(sourceDirectory, relative); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.Copy(absolute, target); + manifest.Source.Files[relative] = Convert.ToHexString(SHA256.HashData(await File.ReadAllBytesAsync(absolute))).ToLowerInvariant(); + } + if (manifest.Source.Lockfiles.Count == 0) manifest.Source.Lockfiles = null; + } + + var problems = manifest.Validate(); + if (problems.Count > 0) + { + result.Problems.AddRange(problems); + return; + } + + await File.WriteAllTextAsync(Path.Combine(packageDirectory, AdapterManifest.FileName), manifest.ToJson()); + + var zip = Path.Combine(output, $"{manifest.Id}{(string.IsNullOrWhiteSpace(manifest.Version) ? "" : "-" + manifest.Version)}.zip"); + if (File.Exists(zip)) File.Delete(zip); + ZipFile.CreateFromDirectory(packageDirectory, zip, CompressionLevel.Optimal, includeBaseDirectory: false); + + result.Manifest = manifest; + result.PackageDirectory = packageDirectory; + result.ZipPath = zip; + } + + /// The SDK and the Bitween kinds, as this tool carries them, under . + public static void WriteOwnPackages(string vendor) + { + var assembly = typeof(PythonBuild).Assembly; + foreach (var name in assembly.GetManifestResourceNames()) + { + var normalized = name.Replace('\\', '/'); + if (!normalized.StartsWith("python/", StringComparison.Ordinal)) continue; + var target = Path.Combine(vendor, normalized["python/".Length..]); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + using var resource = assembly.GetManifestResourceStream(name)!; + using var file = File.Create(target); + resource.CopyTo(file); + } + } + + /// + /// Vendors requirements.txt. Returns the platforms the package is limited to — none when + /// everything is pure Python, or the targets when anything is native — and, when this + /// machine isn't a target, a folder vendored for it alone, to describe the adapter with. + /// + static async Task<(List Platforms, string DescribeOnly)> VendorRequirementsAsync(BuildRequest request, string project, AdapterManifest author, + string vendor, BuildResult result) + { + var requirementsFile = Path.Combine(project, "requirements.txt"); + if (!File.Exists(requirementsFile)) return (author.Platforms is { Count: > 0 } ? author.Platforms : null, null); + + var lines = (await File.ReadAllLinesAsync(requirementsFile)) + .Where(l => !IsOwnPackage(l)) + .ToList(); + if (!lines.Any(l => !string.IsNullOrWhiteSpace(l) && !l.TrimStart().StartsWith('#'))) + return (author.Platforms is { Count: > 0 } ? author.Platforms : null, null); + + var work = Path.Combine(Path.GetTempPath(), "swsl-pybuild", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(work); + try + { + var filtered = Path.Combine(work, "requirements.txt"); + await File.WriteAllLinesAsync(filtered, lines); + + var targets = author.Platforms is { Count: > 0 } ? author.Platforms.ToList() : DefaultNativePlatforms.ToList(); + foreach (var target in targets.Where(t => !PipPlatforms.ContainsKey(t))) + result.Problems.Add($"'{target}' isn't a platform Python packages can be built for: use {string.Join(", ", PipPlatforms.Keys)}"); + if (!result.Succeeded) return (null, null); + + // Wheels only, for the Python the host runs: nothing is compiled here, so what is + // vendored is exactly what pip would install there. + request.Log("Fetching the requirements..."); + var wheels = new Dictionary(); + foreach (var target in targets) + { + var folder = Path.Combine(work, "wheels", target); + var (ok, output) = await PipAsync(request, new[] { "download", "-r", filtered, "-d", folder } + .Concat(WheelOptions(target)).ToArray()); + if (!ok) + { + result.Problems.Add($"pip could not get the requirements for {target}: {LastLines(output)}. " + + "A package with native code needs a wheel for every platform the adapter runs on"); + return (null, null); + } + wheels[target] = folder; + } + + var native = wheels.Values.SelectMany(f => Directory.GetFiles(f, "*.whl")).Any(w => !Path.GetFileName(w).EndsWith("-none-any.whl", StringComparison.OrdinalIgnoreCase)); + if (!native) + { + // One set serves every platform. + var (ok, output) = await PipAsync(request, new[] { "install", "--no-deps", "--no-compile", "--target", vendor } + .Concat(Directory.GetFiles(wheels[targets[0]], "*.whl")).ToArray()); + if (!ok) result.Problems.Add($"pip could not vendor the requirements: {LastLines(output)}"); + return (author.Platforms is { Count: > 0 } ? author.Platforms : null, null); + } + + foreach (var target in targets) + { + var (ok, output) = await PipAsync(request, new[] { "install", "--no-deps", "--no-compile", "--target", Path.Combine(vendor, target) } + .Concat(WheelOptions(target)).Concat(Directory.GetFiles(wheels[target], "*.whl")).ToArray()); + if (!ok) + { + result.Problems.Add($"pip could not vendor the requirements for {target}: {LastLines(output)}"); + return (null, null); + } + } + request.Log($"Some requirements have native code; vendored for {string.Join(", ", targets)}."); + + // The adapter describes itself here, on this machine, which may be none of them. + string describeOnly = null; + var here = Runtimes.AdapterRuntimes.CurrentPlatform; + if (!targets.Contains(here, StringComparer.OrdinalIgnoreCase)) + { + describeOnly = Path.Combine(vendor, here); + var (ok, output) = await PipAsync(request, new[] { "install", "--no-compile", "--target", describeOnly, "-r", filtered }); + if (!ok) + { + result.Problems.Add($"pip could not install the requirements on this machine to describe the adapter: {LastLines(output)}"); + return (null, null); + } + } + return (targets, describeOnly); + } + finally + { + try { Directory.Delete(work, true); } catch { } + } + } + + static IEnumerable WheelOptions(string platform) => + new[] { "--only-binary=:all:", "--python-version", TargetPythonVersion, "--implementation", "cp" } + .Concat(PipPlatforms[platform].SelectMany(tag => new[] { "--platform", tag })); + + static bool IsOwnPackage(string line) + { + var name = Regex.Match(line.Trim(), @"^[A-Za-z0-9_.\-]+").Value; + return OwnPackages.Contains(name, StringComparer.OrdinalIgnoreCase); + } + + static async Task<(bool Ok, string Output)> PipAsync(BuildRequest request, string[] arguments) + { + var start = new ProcessStartInfo(request.Runtimes.PythonExecutable) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + foreach (var argument in new[] { "-m", "pip", "--disable-pip-version-check", "--no-input" }.Concat(arguments)) + start.ArgumentList.Add(argument); + using var process = Process.Start(start)!; + var stdout = process.StandardOutput.ReadToEndAsync(); + var stderr = process.StandardError.ReadToEndAsync(); + await process.WaitForExitAsync(); + return (process.ExitCode == 0, await stdout + await stderr); + } + + static string LastLines(string output) => + string.Join(" ", output.Split('\n').Select(l => l.Trim()).Where(l => l.Length > 0).TakeLast(3)); + + /// + /// The package's entry: puts the vendored code on the path — this platform's folder first, + /// when requirements were vendored per platform — and runs the adapter's own entry as __main__. + /// + internal static string Bootstrap(string entry) => $$""" + # Written by serverless build. Puts the vendored packages on the path and runs {{entry}}. + import os, platform, runpy, sys + + here = os.path.dirname(os.path.abspath(__file__)) + vendor = os.path.join(here, "{{VendorFolder}}") + machine = platform.machine().lower() + arch = "arm64" if machine in ("arm64", "aarch64") else "x64" + system = {"linux": "linux", "darwin": "osx", "win32": "win"}.get(sys.platform, sys.platform) + musl = system == "linux" and "musl" in (platform.libc_ver()[0] or "") + native = os.path.join(vendor, f"{system}-musl-{arch}" if musl else f"{system}-{arch}") + for folder in (vendor, native): + if os.path.isdir(folder): + sys.path.insert(0, folder) + sys.path.insert(0, here) + sys.argv[0] = os.path.join(here, "{{entry}}") + runpy.run_path(sys.argv[0], run_name="__main__") + """; + } +} diff --git a/SW.Serverless.Tooling/Contracts/bitween/python/simplyworks_bitween/__init__.py b/SW.Serverless.Tooling/Contracts/bitween/python/simplyworks_bitween/__init__.py new file mode 100644 index 0000000..43a32fc --- /dev/null +++ b/SW.Serverless.Tooling/Contracts/bitween/python/simplyworks_bitween/__init__.py @@ -0,0 +1,236 @@ +"""The Bitween adapter contract for Python: the kinds of adapter Bitween runs, and what it passes. + +Subclass a kind and implement its methods; the wire names, the encoding and the kind and contract +declarations are taken care of. The contract itself is ``bitween-adapter-contract.v1.json`` in +SW.Bitween.Adapters; this is its Python form, as ``SimplyWorks.Bitween.Adapters`` is its .NET one. + + import simplyworks_serverless as sw +from simplyworks_serverless._runner import _call + from simplyworks_bitween import ExchangeFile, Handler + + class Orders(Handler): + def __init__(self): + sw.expect("Url", description="Where orders go") + + def handle(self, file: ExchangeFile) -> ExchangeFile: + ... + + if __name__ == "__main__": + sw.run(Orders) +""" + +import hashlib +from dataclasses import dataclass, field + +import simplyworks_serverless as sw +from simplyworks_serverless._runner import _call + +CONTRACT = "bitween" +CONTRACT_VERSION = 1 + +__version__ = "10.0.59" + + +@dataclass +class ExchangeFile: + """A file as Bitween passes it to and from adapters. + + ``data`` is the content: text, or base64 for binary content. ``bad_data`` marks a bad response, + a delivery the partner rejected — returned, not raised. + """ + + data: str = "" + filename: str | None = None + bad_data: bool = False + content_type: str | None = None + + @property + def hash(self): + """SHA-1 of ``data`` as lower-case hex, as .NET adapters write it.""" + return hashlib.sha1((self.data or "").encode("utf-8")).hexdigest() + + def to_wire(self): + return {"Filename": self.filename, "Data": self.data or "", "Hash": self.hash, + "BadData": self.bad_data, "ContentType": self.content_type} + + @classmethod + def from_wire(cls, value): + if not isinstance(value, dict): + raise ValueError("an ExchangeFile is a JSON object") + # Hash is recomputed, never trusted; unknown properties are ignored. + return cls(data=value.get("Data") or "", filename=value.get("Filename"), + bad_data=bool(value.get("BadData", False)), content_type=value.get("ContentType")) + + @staticmethod + def __sw_schema__(): + return { + "title": "ExchangeFile", "type": "object", "required": ["Data"], "additionalProperties": True, + "properties": { + "Filename": {"type": ["string", "null"]}, "Data": {"type": "string"}, + "Hash": {"type": ["string", "null"]}, "BadData": {"type": "boolean", "default": False}, + "ContentType": {"type": ["string", "null"]}, + }, + } + + +@dataclass +class ValidationResult: + """What a validator found: each failure as a key — often the field it concerns — and a message. + No failures means valid.""" + + validations: list[tuple[str, str]] = field(default_factory=list) + + @property + def success(self): + return not self.validations + + def add(self, key, message): + self.validations.append((key, message)) + return self + + def to_wire(self): + return {"Success": self.success, "Validations": [{"Key": k, "Value": v} for k, v in self.validations]} + + @classmethod + def from_wire(cls, value): + return cls([(v.get("Key", ""), v.get("Value", "")) for v in (value or {}).get("Validations") or []]) + + @staticmethod + def __sw_schema__(): + return { + "title": "ValidationResult", "type": "object", "required": ["Validations"], "additionalProperties": True, + "properties": { + "Success": {"type": "boolean"}, + "Validations": {"type": "array", "items": { + "type": "object", "required": ["Key", "Value"], + "properties": {"Key": {"type": "string"}, "Value": {"type": "string"}}}}, + }, + } + + +def _as_file(value): + if isinstance(value, ExchangeFile): + return value + if isinstance(value, str): + return ExchangeFile(data=value) + raise TypeError(f"expected an ExchangeFile, got {type(value).__name__}") + + +class _Kind: + __sw_contracts__ = {CONTRACT: CONTRACT_VERSION} + _required = () + + def __sw_check__(self): + # A declared kind without its methods fails when the adapter starts, not on first use. + missing = [m for m in self._required if getattr(getattr(type(self), m), "__sw_abstract__", False)] + if missing: + raise TypeError(f"{type(self).__name__} is a Bitween {self.__sw_kinds__[0]} " + f"but does not implement {', '.join(missing)}") + + +def _abstract(fn): + fn.__sw_abstract__ = True + return fn + + +class Handler(_Kind): + """Delivers a message and returns the partner's response. A rejected delivery is returned with + ``bad_data=True``, not raised.""" + + __sw_kinds__ = ["handler"] + _required = ("handle",) + + @_abstract + def handle(self, file: ExchangeFile) -> ExchangeFile: + raise NotImplementedError + + @sw.command("Handle", description="Delivers a message and returns the partner's response.") + async def _sw_handle(self, file: ExchangeFile) -> ExchangeFile: + return _as_file(await _call(self.handle, file)) + + +class Mapper(_Kind): + """Transforms a message into the shape the next step expects.""" + + __sw_kinds__ = ["mapper"] + _required = ("map",) + + @_abstract + def map(self, file: ExchangeFile) -> ExchangeFile: + raise NotImplementedError + + @sw.command("Handle", description="Transforms a message into the shape the next step expects.") + async def _sw_handle(self, file: ExchangeFile) -> ExchangeFile: + return _as_file(await _call(self.map, file)) + + +class Validator(_Kind): + """Checks a message before it is accepted.""" + + __sw_kinds__ = ["validator"] + _required = ("validate",) + + @_abstract + def validate(self, file: ExchangeFile) -> ValidationResult: + raise NotImplementedError + + @sw.command("Validate", description="Checks a message before it is accepted.") + async def _sw_validate(self, file: ExchangeFile) -> ValidationResult: + result = await _call(self.validate, file) + if result is None: + return ValidationResult() + if isinstance(result, ValidationResult): + return result + # A list of (key, message) pairs, or a dict of key -> message, reads naturally too. + return ValidationResult(list(result.items()) if isinstance(result, dict) else [tuple(r) for r in result]) + + +class Receiver(_Kind): + """Fetches files from a source on a schedule. One session per run: ``initialize``, ``list_files``, + then for each file ``get_file`` and — once it is safely taken in — ``delete_file``, and finally + ``finalize``, which is also called after a failure.""" + + __sw_kinds__ = ["receiver"] + _required = ("list_files", "get_file", "delete_file") + + def initialize(self): + pass + + @_abstract + def list_files(self) -> list[str]: + raise NotImplementedError + + @_abstract + def get_file(self, file_id: str) -> ExchangeFile: + raise NotImplementedError + + @_abstract + def delete_file(self, file_id: str) -> None: + raise NotImplementedError + + def finalize(self): + pass + + @sw.command("Initialize", description="Starts a run.") + async def _sw_initialize(self) -> None: + await _call(self.initialize) + + @sw.command("ListFiles", description="The ids of the files waiting.") + async def _sw_list_files(self) -> list[str]: + return [str(f) for f in (await _call(self.list_files) or [])] + + @sw.command("GetFile", description="One file, by an id ListFiles gave.") + async def _sw_get_file(self, file_id: str) -> ExchangeFile: + return _as_file(await _call(self.get_file, file_id)) + + @sw.command("DeleteFile", description="Removes a file from the source once it is safely taken in.") + async def _sw_delete_file(self, file_id: str) -> None: + await _call(self.delete_file, file_id) + + @sw.command("Finalize", description="Ends a run, after a failure too.") + async def _sw_finalize(self) -> None: + await _call(self.finalize) + + +__all__ = ["CONTRACT", "CONTRACT_VERSION", "ExchangeFile", "Handler", "Mapper", "Receiver", "ValidationResult", + "Validator"] diff --git a/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj b/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj index 71b7936..2097f4c 100644 --- a/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj +++ b/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj @@ -38,6 +38,14 @@ + + + + + + diff --git a/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs b/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs index 8fcdb4a..025842a 100644 --- a/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs +++ b/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs @@ -47,7 +47,10 @@ public static class Scaffolder public const string BitweenAdaptersPackageVersion = "10.0.59"; public static readonly IReadOnlyList Kinds = new[] { "handler", "mapper", "validator", "receiver" }; - public static readonly IReadOnlyList Languages = new[] { "dotnet" }; + public static readonly IReadOnlyList Languages = new[] { "dotnet", "python" }; + + /// The Python SDK the templates name; serverless build vendors the copy it carries. + public const string PythonSdkVersion = "10.1.0"; public static ScaffoldResult Scaffold(ScaffoldRequest request) { @@ -71,10 +74,13 @@ public static ScaffoldResult Scaffold(ScaffoldRequest request) Directory.CreateDirectory(directory); result.ProjectDirectory = directory; - foreach (var (file, content) in DotnetFiles(name, id, request.Kind)) + var files = request.Language == "python" ? PythonFiles(name, id, request.Kind) : DotnetFiles(name, id, request.Kind); + foreach (var (file, content) in files) { var path = Path.Combine(directory, file); - File.WriteAllText(path, content.Replace("\r\n", "\n")); + // Ending in a newline, as text files should: a line appended later stays its own line. + var text = content.Replace("\r\n", "\n"); + File.WriteAllText(path, text.EndsWith('\n') ? text : text + "\n"); result.Files.Add(file); } return result; @@ -248,6 +254,149 @@ static class Program """, }; + static IEnumerable<(string File, string Content)> PythonFiles(string name, string id, string kind) + { + yield return ("adapter.json", $$""" + { + "id": "{{id}}", + "version": "0.1.0", + "displayName": "{{Spaced(name)}}", + "summary": "What this {{kind}} does, in one sentence, for the adapter list.", + "runtime": "python", + "entry": "main.py" + } + """); + + yield return ("main.py", PythonMain(name, kind)); + + yield return ("requirements.txt", $$""" + # What the adapter imports, pinned, one per line; serverless build vendors them into the + # package. The two SDKs are vendored by serverless build itself: they're listed here for + # your editor and for running the tests outside a build. + simplyworks-serverless=={{PythonSdkVersion}} + simplyworks-bitween>={{BitweenAdaptersPackageVersion}} + """); + + yield return ("settings.example.json", """ + { + "BaseUrl": "https://partner.example.test", + "ApiKey": "put a test key here, and keep this file out of version control once it holds one" + } + """); + + yield return (".gitignore", """ + __pycache__/ + .venv/ + bin/ + settings.json + """); + + yield return ("README.md", $$""" + # {{Spaced(name)}} + + A Bitween {{kind}} adapter in Python (3.12 or later). + + ```sh + serverless build # builds bin/serverless/{{id}}-0.1.0.zip + cp settings.example.json settings.json # then fill in real values + serverless test --settings settings.json # checks it against the Bitween contract + serverless publish bin/serverless/{{id}}-0.1.0.zip # with your storage flags + ``` + + Settings are declared in code with `sw.expect`; `serverless build` writes them into the + manifest Bitween reads. Dependencies go in `requirements.txt`, pinned. + """); + } + + static string PythonMain(string name, string kind) => kind switch + { + "receiver" => $$"""" + import simplyworks_serverless as sw + from simplyworks_bitween import ExchangeFile, Receiver + + + class {{name}}(Receiver): + """Fetches files on a schedule. Bitween calls initialize, list_files, then get_file and + delete_file for each file, then finalize.""" + + def __init__(self): + # Declare settings here; read them in the methods with sw.value_of. + sw.expect("BaseUrl", "https://partner.example.test", description="Where files are fetched from.") + sw.expect("ApiKey", secret=True, description="The partner's key.") + + def list_files(self) -> list[str]: + return ["example-1"] + + def get_file(self, file_id: str) -> ExchangeFile: + return ExchangeFile(data='{"id": "%s"}' % file_id, filename=file_id + ".json") + + def delete_file(self, file_id: str) -> None: + pass + + + if __name__ == "__main__": + sw.run({{name}}) + """", + "validator" => $$"""" + import simplyworks_serverless as sw + from simplyworks_bitween import ExchangeFile, ValidationResult, Validator + + + class {{name}}(Validator): + """Checks a message before Bitween accepts it.""" + + def __init__(self): + sw.expect("MaxBytes", "1000000", type="number", description="The largest message accepted.") + + def validate(self, file: ExchangeFile) -> ValidationResult: + result = ValidationResult() + if len(file.data) > int(sw.value_of("MaxBytes")): + result.add("Data", "The message is larger than allowed.") + return result + + + if __name__ == "__main__": + sw.run({{name}}) + """", + "mapper" => $$"""" + import simplyworks_serverless as sw + from simplyworks_bitween import ExchangeFile, Mapper + + + class {{name}}(Mapper): + """Maps a message into the shape the next step expects.""" + + def map(self, file: ExchangeFile) -> ExchangeFile: + # Return the message in its new shape. + return ExchangeFile(data=file.data, filename=file.filename) + + + if __name__ == "__main__": + sw.run({{name}}) + """", + _ => $$"""" + import simplyworks_serverless as sw + from simplyworks_bitween import ExchangeFile, Handler + + + class {{name}}(Handler): + """Delivers a message and returns the partner's response.""" + + def __init__(self): + # Declare settings here; read them in the methods with sw.value_of. + sw.expect("BaseUrl", "https://partner.example.test", description="Where messages go.") + sw.expect("ApiKey", secret=True, description="The partner's key.") + + def handle(self, file: ExchangeFile) -> ExchangeFile: + # Send file.data to the partner. A rejection is returned with bad_data=True, not raised. + return ExchangeFile(data=file.data, filename=file.filename) + + + if __name__ == "__main__": + sw.run({{name}}) + """", + }; + static string Spaced(string name) => Regex.Replace(name, "(?<=[a-z0-9])(?=[A-Z])", " "); } } From ecc6e0cb0a2e14fb4e5af9a0ede0c3a21ecc34fd Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 16:38:20 +0300 Subject: [PATCH 04/17] Start gRPC sessions whose values include nulls A protobuf map can't hold a null, and Ready's startup values were copied in as given: Bitween's gateway runs validators with no correlation id, so any gRPC adapter used as a validator there failed before it was ready, its stream ended by an exception in Attach. Null values are left out, and read as absent, which is what null meant. --- SW.Serverless.UnitTests/GrpcClassicTests.cs | 18 ++++++++++++++++++ .../Resident/ResidentAdapterInstance.cs | 19 +++++++++++++------ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/SW.Serverless.UnitTests/GrpcClassicTests.cs b/SW.Serverless.UnitTests/GrpcClassicTests.cs index 544ce33..64814a7 100644 --- a/SW.Serverless.UnitTests/GrpcClassicTests.cs +++ b/SW.Serverless.UnitTests/GrpcClassicTests.cs @@ -101,6 +101,24 @@ public static async Task ClassCleanup() static IEnumerable RunningKeys() => host.Services.GetRequiredService().Describe().Select(h => h.InstanceKey); + [TestMethod] + public async Task A_session_without_a_correlation_id_or_with_null_values_still_starts() + { + // A protobuf map can't hold a null: Bitween's gateway runs validators with no correlation + // id, and those sessions failed before the adapter was ready. + var service = Service(); + await service.StartAsync(AdapterId, null, new Dictionary { ["Prefix"] = "hi ", ["Unset"] = null }); + try + { + Assert.AreEqual("hi world", await service.InvokeAsync("Greet", "world")); + Assert.IsTrue(string.IsNullOrEmpty(await service.InvokeAsync("Correlation", null))); + } + finally + { + ((IDisposable)service).Dispose(); + } + } + [TestMethod] public async Task A_call_returns_the_adapter_s_answer_using_the_startup_values_it_was_given() { diff --git a/SW.Serverless/Resident/ResidentAdapterInstance.cs b/SW.Serverless/Resident/ResidentAdapterInstance.cs index 91c16af..01ada85 100644 --- a/SW.Serverless/Resident/ResidentAdapterInstance.cs +++ b/SW.Serverless/Resident/ResidentAdapterInstance.cs @@ -8,6 +8,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; @@ -183,12 +184,8 @@ internal async Task RunStreamAsync(IAsyncStreamReader input, Ready = new Ready { MaxInFlight = options.MaxInFlight, - StartupValues = { (IDictionary)(StartupValues == null - ? new Dictionary() - : new Dictionary(StartupValues)) }, - AdapterValues = { (IDictionary)(AdapterValues == null - ? new Dictionary() - : new Dictionary(AdapterValues)) } + StartupValues = { WithoutNulls(StartupValues) }, + AdapterValues = { WithoutNulls(AdapterValues) } } }); @@ -480,6 +477,16 @@ void SendCancel(long id) Send(new HostFrame { Id = id, Cancel = new Cancel() }); } + /// + /// The values that have one. A protobuf map can't hold a null and throws on one, which ended + /// the stream before the adapter was ever ready: a gateway's validator call has no + /// correlation id, for one. Left out, a value reads as absent, which is what null meant. + /// + static IDictionary WithoutNulls(IEnumerable> values) => + values == null + ? new Dictionary() + : values.Where(kv => kv.Key != null && kv.Value != null).ToDictionary(kv => kv.Key, kv => kv.Value); + public async Task PingAsync(TimeSpan timeout) { var id = Interlocked.Increment(ref nextId); From 36ab3ec2e6fc49f234ae2f140903b3e8ebb8ef41 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 16:42:45 +0300 Subject: [PATCH 05/17] Add the Node SDK, @simplyworks/serverless Adapters in JavaScript or TypeScript on Node 22+ declare settings with expect, commands in a static commands map with how each argument and result is encoded, and run with run(), which serves --describe and otherwise the host that started it. Resident adapters add start, stop, status and reset; a call's context publishes events, keeps state, records metrics and has a signal that aborts when the host gives up on the call; logs reach the host. No dependencies: Node's http2 carries the gRPC stream and the SDK its own protobuf codec, whose bytes match the Python SDK's. Typings ship with it. --- .github/workflows/node-sdk.yml | 26 ++ sdk/node/README.md | 41 +++ sdk/node/package.json | 12 + sdk/node/src/index.d.ts | 80 +++++ sdk/node/src/index.js | 597 +++++++++++++++++++++++++++++++++ sdk/node/src/wire.js | 218 ++++++++++++ sdk/node/test/sdk.test.js | 91 +++++ 7 files changed, 1065 insertions(+) create mode 100644 .github/workflows/node-sdk.yml create mode 100644 sdk/node/README.md create mode 100644 sdk/node/package.json create mode 100644 sdk/node/src/index.d.ts create mode 100644 sdk/node/src/index.js create mode 100644 sdk/node/src/wire.js create mode 100644 sdk/node/test/sdk.test.js diff --git a/.github/workflows/node-sdk.yml b/.github/workflows/node-sdk.yml new file mode 100644 index 0000000..75a7bec --- /dev/null +++ b/.github/workflows/node-sdk.yml @@ -0,0 +1,26 @@ +name: Node SDK + +on: + push: + branches: [main] + paths: ["sdk/node/**", ".github/workflows/node-sdk.yml"] + pull_request: + paths: ["sdk/node/**", ".github/workflows/node-sdk.yml"] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + node: ["22", "24"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + - name: Unit tests + working-directory: sdk/node + run: node --test diff --git a/sdk/node/README.md b/sdk/node/README.md new file mode 100644 index 0000000..633aef9 --- /dev/null +++ b/sdk/node/README.md @@ -0,0 +1,41 @@ +# @simplyworks/serverless + +Write SW-Serverless adapters in JavaScript or TypeScript, on Node 22 or later. No dependencies: Node's +own `http2` speaks the host's protocol — gRPC over a Unix socket — and the SDK carries the protobuf +messages itself, so it vendors into an adapter package as plain files. + +```js +const sw = require("@simplyworks/serverless"); + +class Greeter { + static commands = { + Greet: { method: "greet", input: "string", output: "string", description: "Greets someone" }, + }; + + constructor() { + sw.expect("Greeting", { default: "Hello", description: "What to say" }); + sw.expect("ApiKey", { secret: true }); + } + + greet(name) { + return `${sw.valueOf("Greeting")}, ${name}`; + } +} + +sw.run(Greeter); +``` + +- **Settings** are declared with `sw.expect(name, { default, required, secret, description, type })` + and read with `sw.valueOf(name)`. +- **Commands** are declared in `static commands`: the method that runs each, and its `input` and + `output` — `"string"` (raw text), `"bytes"`, `"json"`, or a JSON Schema. Leave `input` out for a + command with no argument, and `output` for one that returns nothing. Methods may be async. +- **Errors** reach the caller with their type and message; throw + `new sw.AdapterError(message, { type: "Acme.Rejected" })` to choose the type. +- **Resident adapters** have a `start` method and run until stopped, with optional `stop`, `status` + and `reset(sessionId)`. `sw.context()` publishes events, keeps small state and records metrics; + its `signal` aborts when the host gives up on a call. +- **Logs** go to the host with `sw.log.info(...)` and the other levels. +- `node main.js --describe` prints what the adapter is; `serverless build` writes it into the manifest. + +For Bitween adapters, `@simplyworks/bitween` has the four kinds ready to extend. Tests: `npm test`. diff --git a/sdk/node/package.json b/sdk/node/package.json new file mode 100644 index 0000000..228f904 --- /dev/null +++ b/sdk/node/package.json @@ -0,0 +1,12 @@ +{ + "name": "@simplyworks/serverless", + "version": "10.1.0", + "description": "Write SW-Serverless adapters in JavaScript or TypeScript: settings, commands, and the host protocol, with no dependencies.", + "main": "src/index.js", + "types": "src/index.d.ts", + "files": ["src"], + "engines": { "node": ">=22" }, + "scripts": { "test": "node --test" }, + "license": "MIT", + "repository": { "type": "git", "url": "https://github.com/simplify9/SW-Serverless", "directory": "sdk/node" } +} diff --git a/sdk/node/src/index.d.ts b/sdk/node/src/index.d.ts new file mode 100644 index 0000000..8343a46 --- /dev/null +++ b/sdk/node/src/index.d.ts @@ -0,0 +1,80 @@ +/** A command's argument or result: raw text, raw bytes, JSON, or JSON described by a JSON Schema. */ +export type PayloadSpec = "string" | "bytes" | "json" | Record; + +export interface CommandSpec { + /** The method on the adapter that runs it. */ + method: string; + /** Omitted when the command takes no argument. */ + input?: PayloadSpec; + /** Omitted when it returns nothing. */ + output?: PayloadSpec; + description?: string; +} + +/** Declared on an adapter class as `static commands`, keyed by the name the host calls. */ +export type Commands = Record; + +export interface SettingOptions { + default?: string | number | boolean; + /** Required unless it has a default. */ + required?: boolean; + /** Masked wherever it is shown. */ + secret?: boolean; + description?: string; + type?: "text" | "multiline" | "number" | "boolean" | "select" | "json"; +} + +export declare function expect(name: string, options?: SettingOptions): string; +/** The call's own properties, then the startup values, then the declared default, then `fallback`. */ +export declare function valueOf(name: string, fallback?: string): string | undefined; +export declare function startupValues(): Record; + +export declare class AdapterError extends Error { + constructor(message: string, options?: { type?: string; detail?: string }); + type?: string; + detail?: string; +} + +export interface Status { + connected?: boolean; + state?: string; + inFlight?: number; + lastError?: string; + lastMessageOn?: Date | string | number; + details?: Record; +} + +export declare class Context { + readonly sessionId: string | null; + readonly command: string | null; + /** Aborted when the host gives up on the call. */ + readonly signal: AbortSignal; + readonly adapterId: string; + readonly instanceKey: string; + /** Aborted once the host has asked the adapter to stop. */ + readonly stopping: AbortSignal; + valueOf(name: string, fallback?: string): string | undefined; + publish(payload: unknown, options?: { dedupeKey?: string; contentType?: string; headers?: Record; endpoint?: string }): Promise; + getState(name: string): Promise; + setState(name: string, value: string): Promise; + deleteState(name: string): Promise; + metric(name: string, value: number, tags?: Record): void; +} + +export declare function context(): Context; + +type LogFn = (message: unknown, error?: unknown) => void; +export declare const log: { trace: LogFn; debug: LogFn; info: LogFn; warn: LogFn; error: LogFn; critical: LogFn }; + +/** + * An adapter class: `static commands` names what the host can call. A `start` method makes it + * resident, with optional `stop`, `status` and `reset(sessionId)`. + */ +export interface AdapterClass { + new (): object; + commands?: Commands; +} + +export declare function describe(adapter: AdapterClass): Record; +export declare function run(adapter: AdapterClass): Promise; +export declare const SDK_VERSION: string; diff --git a/sdk/node/src/index.js b/sdk/node/src/index.js new file mode 100644 index 0000000..4b041d7 --- /dev/null +++ b/sdk/node/src/index.js @@ -0,0 +1,597 @@ +"use strict"; +/** + * Write SW-Serverless adapters in JavaScript or TypeScript, on Node 22 or later. + * + * const sw = require("@simplyworks/serverless"); + * + * class Greeter { + * static commands = { + * Greet: { method: "greet", input: "string", output: "string", description: "Greets someone" }, + * }; + * constructor() { sw.expect("Greeting", { default: "Hello" }); } + * greet(name) { return `${sw.valueOf("Greeting")}, ${name}`; } + * } + * + * sw.run(Greeter); + * + * `node adapter.js --describe` prints what it is; the host runs it otherwise. An adapter with a + * `start` method is resident: it runs until stopped, with `stop`, `status` and `reset` hooks. + * + * No dependencies: Node's own http2 speaks gRPC's transport, and wire.js the protobuf messages. + */ + +const http2 = require("node:http2"); +const net = require("node:net"); +const readline = require("node:readline"); +const { AsyncLocalStorage } = require("node:async_hooks"); +const wire = require("./wire"); + +const SDK_VERSION = "10.1.0"; +const SDK_LANGUAGE = "node"; +const PROTOCOL = 2; +const ATTACH = "/sw.serverless.v1.AdapterHost/Attach"; +const DESCRIBE_FLAG = "--describe"; +const SETTING_TYPES = ["text", "multiline", "number", "boolean", "select", "json"]; +const MAX_MESSAGE = 64 * 1024 * 1024; + +// ILogger levels, which the host's log pipeline reads: Trace 0 .. Critical 5. +const LEVELS = { trace: 0, debug: 1, info: 2, warn: 3, error: 4, critical: 5 }; + +class AdapterError extends Error { + /** An error with a type of the adapter's choosing, which the host records as given. */ + constructor(message, { type, detail } = {}) { + super(message); + this.name = "AdapterError"; + this.type = type; + this.detail = detail; + } +} + +// ---------------------------------------------------------------------------- settings + +const settings = new Map(); +const startupValues = new Map(); +const callStorage = new AsyncLocalStorage(); + +/** + * Declares a setting the adapter reads. Required unless it has a default or `required: false` says + * otherwise. A secret is masked wherever Bitween shows it. Declaring a name again replaces it. + */ +function expect(name, options = {}) { + if (!name || typeof name !== "string") throw new TypeError("a setting needs a name"); + const { default: def, required, secret = false, description, type = "text" } = options; + if (!SETTING_TYPES.includes(type)) throw new TypeError(`setting type must be one of ${SETTING_TYPES.join(", ")}`); + settings.set(name, { + name, + default: def === undefined || def === null ? null : String(def), + required: required === undefined ? def === undefined || def === null : Boolean(required), + secret: Boolean(secret), + description: description ?? null, + type, + }); + return name; +} + +/** + * A setting's value for the current call: the call's own properties, then the values the adapter + * was started with, then the declared default, then `fallback`. + */ +function valueOf(name, fallback = undefined) { + const call = callStorage.getStore(); + if (call && call.properties && name in call.properties) return call.properties[name]; + if (startupValues.has(name)) return startupValues.get(name); + const declared = settings.get(name); + if (declared && declared.default !== null) return declared.default; + return fallback; +} + +const declaredSettings = () => [...settings.values()]; + +// ---------------------------------------------------------------------------- commands + +/** + * The commands an adapter class declares in `static commands`, merged down its prototype chain: + * { WireName: { method, input, output, description } }. `input` and `output` are "string", + * "bytes", "json" or a JSON Schema object (JSON described by it); no `input` means the command + * takes no argument, no `output` that it returns nothing. + */ +function commandsOf(cls) { + const chain = []; + for (let c = cls; c && c !== Function.prototype; c = Object.getPrototypeOf(c)) chain.unshift(c); + const found = {}; + for (const c of chain) { + if (Object.prototype.hasOwnProperty.call(c, "commands")) Object.assign(found, c.commands); + } + for (const [name, spec] of Object.entries(found)) { + if (!spec || typeof spec.method !== "string") throw new TypeError(`command ${name} needs a method name`); + } + return found; +} + +function collect(cls, property) { + const values = []; + for (let c = cls; c && c !== Function.prototype; c = Object.getPrototypeOf(c)) { + if (Object.prototype.hasOwnProperty.call(c, property)) values.unshift(c[property]); + } + return values; +} + +const kindsOf = (cls) => [...new Set(collect(cls, "kinds").flat())]; +const contractsOf = (cls) => Object.assign({}, ...collect(cls, "contracts")); +const isResident = (cls) => typeof cls.prototype.start === "function"; + +function schemaOf(spec) { + if (spec === undefined || spec === null) return null; + if (typeof spec === "object") return spec; + return { string: { type: "string" }, bytes: { type: "string", contentEncoding: "binary" }, json: {} }[spec] ?? {}; +} + +function decodeArgument(spec, payload) { + if (spec === "bytes") return Buffer.from(payload); + const text = Buffer.from(payload).toString("utf8"); + if (spec === "string") return text; + if (!text) return null; + if (spec === undefined || spec === null) { + try { return JSON.parse(text); } catch { return text; } + } + return JSON.parse(text); +} + +function encodeResult(value) { + if (value === undefined || value === null) return Buffer.alloc(0); + if (Buffer.isBuffer(value) || value instanceof Uint8Array) return Buffer.from(value); + if (typeof value === "string") return Buffer.from(value, "utf8"); + return Buffer.from(JSON.stringify(typeof value.toWire === "function" ? value.toWire() : value), "utf8"); +} + +// ---------------------------------------------------------------------------- describe + +function describe(Adapter) { + const warnings = []; + try { + new Adapter(); + } catch (e) { + warnings.push(`the adapter could not be built without its settings (${e.message}); settings it declares when built are missing`); + } + const commands = Object.entries(commandsOf(Adapter)).map(([name, spec]) => ({ + name, + description: spec.description ?? null, + inputSchema: schemaOf(spec.input), + outputSchema: schemaOf(spec.output), + returnsValue: spec.output !== undefined && spec.output !== null, + })); + return { + describeVersion: 1, + sdkLanguage: SDK_LANGUAGE, + sdkVersion: SDK_VERSION, + lifecycle: isResident(Adapter) ? "resident" : "classic", + protocol: { min: PROTOCOL, max: PROTOCOL }, + settings: declaredSettings(), + commands, + kinds: kindsOf(Adapter), + contracts: contractsOf(Adapter), + warnings, + }; +} + +// ---------------------------------------------------------------------------- the connection + +/** One gRPC bidirectional stream, Attach, over the host's Unix socket. */ +class Stream { + constructor(socketPath) { + this.session = http2.connect("http://localhost", { + createConnection: () => net.connect(socketPath), + settings: { initialWindowSize: 16 * 1024 * 1024 }, + maxSessionMemory: 256, + }); + this.session.on("error", () => {}); + // The connection's own window starts at 64 KB whatever the settings say; widen it once connected. + this.session.on("connect", () => { try { this.session.setLocalWindowSize(16 * 1024 * 1024); } catch {} }); + this.call = this.session.request({ + ":method": "POST", ":path": ATTACH, "content-type": "application/grpc", te: "trailers", + "user-agent": "simplyworks-serverless-node", + }); + this.buffer = Buffer.alloc(0); + this.waiting = []; + this.queue = []; + this.ended = false; + this.error = null; + this.call.on("data", (chunk) => this.onData(chunk)); + this.call.on("trailers", (headers) => { + const status = headers["grpc-status"]; + if (status !== undefined && String(status) !== "0") this.error = new Error(`gRPC status ${status}: ${headers["grpc-message"] ?? ""}`); + }); + this.call.on("response", (headers) => { + const status = headers["grpc-status"]; + if (status !== undefined && String(status) !== "0") this.error = new Error(`gRPC status ${status}: ${headers["grpc-message"] ?? ""}`); + }); + const end = (err) => { + if (err && !this.error) this.error = err; + this.ended = true; + for (const w of this.waiting.splice(0)) w(null); + }; + this.call.on("end", () => end()); + this.call.on("close", () => end()); + this.call.on("error", (e) => end(e)); + } + + onData(chunk) { + this.buffer = this.buffer.length ? Buffer.concat([this.buffer, chunk]) : chunk; + while (this.buffer.length >= 5) { + if (this.buffer[0] !== 0) { this.error = new Error("compressed messages are not supported"); this.call.close(); return; } + const length = this.buffer.readUInt32BE(1); + if (this.buffer.length < 5 + length) return; + const message = this.buffer.subarray(5, 5 + length); + this.buffer = this.buffer.subarray(5 + length); + const waiter = this.waiting.shift(); + if (waiter) waiter(message); else this.queue.push(message); + } + } + + /** The next message, or null once the host has ended the call. */ + receive() { + if (this.queue.length) return Promise.resolve(this.queue.shift()); + if (this.ended) return Promise.resolve(null); + return new Promise((resolve) => this.waiting.push(resolve)); + } + + send(message) { + if (message.length > MAX_MESSAGE) throw new Error(`a message of ${message.length} bytes is more than the ${MAX_MESSAGE} the host accepts`); + const head = Buffer.alloc(5); + head.writeUInt32BE(message.length, 1); + return new Promise((resolve, reject) => { + if (this.ended || this.call.destroyed) return reject(new Error("the host closed the stream")); + // write's callback runs once the data is handed on, which is the flow control that matters. + this.call.write(Buffer.concat([head, message]), (err) => (err ? reject(err) : resolve())); + }); + } + + close() { + try { this.call.end(); } catch {} + try { this.session.close(); } catch {} + } +} + +// ---------------------------------------------------------------------------- the runner + +class Context { + constructor(runner, sessionId, command, signal) { + this._runner = runner; + this.sessionId = sessionId; + this.command = command; + /** Aborted when the host gives up on the call. */ + this.signal = signal; + } + + get adapterId() { return this._runner.handshake.adapterId ?? ""; } + get instanceKey() { return this._runner.handshake.instanceKey ?? ""; } + /** Aborted once the host has asked the adapter to stop. */ + get stopping() { return this._runner.stopping.signal; } + + valueOf(name, fallback) { return valueOf(name, fallback); } + + /** + * Hands an event to the host and waits until it is persisted. Acknowledge the source only after + * this resolves: that is what makes delivery at-least-once. Resolves to the host's reference. + */ + async publish(payload, { dedupeKey = "", contentType = "", headers = {}, endpoint = "" } = {}) { + const ack = await this._runner.request("event", { + payload: encodeResult(payload), dedupe_key: dedupeKey, content_type: contentType, headers, endpoint, + }); + if (!ack.accepted) { + const error = ack.error ?? {}; + throw new AdapterError(error.message || "the host did not accept the event", { type: error.type }); + } + return ack.reference ?? ""; + } + + async getState(name) { + const result = await this._state(0, name); + return result.found ? result.value ?? "" : null; + } + + async setState(name, value) { await this._state(1, name, value); } + async deleteState(name) { await this._state(2, name); } + + async _state(op, name, value = "") { + const result = await this._runner.request("state", { op, name, value }); + if (result.error) throw new AdapterError(result.error.message || "state request failed", { type: result.error.type }); + return result; + } + + metric(name, value, tags = {}) { + this._runner.telemetry({ metric: { name, value: Number(value), tags } }); + } +} + +const contextStorage = new AsyncLocalStorage(); +let currentRunner = null; + +/** The current call's Context, or the adapter's own outside a call. */ +function context() { + const ctx = contextStorage.getStore() ?? currentRunner?.rootContext; + if (!ctx) throw new Error("there is no adapter context outside a running adapter"); + return ctx; +} + +function errorType(e) { + if (e && e.type) return e.type; + return (e && e.constructor && e.constructor.name) || "Error"; +} + +class Runner { + constructor(adapter) { + this.adapter = adapter; + this.cls = adapter.constructor; + this.commands = commandsOf(this.cls); + this.handshake = {}; + this.nextId = 1; + this.pending = new Map(); + this.running = new Map(); + this.stopping = new AbortController(); + this.minimumLevel = 1; + this.ready = null; + this.readyResolve = null; + this.telemetryQueue = []; + } + + hello() { + return { + token: this.handshake.token ?? "", + adapter_id: this.handshake.adapterId ?? "", + instance_key: this.handshake.instanceKey ?? "", + protocol_version: PROTOCOL, + sdk_version: SDK_VERSION, + sdk_language: SDK_LANGUAGE, + capabilities: this.capabilities(), + commands: Object.entries(this.commands).map(([name, spec]) => ({ + name, + parameter_type: spec.input === undefined || spec.input === null ? "" : typeof spec.input === "string" ? spec.input : "object", + returns_value: spec.output !== undefined && spec.output !== null, + description: spec.description ?? "", + input_schema: spec.input === undefined || spec.input === null ? "" : JSON.stringify(schemaOf(spec.input)), + output_schema: spec.output === undefined || spec.output === null ? "" : JSON.stringify(schemaOf(spec.output)), + })), + settings: declaredSettings().map((s) => ({ + name: s.name, description: s.description ?? "", required: s.required, secret: s.secret, + default_value: s.default ?? "", type: s.type, + })), + kinds: kindsOf(this.cls), + contracts: contractsOf(this.cls), + }; + } + + capabilities() { + const caps = []; + if (isResident(this.cls)) caps.push("resident"); + if (typeof this.adapter.reset === "function") caps.push("resettable"); + caps.push("cancel"); + for (const name of Object.keys(this.commands)) caps.push(`command:${name}`); + return caps; + } + + async run(line) { + this.handshake = JSON.parse(line); + if (Number(this.handshake.protocol ?? 2) < PROTOCOL) + throw new Error(`the host speaks protocol ${this.handshake.protocol}; this SDK speaks ${PROTOCOL}`); + if (!this.handshake.socket) throw new Error("the handshake names no socket; this SDK runs on Linux and macOS hosts"); + + this.ready = new Promise((resolve) => (this.readyResolve = resolve)); + this.rootContext = new Context(this, null, null, this.stopping.signal); + currentRunner = this; + this.stream = new Stream(this.handshake.socket); + await this.send({ hello: this.hello() }); + + try { + for (;;) { + const data = await Promise.race([ + this.stream.receive(), + new Promise((resolve) => this.stopping.signal.addEventListener("abort", () => resolve(null), { once: true })), + ]); + if (data === null) break; + const frame = wire.decode("HostFrame", data); + if (await this.onFrame(frame)) break; + } + if (this.stream.error && !this.stopping.signal.aborted) throw this.stream.error; + } finally { + this.stopping.abort(); + await this.flushTelemetry(); + this.stream.close(); + } + } + + async onFrame(frame) { + const id = frame.id ?? 0; + if (frame.ready) { + startupValues.clear(); + for (const [k, v] of Object.entries(frame.ready.startup_values ?? {})) startupValues.set(k, v); + if (typeof this.adapter.start === "function") await this.adapter.start(); + this.readyResolve(); + } else if (frame.invoke) { + this.startInvoke(id, frame.invoke); + } else if (frame.cancel) { + this.running.get(id)?.abort(); + } else if (frame.ping) { + this.pong(id); + } else if (frame.reset) { + this.reset(id, frame.reset.session_id ?? ""); + } else if (frame.set_log_level) { + this.minimumLevel = frame.set_log_level.level ?? 0; + } else if (frame.shutdown) { + await this.shutdown(frame.shutdown); + return true; + } else { + const answer = frame.state_result ?? frame.event_ack; + if (answer) { + const waiter = this.pending.get(id); + if (waiter) { this.pending.delete(id); waiter(answer); } + } + } + return false; + } + + startInvoke(id, invoke) { + const controller = new AbortController(); + this.running.set(id, controller); + const task = this.invoke(id, invoke, controller).finally(() => this.running.delete(id)); + controller.task = task; + } + + async invoke(id, invoke, controller) { + const name = invoke.command ?? ""; + let timer; + try { + await this.ready; + const spec = this.commands[name]; + if (!spec) throw new AdapterError(`the adapter has no command named ${name}`, { type: "MissingMethodException" }); + const method = this.adapter[spec.method]; + if (typeof method !== "function") throw new AdapterError(`the adapter has no method ${spec.method} for ${name}`, { type: "MissingMethodException" }); + + const takes = spec.input !== undefined && spec.input !== null; + const args = takes ? [decodeArgument(spec.input, invoke.payload ?? Buffer.alloc(0))] : []; + const ctx = new Context(this, invoke.session_id || String(id), name, controller.signal); + const cancelled = new Promise((_, reject) => { + const fail = () => reject(new AdapterError("the call was cancelled", { type: "OperationCanceledException" })); + if (controller.signal.aborted) fail(); + controller.signal.addEventListener("abort", fail, { once: true }); + }); + if ((invoke.timeout_seconds ?? 0) > 0) timer = setTimeout(() => controller.abort(), invoke.timeout_seconds * 1000); + + const result = await Promise.race([ + callStorage.run({ properties: invoke.properties ?? {} }, () => + contextStorage.run(ctx, () => Promise.resolve().then(() => method.apply(this.adapter, args)))), + cancelled, + ]); + const payload = spec.output !== undefined && spec.output !== null ? encodeResult(result) : Buffer.alloc(0); + await this.send({ id, invoke_result: { payload } }); + } catch (e) { + await this.send({ id, invoke_result: { error: { type: errorType(e), message: String(e?.message ?? e), detail: e?.detail ?? e?.stack ?? "" } } }).catch(() => {}); + } finally { + clearTimeout(timer); + } + } + + async pong(id) { + const pong = { connected: true, state: "Running", in_flight: this.running.size }; + if (typeof this.adapter.status === "function") { + try { + const s = (await this.adapter.status()) ?? {}; + if ("connected" in s) pong.connected = Boolean(s.connected); + if (s.state) pong.state = s.state; + if (s.inFlight !== undefined) pong.in_flight = s.inFlight; + if (s.lastError) pong.last_error = s.lastError; + if (s.lastMessageOn) pong.last_message_unix_ms = new Date(s.lastMessageOn).getTime(); + if (s.details) pong.details = Object.fromEntries(Object.entries(s.details).map(([k, v]) => [k, String(v)])); + } catch (e) { + Object.assign(pong, { connected: false, state: "StatusFailed", last_error: String(e?.message ?? e) }); + } + } + await this.send({ id, pong }).catch(() => {}); + } + + async reset(id, sessionId) { + const result = {}; + if (typeof this.adapter.reset === "function") { + try { await this.adapter.reset(sessionId); } + catch (e) { result.error = { type: errorType(e), message: String(e?.message ?? e), detail: e?.stack ?? "" }; } + } + await this.send({ id, invoke_result: result }).catch(() => {}); + } + + async shutdown(shutdown) { + // Ask it to stop first, let commands already running answer, and only then go: a drain + // promised those answers. + const deadline = Date.now() + (shutdown.drain ? 30000 : 5000); + const within = (promise) => Promise.race([promise, new Promise((r) => setTimeout(r, Math.max(100, deadline - Date.now())))]); + if (typeof this.adapter.stop === "function") { + try { await within(Promise.resolve(this.adapter.stop())); } catch (e) { this.log(3, `stop() failed: ${e?.message ?? e}`); } + } + const tasks = [...this.running.values()].map((c) => c.task).filter(Boolean); + if (tasks.length) await within(Promise.allSettled(tasks)); + } + + send(frame) { + return this.stream.send(wire.encode("AdapterFrame", frame)); + } + + /** Sends a frame of the adapter's own and waits for the host's answer to it. */ + request(kind, body) { + const id = this.nextId++; + return new Promise((resolve, reject) => { + if (this.stopping.signal.aborted) return reject(new AdapterError("the adapter is stopping", { type: "OperationCanceledException" })); + this.pending.set(id, resolve); + this.stopping.signal.addEventListener("abort", () => { + if (this.pending.delete(id)) reject(new AdapterError("the adapter is stopping", { type: "OperationCanceledException" })); + }, { once: true }); + this.send({ id, [kind]: body }).catch((e) => { this.pending.delete(id); reject(e); }); + }); + } + + /** Logs and metrics: sent in the background, and dropped rather than allowed to hold anything up. */ + telemetry(frame) { + if (this.telemetryQueue.length > 10000) return; + this.telemetryQueue.push(frame); + if (this.telemetryQueue.length === 1) queueMicrotask(() => this.flushTelemetry()); + } + + async flushTelemetry() { + while (this.telemetryQueue.length) { + const frame = this.telemetryQueue.shift(); + try { await this.send(frame); } catch { this.telemetryQueue.length = 0; return; } + } + } + + log(level, message, error) { + if (level < this.minimumLevel) return; + this.telemetry({ log: { level, message: String(message), exception: error ? String(error.stack ?? error) : "", timestamp_unix_ms: Date.now() } }); + } +} + +/** Logs that reach the host: sw.log.info("…"), and debug, warn, error, critical. */ +const log = Object.fromEntries(Object.entries(LEVELS).map(([name, level]) => [name, (message, error) => { + if (currentRunner) currentRunner.log(level, message, error); + else (level >= 3 ? console.error : console.log)(message, error ?? ""); +}])); + +function checkKinds(adapter) { + if (typeof adapter.__swCheck === "function") adapter.__swCheck(); +} + +/** + * Runs the adapter: describes it for --describe, otherwise serves the host that started it. + * `Adapter` is a class built with no arguments. + */ +function run(Adapter) { + if (process.argv.slice(2).includes(DESCRIBE_FLAG)) { + process.stdout.write(JSON.stringify(describe(Adapter), null, 2) + "\n"); + return Promise.resolve(); + } + + const adapter = new Adapter(); + checkKinds(adapter); + + const input = readline.createInterface({ input: process.stdin }); + let runner = null; + return new Promise((resolve) => { + let first = true; + input.on("line", (line) => { + if (!first) return; + first = false; + runner = new Runner(adapter); + runner.run(line).then( + () => { resolve(); process.exit(0); }, + (e) => { console.error(e?.stack ?? e); process.exit(1); }); + }); + // Stdin stays open while the host lives; end of file means it's gone. + input.on("close", () => { + if (first) { console.error("stdin closed before the handshake arrived; the host is gone"); process.exit(1); } + runner?.stopping.abort(); + }); + }); +} + +module.exports = { + AdapterError, Context, SDK_VERSION, context, declaredSettings, describe, expect, log, run, valueOf, + startupValues: () => Object.fromEntries(startupValues), + _internal: { commandsOf, decodeArgument, encodeResult, schemaOf, wire }, +}; diff --git a/sdk/node/src/wire.js b/sdk/node/src/wire.js new file mode 100644 index 0000000..bb16d6a --- /dev/null +++ b/sdk/node/src/wire.js @@ -0,0 +1,218 @@ +"use strict"; +/** + * The protobuf wire format, for the messages in adapter.proto and nothing else. + * + * Hand-written rather than generated so the SDK needs no packages: messages are plain objects keyed + * by the proto's field names, and each schema below mirrors adapter.proto field for field. A field + * the schema doesn't know is skipped when decoding, as protobuf does, so a newer host stays readable. + * The same schemas as the Python SDK's _wire.py. + */ + +const VARINT = 0, FIXED64 = 1, LENGTH = 2, FIXED32 = 5; + +const SCHEMAS = { + // host -> adapter + HostFrame: { + 1: ["id", "int"], 2: ["traceparent", "string"], + 3: ["ready", ["message", "Ready"]], 4: ["invoke", ["message", "Invoke"]], + 5: ["ping", ["message", "Empty"]], 6: ["set_log_level", ["message", "SetLogLevel"]], + 7: ["reset", ["message", "Reset"]], 8: ["shutdown", ["message", "Shutdown"]], + 9: ["event_ack", ["message", "EventAck"]], 10: ["state_result", ["message", "StateResult"]], + 11: ["cancel", ["message", "Empty"]], + }, + Empty: {}, + Ready: { 1: ["max_in_flight", "int"], 2: ["startup_values", "map"], 3: ["adapter_values", "map"] }, + Invoke: { + 1: ["command", "string"], 2: ["payload", "bytes"], 3: ["timeout_seconds", "int"], + 4: ["session_id", "string"], 5: ["properties", "map"], + }, + SetLogLevel: { 1: ["level", "int"] }, + Reset: { 1: ["session_id", "string"] }, + Shutdown: { 1: ["reason", "string"], 2: ["drain", "bool"] }, + StateResult: { 1: ["found", "bool"], 2: ["value", "string"], 3: ["error", ["message", "Error"]] }, + EventAck: { 1: ["accepted", "bool"], 2: ["reference", "string"], 3: ["error", ["message", "Error"]] }, + // adapter -> host + AdapterFrame: { + 1: ["id", "int"], 2: ["traceparent", "string"], + 3: ["hello", ["message", "Hello"]], 4: ["invoke_result", ["message", "InvokeResult"]], + 5: ["event", ["message", "Event"]], 6: ["log", ["message", "LogEntry"]], + 7: ["metric", ["message", "Metric"]], 8: ["pong", ["message", "Pong"]], + 9: ["state", ["message", "StateRequest"]], + }, + Hello: { + 1: ["token", "string"], 2: ["adapter_id", "string"], 3: ["instance_key", "string"], + 4: ["protocol_version", "int"], 5: ["sdk_version", "string"], + 6: ["capabilities", ["repeated", "string"]], 7: ["commands", ["repeated_message", "CommandInfo"]], + 8: ["sdk_language", "string"], 9: ["settings", ["repeated_message", "SettingInfo"]], + 10: ["kinds", ["repeated", "string"]], 11: ["contracts", "map_int"], + }, + SettingInfo: { + 1: ["name", "string"], 2: ["description", "string"], 3: ["required", "bool"], + 4: ["secret", "bool"], 5: ["default_value", "string"], 6: ["type", "string"], + }, + CommandInfo: { + 1: ["name", "string"], 2: ["parameter_type", "string"], 3: ["parameter_schema", "string"], + 4: ["returns_value", "bool"], 5: ["description", "string"], + 6: ["input_schema", "string"], 7: ["output_schema", "string"], + }, + InvokeResult: { 1: ["payload", "bytes"], 2: ["error", ["message", "Error"]] }, + Error: { 1: ["type", "string"], 2: ["message", "string"], 3: ["detail", "string"] }, + Event: { + 1: ["payload", "bytes"], 2: ["dedupe_key", "string"], 3: ["content_type", "string"], + 4: ["headers", "map"], 5: ["endpoint", "string"], + }, + StateRequest: { 1: ["op", "int"], 2: ["name", "string"], 3: ["value", "string"] }, + LogEntry: { + 1: ["level", "int"], 2: ["message", "string"], 3: ["exception", "string"], + 4: ["properties", "map"], 5: ["timestamp_unix_ms", "int"], + }, + Metric: { 1: ["name", "string"], 2: ["value", "double"], 3: ["tags", "map"] }, + Pong: { + 1: ["connected", "bool"], 2: ["state", "string"], 3: ["last_message_unix_ms", "int"], + 4: ["in_flight", "int"], 5: ["last_error", "string"], 6: ["details", "map"], + }, + _MapEntry: { 1: ["key", "string"], 2: ["value", "string"] }, + _MapIntEntry: { 1: ["key", "string"], 2: ["value", "int"] }, +}; + +const BY_NAME = Object.fromEntries(Object.entries(SCHEMAS).map(([msg, fields]) => + [msg, Object.fromEntries(Object.entries(fields).map(([n, [name, kind]]) => [name, [Number(n), kind]]))])); + +class WireError extends Error {} + +// ---------------------------------------------------------------------------- primitives + +function varint(value) { + let v = BigInt(value); + if (v < 0n) v += 1n << 64n; // int32/int64 negatives are ten-byte two's complement + const out = []; + do { + let byte = Number(v & 0x7fn); + v >>= 7n; + if (v) byte |= 0x80; + out.push(byte); + } while (v); + return Buffer.from(out); +} + +function readVarint(data, pos) { + let result = 0n; + let shift = 0n; + for (;;) { + if (pos >= data.length) throw new WireError("truncated varint"); + const byte = data[pos++]; + result |= BigInt(byte & 0x7f) << shift; + if (!(byte & 0x80)) return [result, pos]; + shift += 7n; + if (shift > 63n) throw new WireError("varint too long"); + } +} + +const key = (number, wireType) => varint((number << 3) | wireType); +const lengthDelimited = (number, payload) => Buffer.concat([key(number, LENGTH), varint(payload.length), payload]); + +// ---------------------------------------------------------------------------- encode + +function encode(messageName, message) { + const fields = BY_NAME[messageName]; + const parts = []; + for (const [name, value] of Object.entries(message)) { + if (value === undefined || value === null || !(name in fields)) continue; + const [number, kind] = fields[name]; + parts.push(encodeField(number, kind, value)); + } + return Buffer.concat(parts); +} + +function encodeField(number, kind, value) { + if (Array.isArray(kind)) { + const [tag, inner] = kind; + if (tag === "message") return lengthDelimited(number, encode(inner, value)); + if (tag === "repeated") return Buffer.concat(value.map((item) => encodeField(number, inner, item))); + if (tag === "repeated_message") return Buffer.concat(value.map((item) => lengthDelimited(number, encode(inner, item)))); + throw new WireError(`unknown kind ${kind}`); + } + // proto3: default values are not written + switch (kind) { + case "int": return value ? Buffer.concat([key(number, VARINT), varint(Math.trunc(Number(value)))]) : Buffer.alloc(0); + case "bool": return value ? Buffer.concat([key(number, VARINT), Buffer.from([1])]) : Buffer.alloc(0); + case "double": { + if (!value) return Buffer.alloc(0); + const b = Buffer.alloc(8); + b.writeDoubleLE(Number(value)); + return Buffer.concat([key(number, FIXED64), b]); + } + case "string": return value ? lengthDelimited(number, Buffer.from(String(value), "utf8")) : Buffer.alloc(0); + case "bytes": return value && value.length ? lengthDelimited(number, Buffer.from(value)) : Buffer.alloc(0); + case "map": + return Buffer.concat(Object.entries(value).filter(([, v]) => v !== undefined && v !== null).map(([k, v]) => + lengthDelimited(number, Buffer.concat([encodeField(1, "string", k), encodeField(2, "string", v)])))); + case "map_int": + return Buffer.concat(Object.entries(value).map(([k, v]) => + lengthDelimited(number, Buffer.concat([encodeField(1, "string", k), encodeField(2, "int", v)])))); + default: throw new WireError(`unknown kind ${kind}`); + } +} + +// ---------------------------------------------------------------------------- decode + +function decode(messageName, data) { + const fields = SCHEMAS[messageName]; + const message = {}; + let pos = 0; + while (pos < data.length) { + let k; + [k, pos] = readVarint(data, pos); + const number = Number(k >> 3n); + const wireType = Number(k & 7n); + let raw; + if (wireType === VARINT) [raw, pos] = readVarint(data, pos); + else if (wireType === FIXED64) { raw = data.subarray(pos, pos + 8); pos += 8; } + else if (wireType === FIXED32) { raw = data.subarray(pos, pos + 4); pos += 4; } + else if (wireType === LENGTH) { + let length; + [length, pos] = readVarint(data, pos); + length = Number(length); + raw = data.subarray(pos, pos + length); + if (raw.length !== length) throw new WireError("truncated field"); + pos += length; + } else throw new WireError(`unsupported wire type ${wireType}`); + + if (!(number in fields)) continue; // a field from a newer host + const [name, kind] = fields[number]; + decodeField(message, name, kind, raw); + } + return message; +} + +function decodeField(message, name, kind, raw) { + if (Array.isArray(kind)) { + const [tag, inner] = kind; + if (tag === "message") message[name] = decode(inner, raw); + else if (tag === "repeated") (message[name] ??= []).push(scalar(inner, raw)); + else if (tag === "repeated_message") (message[name] ??= []).push(decode(inner, raw)); + return; + } + if (kind === "map" || kind === "map_int") { + const entry = decode(kind === "map" ? "_MapEntry" : "_MapIntEntry", raw); + (message[name] ??= {})[entry.key ?? ""] = entry.value ?? (kind === "map_int" ? 0 : ""); + return; + } + message[name] = scalar(kind, raw); +} + +function scalar(kind, raw) { + switch (kind) { + case "int": { + const signed = raw >= 1n << 63n ? raw - (1n << 64n) : raw; + return Number(signed); + } + case "bool": return raw !== 0n; + case "double": return raw.readDoubleLE(0); + case "string": return Buffer.from(raw).toString("utf8"); + case "bytes": return Buffer.from(raw); + default: throw new WireError(`unknown kind ${kind}`); + } +} + +module.exports = { encode, decode, WireError, SCHEMAS, _key: key, VARINT, LENGTH }; diff --git a/sdk/node/test/sdk.test.js b/sdk/node/test/sdk.test.js new file mode 100644 index 0000000..2e093c9 --- /dev/null +++ b/sdk/node/test/sdk.test.js @@ -0,0 +1,91 @@ +"use strict"; +const test = require("node:test"); +const assert = require("node:assert"); +const sw = require("../src"); +const { wire, decodeArgument, encodeResult, commandsOf } = sw._internal; + +test("a frame round-trips with every kind of field", () => { + const frame = { + id: 42, hello: { + token: "t", protocol_version: 2, capabilities: ["cancel", "command:Greet"], + commands: [{ name: "Greet", returns_value: true, input_schema: '{"type":"string"}' }], + settings: [{ name: "Url", required: true, secret: true }], kinds: ["handler"], contracts: { bitween: 1 }, + }, + }; + assert.deepStrictEqual(wire.decode("AdapterFrame", wire.encode("AdapterFrame", frame)), frame); +}); + +test("the bytes are the Python SDK's, and .NET's protobuf", () => { + // Written by simplyworks_serverless._wire for the same frame. + const frame = { id: -7, metric: { name: "m", value: 2.5, tags: { a: "b" } } }; + assert.strictEqual(wire.encode("AdapterFrame", frame).toString("hex"), + "08f9ffffffffffffffff013a140a016d1100000000000004401a060a0161120162"); +}); + +test("an empty message is present, and unknown fields are skipped", () => { + assert.deepStrictEqual(wire.decode("HostFrame", Buffer.from([0x2a, 0x00])), { ping: {} }); + const unknown = Buffer.concat([Buffer.from([0x9a, 0x06, 0x03]), Buffer.from("abc"), wire.encode("HostFrame", { id: 3 })]); + assert.deepStrictEqual(wire.decode("HostFrame", unknown), { id: 3 }); +}); + +test("strings are raw text, bytes raw, and anything else JSON", () => { + assert.strictEqual(encodeResult("hé").toString("hex"), "68c3a9"); + assert.strictEqual(decodeArgument("string", Buffer.from("hé")), "hé"); + assert.deepStrictEqual(decodeArgument("json", Buffer.from('{"a":1}')), { a: 1 }); + assert.deepStrictEqual(decodeArgument("bytes", Buffer.from([1, 2])), Buffer.from([1, 2])); + assert.strictEqual(encodeResult(undefined).length, 0); + assert.strictEqual(encodeResult({ toWire: () => ({ X: 1 }) }).toString(), '{"X":1}'); +}); + +class Example { + static commands = { + Send: { method: "send", input: { type: "object" }, output: "string", description: "Sends it" }, + Ping: { method: "ping" }, + }; + constructor() { + sw.expect("Url", { description: "Where to send" }); + sw.expect("Retries", { default: 3, type: "number" }); + sw.expect("Key", { secret: true, required: false }); + } + send() { return "sent"; } + ping() {} +} + +test("describes settings, commands and lifecycle", () => { + const d = sw.describe(Example); + assert.strictEqual(d.sdkLanguage, "node"); + assert.strictEqual(d.lifecycle, "classic"); + assert.deepStrictEqual(d.protocol, { min: 2, max: 2 }); + const s = Object.fromEntries(d.settings.map((x) => [x.name, x])); + assert.strictEqual(s.Url.required, true); + assert.strictEqual(s.Retries.required, false); + assert.strictEqual(s.Retries.default, "3"); + assert.strictEqual(s.Key.secret, true); + assert.strictEqual(s.Key.required, false); + const c = Object.fromEntries(d.commands.map((x) => [x.name, x])); + assert.strictEqual(c.Send.description, "Sends it"); + assert.strictEqual(c.Send.returnsValue, true); + assert.strictEqual(c.Ping.returnsValue, false); + assert.strictEqual(c.Ping.inputSchema, null); +}); + +test("an adapter with a start method is resident, and commands are inherited", () => { + class Listener extends Example { + static commands = { Extra: { method: "send", output: "string" } }; + start() {} + } + assert.strictEqual(sw.describe(Listener).lifecycle, "resident"); + assert.deepStrictEqual(Object.keys(commandsOf(Listener)).sort(), ["Extra", "Ping", "Send"]); +}); + +test("an adapter that cannot be built is still described, with a warning", () => { + class Broken { constructor() { throw new Error("needs settings"); } } + assert.strictEqual(sw.describe(Broken).warnings.length, 1); +}); + +test("values come from startup, then the default, then the fallback", () => { + sw.expect("Mode", { default: "fast" }); + assert.strictEqual(sw.valueOf("Mode"), "fast"); + assert.strictEqual(sw.valueOf("Missing"), undefined); + assert.strictEqual(sw.valueOf("Missing", "x"), "x"); +}); From 6d3f739858788ace2586c41b297b9d610a55f579 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 16:42:45 +0300 Subject: [PATCH 06/17] Run Node adapters through the real host in the tests The same ground as the Python tests, with the adapters built by serverless build: classic sessions, a resident instance, large payloads both ways, errors, a cancelled call, concurrent calls, and the Bitween handler and receiver in JavaScript and validator in TypeScript. Tests needing Node to strip TypeScript types are inconclusive on a Node older than 22.13. --- SW.Serverless.UnitTests/NodeAdapterTests.cs | 384 ++++++++++++++++++ .../NodeAdapters/bitween_handler.js | 19 + .../NodeAdapters/bitween_receiver.js | 28 ++ .../NodeAdapters/bitween_validator.ts | 17 + .../NodeAdapters/classic.js | 41 ++ .../NodeAdapters/resident.js | 38 ++ 6 files changed, 527 insertions(+) create mode 100644 SW.Serverless.UnitTests/NodeAdapterTests.cs create mode 100644 SW.Serverless.UnitTests/NodeAdapters/bitween_handler.js create mode 100644 SW.Serverless.UnitTests/NodeAdapters/bitween_receiver.js create mode 100644 SW.Serverless.UnitTests/NodeAdapters/bitween_validator.ts create mode 100644 SW.Serverless.UnitTests/NodeAdapters/classic.js create mode 100644 SW.Serverless.UnitTests/NodeAdapters/resident.js diff --git a/SW.Serverless.UnitTests/NodeAdapterTests.cs b/SW.Serverless.UnitTests/NodeAdapterTests.cs new file mode 100644 index 0000000..dfd7ee1 --- /dev/null +++ b/SW.Serverless.UnitTests/NodeAdapterTests.cs @@ -0,0 +1,384 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json.Linq; +using SW.CloudFiles.Extensions; +using SW.PrimitiveTypes; +using SW.Serverless.Resident; +using SW.Serverless.UnitTests.Fixtures; +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace SW.Serverless.UnitTests +{ + /// + /// Adapters written in JavaScript and TypeScript with @simplyworks/serverless, built by serverless + /// build and run by the real host: classic sessions as Bitween runs handlers, resident instances, + /// and the Bitween kinds written with @simplyworks/bitween — the validator in TypeScript. + /// + [TestClass] + public class NodeAdapterTests + { + static IHost host; + static string workDirectory; + + static readonly (string Id, string Script, string Lifecycle)[] Adapters = + { + ("test.node.classic", "classic.js", "classic"), + ("test.node.resident", "resident.js", "resident"), + ("test.node.handler", "bitween_handler.js", "classic"), + ("test.node.receiver", "bitween_receiver.js", "classic"), + ("test.node.validator", "bitween_validator.ts", "classic"), + }; + + [ClassInitialize] + public static async Task ClassInitialize(TestContext context) + { + workDirectory = Path.Combine(Path.GetTempPath(), "swsl-node", Guid.NewGuid().ToString("N")); + host = Host.CreateDefaultBuilder() + .ConfigureLogging(l => l.ClearProviders()) + .ConfigureServices(s => + { + s.AddLocalTestsCloudFiles(o => o.BucketName = TestStore.BucketName + "-node"); + s.AddServerless(o => + { + o.AdapterRemotePath = "adapters"; + o.AdapterLocalPath = Path.Combine(workDirectory, "installed"); + o.AdapterMetadataCacheDuration = 1; + o.CommandTimeout = 30; + }); + s.AddSingleton(); + s.AddSingleton(sp => sp.GetRequiredService()); + s.AddResidentAdapters(o => + { + o.SocketPath = $"/tmp/swsl-nd{Environment.ProcessId}.sock"; + o.PipeName = $"swsl-nd{Environment.ProcessId}"; + o.HeartbeatInterval = TimeSpan.FromSeconds(2); + o.HandshakeTimeout = TimeSpan.FromSeconds(60); + }); + }) + .Build(); + await host.StartAsync(); + + var files = host.Services.GetRequiredService(); + foreach (var (id, script, lifecycle) in Adapters) + { + if (script.EndsWith(".ts") && !NodePackage.CanStripTypes) continue; + var (zip, entry) = await NodePackage.BuildAsync(workDirectory, id, script); + await using var package = File.OpenRead(zip); + await files.WriteAsync(package, new WriteFileSettings + { + Key = $"adapters-versions/{id}/1.0.0", + ContentType = "application/zip", + Metadata = new Dictionary + { + ["EntryAssembly"] = entry, + ["Hash"] = "nd-" + Guid.NewGuid().ToString("N")[..12], + ["Protocol"] = "2", + ["Lifecycle"] = lifecycle, + } + }); + } + } + + [ClassCleanup] + public static async Task ClassCleanup() + { + if (host != null) await host.StopAsync(); + host?.Dispose(); + try { Directory.Delete(workDirectory, true); } catch { } + } + + static IServerlessService Service() => host.Services.GetRequiredService(); + static IResidentAdapterHost Residents() => host.Services.GetRequiredService(); + + static async Task InSession(string adapterId, IDictionary values, Func> call) + { + var service = Service(); + await service.StartAsync($"{adapterId}/1.0.0", "corr-nd", values ?? new Dictionary()); + try { return await call(service); } + finally { ((IDisposable)service).Dispose(); } + } + + [TestMethod] + public async Task A_classic_node_adapter_answers_with_the_values_it_was_started_with() + { + await InSession("test.node.classic", new Dictionary { ["Prefix"] = "hi " }, async s => + { + Assert.AreEqual("hi world", await s.InvokeAsync("Greet", "world")); + Assert.AreEqual("corr-nd", await s.InvokeAsync("Correlation", null)); + Assert.AreEqual(5, await s.InvokeAsync("Add", new { A = 2, B = 3 })); + // Unicode survives both ways. + Assert.AreEqual("hi مرحبا ✓", await s.InvokeAsync("Greet", "مرحبا ✓")); + return 0; + }); + } + + [TestMethod] + public async Task Payloads_past_the_http2_window_cross_both_ways() + { + // 64 KB is HTTP/2's default window: these only arrive if flow control works. + await InSession("test.node.classic", null, async s => + { + var big = await s.InvokeAsync("Big", 3 * 1024 * 1024); + Assert.AreEqual(3 * 1024 * 1024, big.Length); + Assert.AreEqual(2 * 1024 * 1024 + 7, await s.InvokeAsync("Length", new string('y', 2 * 1024 * 1024 + 7))); + return 0; + }); + } + + [TestMethod] + public async Task An_error_reaches_the_caller_with_its_type_and_message() + { + await InSession("test.node.classic", null, async s => + { + // Raised as the adapter raised it: its type, as the adapter named it, and its message. + var rejected = await Assert.ThrowsExceptionAsync(() => s.InvokeAsync("Fail", "no stock")); + Assert.AreEqual("Acme.Rejected", rejected.AdapterExceptionType); + StringAssert.Contains(rejected.Message, "no stock"); + + var crashed = await Assert.ThrowsExceptionAsync(() => s.InvokeAsync("Crash", null)); + Assert.AreEqual("TypeError", crashed.AdapterExceptionType); + StringAssert.Contains(crashed.Detail, "main.js"); + + var unknown = await Assert.ThrowsExceptionAsync(() => s.InvokeAsync("Teleport", null)); + StringAssert.Contains(unknown.Message, "Teleport"); + + // The session is still good after all three. + Assert.AreEqual("x", await s.InvokeAsync("Big", 1)); + return 0; + }); + } + + [TestMethod] + public async Task A_call_that_times_out_is_cancelled_and_the_adapter_keeps_answering() + { + var residents = Residents(); + var instance = await residents.StartExclusiveAsync(new AdapterSpec + { + AdapterId = "test.node.classic/1.0.0", + InstanceKey = "nd-cancel", + StartupValues = new Dictionary { ["Prefix"] = "> " }, + }); + try + { + var started = DateTime.UtcNow; + await Assert.ThrowsExceptionAsync(() => instance.InvokeAsync("Slow", 30.0, timeoutSeconds: 1)); + Assert.IsTrue(DateTime.UtcNow - started < TimeSpan.FromSeconds(10)); + + // Several at once on one instance, while the cancelled one is gone. + var answers = await Task.WhenAll(Enumerable.Range(0, 20) + .Select(i => instance.InvokeAsync("Greet", "n" + i, timeoutSeconds: 15))); + CollectionAssert.AreEqual(Enumerable.Range(0, 20).Select(i => "> n" + i).ToArray(), answers); + } + finally + { + await residents.StopAsync("test.node.classic/1.0.0", "nd-cancel", drain: false); + } + } + + [TestMethod] + public async Task A_command_with_no_result_completes() + { + await InSession("test.node.classic", null, async s => + { + await s.InvokeAsync("Nothing", null); + return 0; + }); + } + + [TestMethod] + public async Task A_resident_node_adapter_starts_reports_status_publishes_keeps_state_resets_and_stops() + { + var residents = Residents(); + var sink = host.Services.GetRequiredService(); + var instance = await residents.StartExclusiveAsync(new AdapterSpec + { + AdapterId = "test.node.resident/1.0.0", + InstanceKey = "nd-resident", + }); + try + { + Assert.AreEqual(InstanceState.Ready, instance.State); + Assert.IsTrue(await instance.InvokeAsync("Started", timeoutSeconds: 15)); + + var described = residents.Describe().Single(d => d.InstanceKey == "nd-resident"); + Assert.AreEqual("node", described.SdkLanguage); + + var pong = await instance.PingAsync(TimeSpan.FromSeconds(10)); + Assert.AreEqual("Listening", pong.State); + Assert.IsTrue(pong.Connected); + + var reference = await instance.InvokeAsync("Publish", "order-1", timeoutSeconds: 15); + StringAssert.StartsWith(reference, "ref-"); + Assert.IsTrue(sink.Delivered.Any(d => d.Body == "order-1" && d.DedupeKey == "k-order-1" && d.Endpoint == "tests")); + + // An empty string is an empty payload, which reads back as null — as from a .NET adapter. + Assert.IsTrue(string.IsNullOrEmpty(await instance.InvokeAsync("Remember", "first", timeoutSeconds: 15))); + Assert.AreEqual("first", await instance.InvokeAsync("Remember", "second", timeoutSeconds: 15)); + await instance.InvokeAsync("Forget", timeoutSeconds: 15); + Assert.IsTrue(string.IsNullOrEmpty(await instance.InvokeAsync("Remember", "third", timeoutSeconds: 15))); + + await instance.ResetAsync("session-9"); + CollectionAssert.AreEqual(new[] { "session-9" }, await instance.InvokeAsync("Resets", timeoutSeconds: 15)); + } + finally + { + await residents.StopAsync("test.node.resident/1.0.0", "nd-resident", drain: true); + } + Assert.IsFalse(residents.Describe().Any(d => d.InstanceKey == "nd-resident")); + } + + [TestMethod] + public async Task A_bitween_handler_in_javascript_takes_and_returns_exchange_files_as_dotnet_ones() + { + var answer = await InSession("test.node.handler", new Dictionary { ["Partner"] = "acme" }, s => + s.InvokeAsync("Handle", new { Data = "{\"orderId\":\"SO-1\"}", Filename = "order.json", BadData = false })); + + Assert.AreEqual("answer.json", (string)answer["Filename"]); + Assert.IsFalse((bool)answer["BadData"]); + var data = JObject.Parse((string)answer["Data"]); + Assert.AreEqual("acme", (string)data["to"]); + Assert.AreEqual("SO-1", (string)data["orderId"]); + Assert.AreEqual("order.json", (string)data["from"]); + // Hash is what .NET's ExchangeFile computes: SHA-1 of Data, lower-case hex. + using var sha1 = System.Security.Cryptography.SHA1.Create(); + Assert.AreEqual(Convert.ToHexString(sha1.ComputeHash(Encoding.UTF8.GetBytes((string)answer["Data"]))).ToLowerInvariant(), + (string)answer["Hash"]); + + var rejected = await InSession("test.node.handler", new Dictionary { ["Partner"] = "acme" }, s => + s.InvokeAsync("Handle", new { Data = "{\"reject\":true}" })); + Assert.IsTrue((bool)rejected["BadData"], "a rejected delivery is returned, not raised"); + } + + [TestMethod] + public async Task A_bitween_validator_in_typescript_reports_each_failure() + { + NodePackage.RequireTypeStripping(); + var result = await InSession("test.node.validator", null, s => + s.InvokeAsync("Validate", new { Data = "{\"lines\":[]}" })); + + Assert.IsFalse((bool)result["Success"]); + CollectionAssert.AreEqual(new[] { "orderId", "lines" }, + result["Validations"]!.Select(v => (string)v["Key"]).ToArray()); + + var valid = await InSession("test.node.validator", null, s => + s.InvokeAsync("Validate", new { Data = "{\"orderId\":\"SO-1\",\"lines\":[1]}" })); + Assert.IsTrue((bool)valid["Success"]); + } + + [TestMethod] + public async Task A_bitween_receiver_in_javascript_runs_a_session_in_the_contract_s_order() + { + var root = Path.Combine(workDirectory, "receiver"); + var folder = Path.Combine(root, "inbox"); + Directory.CreateDirectory(folder); + File.WriteAllText(Path.Combine(folder, "a.json"), "{\"n\":1}"); + File.WriteAllText(Path.Combine(folder, "b.json"), "{\"n\":2}"); + + var taken = await InSession("test.node.receiver", new Dictionary { ["Folder"] = folder }, async s => + { + var got = new List(); + await s.InvokeAsync("Initialize", null); + foreach (var id in await s.InvokeAsync("ListFiles", null)) + { + var file = await s.InvokeAsync("GetFile", id); + got.Add((string)file["Filename"] + "=" + (string)file["Data"]); + await s.InvokeAsync("DeleteFile", id); + } + await s.InvokeAsync("Finalize", null); + return got; + }); + + CollectionAssert.AreEqual(new[] { "a.json={\"n\":1}", "b.json={\"n\":2}" }, taken); + Assert.AreEqual(0, Directory.GetFiles(folder).Length); + Assert.AreEqual("Initialize,ListFiles,GetFile,DeleteFile,GetFile,DeleteFile,Finalize", + File.ReadAllText(Path.Combine(root, "calls.txt"))); + } + + [TestMethod] + public async Task A_node_adapter_describes_itself_for_the_manifest_and_typescript_runs_as_javascript() + { + NodePackage.RequireTypeStripping(); + var (zip, entry) = await NodePackage.BuildAsync(workDirectory, "test.node.describe", "bitween_validator.ts"); + Assert.AreEqual("main.js", entry, "the TypeScript entry runs as the JavaScript Node strips it to"); + + using var archive = ZipFile.OpenRead(zip); + var manifest = Contract.Catalog.AdapterManifest.Parse(new StreamReader(archive.GetEntry("adapter.json")!.Open()).ReadToEnd()); + Assert.AreEqual("node", manifest.Runtime); + Assert.AreEqual("typescript", manifest.Language); + Assert.AreEqual(2, manifest.Protocol.Min); + CollectionAssert.AreEqual(new[] { "validator" }, manifest.Kinds); + Assert.AreEqual(1, manifest.Contracts["bitween"]); + Assert.IsNotNull(archive.GetEntry("node_modules/@simplyworks/serverless/src/index.js")); + Assert.IsNotNull(archive.GetEntry("node_modules/@simplyworks/bitween/src/index.js")); + Assert.IsNull(archive.GetEntry("main.ts"), "the package runs JavaScript"); + Assert.IsNotNull(archive.GetEntry("source/main.ts"), "the source is what was written"); + } + } + + /// A Node adapter's project — the script as main.js or main.ts, and its adapter.json — built by serverless build. + static class NodePackage + { + static readonly Lazy canStripTypes = new(() => + { + try + { + using var node = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo("node", + "-e \"process.exit(typeof require('node:module').stripTypeScriptTypes === 'function' ? 0 : 1)\"") + { RedirectStandardOutput = true, RedirectStandardError = true }); + node!.WaitForExit(); + return node.ExitCode == 0; + } + catch + { + return false; + } + }); + + /// Whether this machine's Node strips TypeScript types, which takes 22.13 or later. + public static bool CanStripTypes => canStripTypes.Value; + + public static void RequireTypeStripping() + { + if (!CanStripTypes) Assert.Inconclusive("this machine's node can't strip TypeScript types; it takes Node 22.13 or later"); + } + + public static async Task<(string Zip, string Entry)> BuildAsync(string work, string id, string script) + { + var root = RepositoryRoot(); + var project = Path.Combine(work, "projects", id); + Directory.CreateDirectory(project); + var entry = "main" + Path.GetExtension(script); + File.Copy(Path.Combine(root, "SW.Serverless.UnitTests", "NodeAdapters", script), Path.Combine(project, entry)); + File.WriteAllText(Path.Combine(project, "adapter.json"), new JObject + { + ["id"] = id, + ["version"] = "1.0.0", + ["runtime"] = "node", + ["entry"] = entry, + }.ToString()); + + var built = await Tooling.Building.PackageBuilder.BuildAsync(new Tooling.Building.BuildRequest + { + ProjectDirectory = project, + OutputDirectory = Path.Combine(work, "built", id), + }); + Assert.IsTrue(built.Succeeded, string.Join("; ", built.Problems)); + return (built.ZipPath, built.Manifest.Entry); + } + + static string RepositoryRoot() + { + var dir = new DirectoryInfo(AppContext.BaseDirectory); + while (dir != null && !File.Exists(Path.Combine(dir.FullName, "SW.Serverless.sln"))) dir = dir.Parent; + return dir!.FullName; + } + } +} diff --git a/SW.Serverless.UnitTests/NodeAdapters/bitween_handler.js b/SW.Serverless.UnitTests/NodeAdapters/bitween_handler.js new file mode 100644 index 0000000..4e924ba --- /dev/null +++ b/SW.Serverless.UnitTests/NodeAdapters/bitween_handler.js @@ -0,0 +1,19 @@ +// A Bitween handler written with @simplyworks/bitween. +const sw = require("@simplyworks/serverless"); +const { ExchangeFile, Handler } = require("@simplyworks/bitween"); + +class Orders extends Handler { + constructor() { + super(); + sw.expect("Partner", { description: "Who receives the orders" }); + } + + handle(file) { + const order = JSON.parse(file.data); + if (order.reject) return new ExchangeFile({ data: '{"error":"rejected"}', badData: true, contentType: "application/json" }); + const answer = { to: sw.valueOf("Partner"), orderId: order.orderId, from: file.filename }; + return new ExchangeFile({ data: JSON.stringify(answer), filename: "answer.json", contentType: "application/json" }); + } +} + +sw.run(Orders); diff --git a/SW.Serverless.UnitTests/NodeAdapters/bitween_receiver.js b/SW.Serverless.UnitTests/NodeAdapters/bitween_receiver.js new file mode 100644 index 0000000..f914647 --- /dev/null +++ b/SW.Serverless.UnitTests/NodeAdapters/bitween_receiver.js @@ -0,0 +1,28 @@ +// A Bitween receiver over a folder, written with @simplyworks/bitween. +const fs = require("node:fs"); +const path = require("node:path"); +const sw = require("@simplyworks/serverless"); +const { ExchangeFile, Receiver } = require("@simplyworks/bitween"); + +class Folder extends Receiver { + constructor() { + super(); + sw.expect("Folder"); + this.calls = []; + } + + get folder() { return sw.valueOf("Folder"); } + initialize() { this.calls.push("Initialize"); } + listFiles() { this.calls.push("ListFiles"); return fs.readdirSync(this.folder).sort(); } + getFile(id) { + this.calls.push("GetFile"); + return new ExchangeFile({ data: fs.readFileSync(path.join(this.folder, id), "utf8"), filename: id }); + } + deleteFile(id) { this.calls.push("DeleteFile"); fs.rmSync(path.join(this.folder, id)); } + finalize() { + this.calls.push("Finalize"); + fs.writeFileSync(path.join(this.folder, "..", "calls.txt"), this.calls.join(",")); + } +} + +sw.run(Folder); diff --git a/SW.Serverless.UnitTests/NodeAdapters/bitween_validator.ts b/SW.Serverless.UnitTests/NodeAdapters/bitween_validator.ts new file mode 100644 index 0000000..ff34fb0 --- /dev/null +++ b/SW.Serverless.UnitTests/NodeAdapters/bitween_validator.ts @@ -0,0 +1,17 @@ +// A Bitween validator in TypeScript: run as Node runs it once serverless build strips the types. +const sw = require("@simplyworks/serverless"); +const { ExchangeFile, ValidationResult, Validator } = require("@simplyworks/bitween"); + +interface Order { orderId?: string; lines?: unknown[] } + +class Orders extends Validator { + validate(file: typeof ExchangeFile.prototype): typeof ValidationResult.prototype { + const order: Order = JSON.parse(file.data); + const result = new ValidationResult(); + if (!order.orderId) result.add("orderId", "An order needs an id."); + if (!order.lines || order.lines.length === 0) result.add("lines", "An order needs at least one line."); + return result; + } +} + +sw.run(Orders); diff --git a/SW.Serverless.UnitTests/NodeAdapters/classic.js b/SW.Serverless.UnitTests/NodeAdapters/classic.js new file mode 100644 index 0000000..0edee5c --- /dev/null +++ b/SW.Serverless.UnitTests/NodeAdapters/classic.js @@ -0,0 +1,41 @@ +// A classic adapter in JavaScript, called as Bitween calls handlers: one session, one call at a time. +const sw = require("@simplyworks/serverless"); + +class Classic { + static commands = { + Greet: { method: "greet", input: "string", output: "string", description: "Greets someone" }, + Correlation: { method: "correlation", output: "string" }, + Add: { method: "add", input: { type: "object", properties: { A: { type: "integer" }, B: { type: "integer" } } }, output: { type: "integer" } }, + Fail: { method: "fail", input: "string", output: "string" }, + Crash: { method: "crash", output: "string" }, + Big: { method: "big", input: "json", output: "string" }, + Length: { method: "length", input: "string", output: "json" }, + Bytes: { method: "bytes", input: "bytes", output: "bytes" }, + Nothing: { method: "nothing" }, + Slow: { method: "slow", input: "json", output: "string" }, + }; + + constructor() { + sw.expect("Prefix", { description: "Put before every greeting" }); + sw.expect("Secret", { default: "s3cret", secret: true }); + } + + greet(name) { return sw.valueOf("Prefix") + name; } + correlation() { return sw.valueOf("CorrelationId"); } + async add({ A, B }) { return A + B; } + fail(message) { throw new sw.AdapterError(message, { type: "Acme.Rejected" }); } + crash() { return undefined.missing; } + big(size) { return "x".repeat(size); } + length(text) { return text.length; } + bytes(data) { return Buffer.from(data).reverse(); } + nothing() { sw.log.info("did nothing, as asked"); } + slow(seconds) { + const signal = sw.context().signal; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => resolve("finished"), seconds * 1000); + signal.addEventListener("abort", () => { clearTimeout(timer); reject(new Error("cancelled")); }); + }); + } +} + +sw.run(Classic); diff --git a/SW.Serverless.UnitTests/NodeAdapters/resident.js b/SW.Serverless.UnitTests/NodeAdapters/resident.js new file mode 100644 index 0000000..cc63134 --- /dev/null +++ b/SW.Serverless.UnitTests/NodeAdapters/resident.js @@ -0,0 +1,38 @@ +// A resident adapter in JavaScript: started, kept running, asked for its status, reset and stopped. +const sw = require("@simplyworks/serverless"); + +class Resident { + static commands = { + Started: { method: "isStarted", output: "json" }, + Publish: { method: "publish", input: "string", output: "string" }, + Remember: { method: "remember", input: "string", output: "string" }, + Forget: { method: "forget" }, + Resets: { method: "getResets", output: "json" }, + }; + + constructor() { + this.started = false; + this.resets = []; + sw.expect("Name", { default: "resident" }); + } + + async start() { this.started = true; } + async stop() { this.started = false; } + status() { return { connected: this.started, state: "Listening", details: { resets: this.resets.length } }; } + reset(sessionId) { this.resets.push(sessionId); } + + isStarted() { return this.started; } + publish(text) { + return sw.context().publish(text, { dedupeKey: "k-" + text, contentType: "text/plain", endpoint: "tests" }); + } + async remember(value) { + const ctx = sw.context(); + const before = await ctx.getState("memory"); + await ctx.setState("memory", value); + return before ?? ""; + } + async forget() { await sw.context().deleteState("memory"); } + getResets() { return this.resets; } +} + +sw.run(Resident); From 7b5bb55293e70b15c06218ae2f0b75184d84e433 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 16:42:45 +0300 Subject: [PATCH 07/17] Build, scaffold, test and run Node adapters with the CLI serverless init --lang node or typescript writes a working adapter of any Bitween kind; the TypeScript one is an ES module checked by tsc --noEmit. serverless build packages it: its files as they are, TypeScript turned into JavaScript by Node's own type stripping, so no compiler is needed and syntax that would need compiling is refused; the SDK and Bitween kinds vendored into node_modules from copies the CLI carries; and package.json's other dependencies installed by npm without running their scripts. A dependency with native code is refused unless the adapter is limited to the platform it is built on. test and run build a Node project folder first. --- .../CliCommandTests.cs | 71 ++++- SW.Serverless.Installer/AdapterCommands.cs | 21 +- SW.Serverless.Tooling/Building/NodeBuild.cs | 285 ++++++++++++++++++ .../Building/PackageBuilder.cs | 7 +- .../node/@simplyworks/bitween/package.json | 13 + .../node/@simplyworks/bitween/src/index.d.ts | 46 +++ .../node/@simplyworks/bitween/src/index.js | 203 +++++++++++++ .../SW.Serverless.Tooling.csproj | 6 +- .../Scaffolding/Scaffolder.cs | 174 ++++++++++- 9 files changed, 812 insertions(+), 14 deletions(-) create mode 100644 SW.Serverless.Tooling/Building/NodeBuild.cs create mode 100644 SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/package.json create mode 100644 SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/src/index.d.ts create mode 100644 SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/src/index.js diff --git a/SW.Serverless.Installer.UnitTests/CliCommandTests.cs b/SW.Serverless.Installer.UnitTests/CliCommandTests.cs index b5394b5..b6eaf1f 100644 --- a/SW.Serverless.Installer.UnitTests/CliCommandTests.cs +++ b/SW.Serverless.Installer.UnitTests/CliCommandTests.cs @@ -67,9 +67,9 @@ public async Task Init_makes_a_project_and_refuses_what_it_can_t_make() StringAssert.Contains(File.ReadAllText(Path.Combine(project, "Program.cs")), "IBitweenValidator"); Assert.AreEqual(Program.Failure, (await Cli("init", "AcmeOrders", "--dir", work)).Exit, "an existing project isn't overwritten"); - var (nodeExit, nodeOutput) = await Cli("init", "NodeOrders", "--lang", "node", "--dir", work); - Assert.AreEqual(Program.Failure, nodeExit); - StringAssert.Contains(nodeOutput, "arrives with that language's SDK"); + var (goExit, goOutput) = await Cli("init", "GoOrders", "--lang", "go", "--dir", work); + Assert.AreEqual(Program.Failure, goExit); + StringAssert.Contains(goOutput, "arrives with that language's SDK"); } [TestMethod] @@ -242,4 +242,69 @@ public void The_python_bitween_kinds_the_cli_carries_are_bitween_s() Assert.AreEqual(File.ReadAllText(original), File.ReadAllText(copy), "copy Bitween-api/sdk/python/src/simplyworks_bitween into SW.Serverless.Tooling/Contracts/bitween/python"); } + + /// + /// A JavaScript or TypeScript adapter, from init to a package that conforms: TypeScript's types + /// stripped by Node itself, the SDKs vendored into node_modules from the copies the CLI carries. + /// + [DataTestMethod] + [DataRow("node", "handler")] + [DataRow("node", "receiver")] + [DataRow("typescript", "mapper")] + [DataRow("typescript", "validator")] + public async Task What_init_writes_in_node_builds_and_conforms(string language, string kind) + { + if (language == "typescript" && !NodeStripsTypes()) + Assert.Inconclusive("this machine's node can't strip TypeScript types; it takes Node 22.13 or later"); + var work = WorkFolder(); + var name = (language == "node" ? "Js" : "Ts") + char.ToUpper(kind[0]) + kind[1..]; + Assert.AreEqual(Program.Success, (await Cli("init", name, "--lang", language, "--kind", kind, "--dir", work)).Exit); + var project = Path.Combine(work, name); + + var build = await Cli("build", project); + Assert.AreEqual(Program.Success, build.Exit, build.Output); + + var package = Path.Combine(project, "bin", "serverless", "package"); + var manifest = SW.Serverless.Contract.Catalog.AdapterManifest.Parse(File.ReadAllText(Path.Combine(package, "adapter.json"))); + Assert.AreEqual("node", manifest.Runtime); + Assert.AreEqual("main.js", manifest.Entry); + Assert.AreEqual(language == "node" ? "javascript" : "typescript", manifest.Language); + CollectionAssert.AreEqual(new[] { kind }, manifest.Kinds); + Assert.IsTrue(File.Exists(Path.Combine(package, "node_modules", "@simplyworks", "serverless", "src", "index.js"))); + Assert.IsTrue(File.Exists(Path.Combine(package, "node_modules", "@simplyworks", "bitween", "src", "index.js"))); + Assert.IsTrue(manifest.Source.Files.ContainsKey(language == "node" ? "main.js" : "main.ts")); + + var settings = Path.Combine(work, "settings.json"); + File.WriteAllText(settings, """{ "ApiKey": "k" }"""); + var test = await Cli("test", project, "--settings", settings); + Assert.AreEqual(Program.Success, test.Exit, test.Output); + } + + static bool NodeStripsTypes() + { + try + { + using var node = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo("node", + "-e \"process.exit(typeof require('node:module').stripTypeScriptTypes === 'function' ? 0 : 1)\"") + { RedirectStandardOutput = true, RedirectStandardError = true }); + node!.WaitForExit(); + return node.ExitCode == 0; + } + catch + { + return false; + } + } + + [TestMethod] + public void The_node_bitween_kinds_the_cli_carries_are_bitween_s() + { + var root = RepositoryRoot(); + var original = Path.GetFullPath(Path.Combine(root, "..", "Bitween-api", "sdk", "node")); + if (!Directory.Exists(original)) Assert.Inconclusive($"Bitween-api isn't beside this repository ({original})"); + var copy = Path.Combine(root, "SW.Serverless.Tooling", "Contracts", "bitween", "node", "@simplyworks", "bitween"); + foreach (var file in new[] { "package.json", "src/index.js", "src/index.d.ts" }) + Assert.AreEqual(File.ReadAllText(Path.Combine(original, file)), File.ReadAllText(Path.Combine(copy, file)), + $"copy Bitween-api/sdk/node/{file} into SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween"); + } } diff --git a/SW.Serverless.Installer/AdapterCommands.cs b/SW.Serverless.Installer/AdapterCommands.cs index 40a063f..fbe3028 100644 --- a/SW.Serverless.Installer/AdapterCommands.cs +++ b/SW.Serverless.Installer/AdapterCommands.cs @@ -21,7 +21,7 @@ public class InitCliOptions [Value(0, Required = true, MetaName = "name", HelpText = "The adapter's name, e.g. AcmeOrders: its folder, project and class.")] public string Name { get; set; } - [Option("lang", Default = "dotnet", HelpText = "Its language: dotnet or python. Node and Go arrive with their SDKs.")] + [Option("lang", Default = "dotnet", HelpText = "Its language: dotnet, python, node (JavaScript) or typescript. Go arrives with its SDK.")] public string Language { get; set; } [Option("kind", Default = "handler", HelpText = "handler, mapper, validator or receiver.")] @@ -285,7 +285,7 @@ public static Task Publish(PublishPackageCliOptions opts, Func TryDelete(folder)); } - if (Directory.Exists(path) && (Directory.GetFiles(path, "*.*proj").Length > 0 || IsUnbuiltPython(path))) + if (Directory.Exists(path) && (Directory.GetFiles(path, "*.*proj").Length > 0 || IsUnbuiltScript(path))) { var output = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "swsl-cli", Guid.NewGuid().ToString("N")); var built = await PackageBuilder.BuildAsync(new BuildRequest { ProjectDirectory = path, OutputDirectory = output, Log = Console.WriteLine }); @@ -300,15 +300,22 @@ public static Task Publish(PublishPackageCliOptions opts, Func { }); } - /// A Python project rather than a built package: its manifest names Python, and no build has written the entry. - static bool IsUnbuiltPython(string folder) + /// + /// A Python or Node project rather than a built package: its manifest names the runtime, and + /// nothing a build writes is there — the Python entry, or the SDK in node_modules. + /// + static bool IsUnbuiltScript(string folder) { var manifestPath = System.IO.Path.Combine(folder, AdapterManifest.FileName); - if (!File.Exists(manifestPath) || File.Exists(System.IO.Path.Combine(folder, PythonBuild.EntryScript))) return false; + if (!File.Exists(manifestPath)) return false; try { - return string.Equals(AdapterManifest.Parse(File.ReadAllText(manifestPath)).Runtime, AdapterManifest.PythonRuntime, - StringComparison.OrdinalIgnoreCase); + var runtime = AdapterManifest.Parse(File.ReadAllText(manifestPath)).Runtime; + if (string.Equals(runtime, AdapterManifest.PythonRuntime, StringComparison.OrdinalIgnoreCase)) + return !File.Exists(System.IO.Path.Combine(folder, PythonBuild.EntryScript)); + if (string.Equals(runtime, AdapterManifest.NodeRuntime, StringComparison.OrdinalIgnoreCase)) + return !File.Exists(System.IO.Path.Combine(folder, "node_modules", "@simplyworks", "serverless", "package.json")); + return false; } catch (JsonException) { diff --git a/SW.Serverless.Tooling/Building/NodeBuild.cs b/SW.Serverless.Tooling/Building/NodeBuild.cs new file mode 100644 index 0000000..d0f9c4f --- /dev/null +++ b/SW.Serverless.Tooling/Building/NodeBuild.cs @@ -0,0 +1,285 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using SW.Serverless.Contract.Catalog; + +namespace SW.Serverless.Tooling.Building +{ + /// + /// serverless build for a JavaScript or TypeScript adapter. Node runs JavaScript from its source, + /// so the package is the adapter's files as they are, TypeScript among them with its types + /// stripped by Node itself — no compiler, no packages — and a node_modules beside them: the SDK + /// and the Bitween kinds from copies this tool carries, and whatever package.json depends on, + /// through npm. + /// + /// + /// A dependency with a native addon is built for the machine npm runs on, which is not something + /// that can be done for another platform from here. It is refused unless adapter.json limits the + /// adapter to platforms this machine is one of, and the manifest then lists them. + /// + public static class NodeBuild + { + public const string DefaultEntry = "main.js"; + public const string DefaultRuntimeVersion = ">=22"; + + static readonly string[] OwnPackages = { "@simplyworks/serverless", "@simplyworks/bitween" }; + static readonly string[] TypeScript = { ".ts", ".mts", ".cts" }; + + internal static async Task BuildAsync(BuildRequest request, string project, AdapterManifest author, BuildResult result) + { + var entry = string.IsNullOrWhiteSpace(author.Entry) ? DefaultEntry : author.Entry.Replace('\\', '/'); + if (!File.Exists(Path.Combine(project, entry))) + { + result.Problems.Add($"there is no {entry} in {project}; set \"entry\" in {AdapterManifest.FileName} to the script that runs the adapter"); + return; + } + + // Source first, as for every language: a secret found here stops the build before anything is produced. + var source = PackageBuilder.CollectSource(project, null, request, result); + if (!result.Succeeded || request.DryRun) return; + + var output = Path.GetFullPath(request.OutputDirectory ?? Path.Combine(project, "bin", "serverless")); + var packageDirectory = Path.Combine(output, "package"); + if (Directory.Exists(packageDirectory)) Directory.Delete(packageDirectory, true); + Directory.CreateDirectory(packageDirectory); + + foreach (var (relative, absolute) in source) + { + var target = Path.Combine(packageDirectory, relative); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.Copy(absolute, target); + } + + var typeScript = source.Keys.Where(IsTypeScript).Where(k => !k.EndsWith(".d.ts", StringComparison.OrdinalIgnoreCase)).ToList(); + if (typeScript.Count > 0) + { + request.Log($"Stripping the types from {typeScript.Count} TypeScript file{(typeScript.Count == 1 ? "" : "s")}..."); + if (!await StripTypesAsync(request, packageDirectory, typeScript, result)) return; + if (IsTypeScript(entry)) entry = JavaScriptName(entry); + } + + var modules = Path.Combine(packageDirectory, "node_modules"); + var platforms = await InstallDependenciesAsync(request, project, author, packageDirectory, result); + if (!result.Succeeded) return; + WriteOwnPackages(modules); + + request.Log("Asking the adapter to describe itself..."); + var (description, problem) = await LocalAdapterHost.DescribeAsync(Path.Combine(packageDirectory, entry), + AdapterManifest.NodeRuntime, request.Runtimes); + if (description == null) + { + result.Problems.Add($"{problem}. A Node adapter describes itself through run() from @simplyworks/serverless; make sure {entry} calls it"); + return; + } + result.Warnings.AddRange(description.Warnings); + + var manifest = PackageBuilder.ManifestFrom(author, false, description, entry, typeScript.Count > 0 ? "typescript" : "javascript"); + manifest.Runtime = AdapterManifest.NodeRuntime; + manifest.RuntimeVersion ??= DefaultRuntimeVersion; + manifest.Platforms = platforms; + manifest.Lifecycle = description.Lifecycle == AdapterManifest.ResidentLifecycle + ? AdapterManifest.ResidentLifecycle + : AdapterManifest.ClassicLifecycle; + manifest.Protocol = new AdapterProtocolRange { Min = 2, Max = 2 }; + + if (request.IncludeSource) + { + manifest.Source = new AdapterSource + { + BuildCommand = "serverless build", + Lockfiles = source.Keys.Where(PackageBuilder.IsLockfile).OrderBy(k => k).ToList(), + }; + var sourceDirectory = Path.Combine(packageDirectory, AdapterSource.DefaultPath); + foreach (var (relative, absolute) in source.OrderBy(s => s.Key, StringComparer.Ordinal)) + { + var target = Path.Combine(sourceDirectory, relative); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.Copy(absolute, target); + manifest.Source.Files[relative] = Convert.ToHexString(SHA256.HashData(await File.ReadAllBytesAsync(absolute))).ToLowerInvariant(); + } + if (manifest.Source.Lockfiles.Count == 0) manifest.Source.Lockfiles = null; + } + + var problems = manifest.Validate(); + if (problems.Count > 0) + { + result.Problems.AddRange(problems); + return; + } + + await File.WriteAllTextAsync(Path.Combine(packageDirectory, AdapterManifest.FileName), manifest.ToJson()); + + var zip = Path.Combine(output, $"{manifest.Id}{(string.IsNullOrWhiteSpace(manifest.Version) ? "" : "-" + manifest.Version)}.zip"); + if (File.Exists(zip)) File.Delete(zip); + ZipFile.CreateFromDirectory(packageDirectory, zip, CompressionLevel.Optimal, includeBaseDirectory: false); + + result.Manifest = manifest; + result.PackageDirectory = packageDirectory; + result.ZipPath = zip; + } + + static bool IsTypeScript(string path) => TypeScript.Contains(Path.GetExtension(path), StringComparer.OrdinalIgnoreCase); + + static string JavaScriptName(string path) => Path.ChangeExtension(path, Path.GetExtension(path).ToLowerInvariant() switch + { + ".mts" => ".mjs", + ".cts" => ".cjs", + _ => ".js", + }).Replace('\\', '/'); + + /// + /// Each TypeScript file becomes JavaScript beside it, by Node's own type stripping (Node 22.13 + /// or later): what TypeScript erases is erased, and anything it would have to compile — an + /// enum, a namespace — is refused with its file and line, rather than built differently. + /// + static async Task StripTypesAsync(BuildRequest request, string packageDirectory, List files, BuildResult result) + { + const string script = """ + const fs = require("node:fs"); + const { stripTypeScriptTypes } = require("node:module"); + if (typeof stripTypeScriptTypes !== "function") { + console.error("this Node can't strip TypeScript types; use Node 22.13 or later"); + process.exit(2); + } + let failed = false; + for (const [from, to] of JSON.parse(fs.readFileSync(0, "utf8"))) { + try { + fs.writeFileSync(to, stripTypeScriptTypes(fs.readFileSync(from, "utf8"), { mode: "strip" })); + fs.rmSync(from); + } catch (e) { + console.error(`${from}: ${e.message}`); + failed = true; + } + } + process.exit(failed ? 1 : 0); + """; + var pairs = files.Select(f => new[] { Path.Combine(packageDirectory, f), Path.Combine(packageDirectory, JavaScriptName(f)) }).ToList(); + var (ok, output) = await RunAsync(request.Runtimes.NodeExecutable, packageDirectory, JsonSerializer.Serialize(pairs), + "--no-warnings", "-e", script); + if (!ok) + result.Problems.Add($"the TypeScript could not be turned into JavaScript: {output.Trim()}. " + + "Only syntax TypeScript erases is supported: no enums, namespaces or parameter properties"); + return ok; + } + + /// + /// npm installs package.json's dependencies, other than the SDKs, into the package. Returns + /// the platforms the package is limited to: none, unless a dependency has a native addon. + /// + static async Task> InstallDependenciesAsync(BuildRequest request, string project, AdapterManifest author, + string packageDirectory, BuildResult result) + { + var authored = author.Platforms is { Count: > 0 } ? author.Platforms : null; + var packageJson = Path.Combine(packageDirectory, "package.json"); + if (!File.Exists(packageJson)) return authored; + + var document = JsonNode.Parse(await File.ReadAllTextAsync(packageJson))?.AsObject(); + var dependencies = document?["dependencies"]?.AsObject(); + if (dependencies == null) return authored; + foreach (var own in OwnPackages) dependencies.Remove(own); + document.Remove("devDependencies"); + if (dependencies.Count == 0) return authored; + + // npm works on a copy, so the package's own package.json keeps what the author wrote. + var work = Path.Combine(Path.GetTempPath(), "swsl-nodebuild", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(work); + try + { + await File.WriteAllTextAsync(Path.Combine(work, "package.json"), document.ToJsonString()); + var lockfile = Path.Combine(project, "package-lock.json"); + var useLock = File.Exists(lockfile); + if (useLock) File.Copy(lockfile, Path.Combine(work, "package-lock.json")); + + request.Log("Installing the dependencies..."); + // Scripts are not run: a dependency's install script is code nobody reviewed, run on + // the build machine. An addon that needs one to build is refused below anyway. + var (ok, output) = await RunAsync("npm", work, null, + useLock ? "ci" : "install", "--omit=dev", "--ignore-scripts", "--no-audit", "--no-fund", "--loglevel=error"); + if (!ok) + { + result.Problems.Add($"npm could not install the dependencies: {string.Join(" ", output.Split('\n').Select(l => l.Trim()).Where(l => l.Length > 0).TakeLast(3))}"); + return null; + } + + var modules = Path.Combine(work, "node_modules"); + if (!Directory.Exists(modules)) return authored; + var addons = Directory.EnumerateFiles(modules, "*.node", SearchOption.AllDirectories).ToList(); + if (addons.Count > 0) + { + var here = Runtimes.AdapterRuntimes.CurrentPlatform; + if (authored == null || authored.Count != 1 || !string.Equals(authored[0], here, StringComparison.OrdinalIgnoreCase)) + { + var addon = Path.GetRelativePath(modules, addons[0]).Replace('\\', '/'); + var package = addon.StartsWith('@') ? string.Join('/', addon.Split('/').Take(2)) : addon.Split('/')[0]; + result.Problems.Add($"{package} has native code ({addon}), and which of it loads depends on the platform npm installed it on. " + + $"Build on the platform the adapter runs on, with \"platforms\": [\"\"] in {AdapterManifest.FileName}; here that is {here}"); + return null; + } + } + + CopyFolder(modules, Path.Combine(packageDirectory, "node_modules")); + return authored; + } + finally + { + try { Directory.Delete(work, true); } catch { } + } + } + + /// The SDK and the Bitween kinds, as this tool carries them, under . + public static void WriteOwnPackages(string modules) + { + var assembly = typeof(NodeBuild).Assembly; + foreach (var name in assembly.GetManifestResourceNames()) + { + var normalized = name.Replace('\\', '/'); + if (!normalized.StartsWith("node/", StringComparison.Ordinal)) continue; + var target = Path.Combine(modules, normalized["node/".Length..]); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + using var resource = assembly.GetManifestResourceStream(name)!; + using var file = File.Create(target); + resource.CopyTo(file); + } + } + + static void CopyFolder(string from, string to) + { + foreach (var file in Directory.EnumerateFiles(from, "*", SearchOption.AllDirectories)) + { + var target = Path.Combine(to, Path.GetRelativePath(from, file)); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.Copy(file, target, true); + } + } + + static async Task<(bool Ok, string Output)> RunAsync(string executable, string workingDirectory, string stdin, params string[] arguments) + { + var start = new ProcessStartInfo(executable) + { + WorkingDirectory = workingDirectory, + RedirectStandardInput = stdin != null, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + foreach (var argument in arguments) start.ArgumentList.Add(argument); + using var process = Process.Start(start)!; + if (stdin != null) + { + await process.StandardInput.WriteAsync(stdin); + process.StandardInput.Close(); + } + var stdout = process.StandardOutput.ReadToEndAsync(); + var stderr = process.StandardError.ReadToEndAsync(); + await process.WaitForExitAsync(); + return (process.ExitCode == 0, await stdout + await stderr); + } + } +} diff --git a/SW.Serverless.Tooling/Building/PackageBuilder.cs b/SW.Serverless.Tooling/Building/PackageBuilder.cs index c4d60f4..afb684b 100644 --- a/SW.Serverless.Tooling/Building/PackageBuilder.cs +++ b/SW.Serverless.Tooling/Building/PackageBuilder.cs @@ -84,9 +84,14 @@ public static async Task BuildAsync(BuildRequest request) await PythonBuild.BuildAsync(request, project, author, result); return result; } + if (string.Equals(runtime, AdapterManifest.NodeRuntime, StringComparison.OrdinalIgnoreCase)) + { + await NodeBuild.BuildAsync(request, project, author, result); + return result; + } if (!string.Equals(runtime, AdapterManifest.DotnetRuntime, StringComparison.OrdinalIgnoreCase)) { - result.Problems.Add($"building a '{runtime}' adapter arrives with that language's SDK; this build does .NET and Python"); + result.Problems.Add($"building a '{runtime}' adapter arrives with that language's SDK; this build does .NET, Python and Node"); return result; } diff --git a/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/package.json b/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/package.json new file mode 100644 index 0000000..b9a3a4a --- /dev/null +++ b/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/package.json @@ -0,0 +1,13 @@ +{ + "name": "@simplyworks/bitween", + "version": "10.0.59", + "description": "The Bitween adapter contract for JavaScript and TypeScript: handlers, mappers, validators and receivers.", + "main": "src/index.js", + "types": "src/index.d.ts", + "files": ["src"], + "engines": { "node": ">=22" }, + "peerDependencies": { "@simplyworks/serverless": ">=10.1.0" }, + "scripts": { "test": "node --test" }, + "license": "MIT", + "repository": { "type": "git", "url": "https://github.com/simplify9/Bitween-api", "directory": "sdk/node" } +} diff --git a/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/src/index.d.ts b/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/src/index.d.ts new file mode 100644 index 0000000..7adaabc --- /dev/null +++ b/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/src/index.d.ts @@ -0,0 +1,46 @@ +export declare const CONTRACT: "bitween"; +export declare const CONTRACT_VERSION: 1; + +export declare class ExchangeFile { + constructor(init?: { data?: string; filename?: string | null; badData?: boolean; contentType?: string | null }); + /** The content: text, or base64 for binary content. */ + data: string; + filename: string | null; + /** A bad response: a delivery the partner rejected. Returned, not thrown. */ + badData: boolean; + contentType: string | null; + /** SHA-1 of data as lower-case hex. */ + readonly hash: string; + toWire(): Record; + static fromWire(value: unknown): ExchangeFile; +} + +export declare class ValidationResult { + constructor(validations?: [string, string][]); + validations: [string, string][]; + readonly success: boolean; + add(key: string, message: string): this; + toWire(): Record; +} + +type Awaitable = T | Promise; + +export declare abstract class Handler { + abstract handle(file: ExchangeFile): Awaitable; +} + +export declare abstract class Mapper { + abstract map(file: ExchangeFile): Awaitable; +} + +export declare abstract class Validator { + abstract validate(file: ExchangeFile): Awaitable | null | void>; +} + +export declare abstract class Receiver { + initialize(): Awaitable; + abstract listFiles(): Awaitable; + abstract getFile(fileId: string): Awaitable; + abstract deleteFile(fileId: string): Awaitable; + finalize(): Awaitable; +} diff --git a/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/src/index.js b/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/src/index.js new file mode 100644 index 0000000..35d1e8d --- /dev/null +++ b/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/src/index.js @@ -0,0 +1,203 @@ +"use strict"; +/** + * The Bitween adapter contract for JavaScript and TypeScript: the kinds of adapter Bitween runs, and + * what it passes. Extend a kind and implement its methods; the wire names, the encoding and the kind + * and contract declarations are taken care of. The contract itself is bitween-adapter-contract.v1.json + * in SW.Bitween.Adapters; this is its JavaScript form. + * + * const sw = require("@simplyworks/serverless"); + * const { ExchangeFile, Handler } = require("@simplyworks/bitween"); + * + * class Orders extends Handler { + * constructor() { super(); sw.expect("Url", { description: "Where orders go" }); } + * handle(file) { return new ExchangeFile({ data: file.data, filename: file.filename }); } + * } + * + * sw.run(Orders); + */ + +const crypto = require("node:crypto"); + +const CONTRACT = "bitween"; +const CONTRACT_VERSION = 1; +const VERSION = "10.0.59"; + +const EXCHANGE_FILE_SCHEMA = { + title: "ExchangeFile", type: "object", required: ["Data"], additionalProperties: true, + properties: { + Filename: { type: ["string", "null"] }, Data: { type: "string" }, Hash: { type: ["string", "null"] }, + BadData: { type: "boolean", default: false }, ContentType: { type: ["string", "null"] }, + }, +}; + +const VALIDATION_RESULT_SCHEMA = { + title: "ValidationResult", type: "object", required: ["Validations"], additionalProperties: true, + properties: { + Success: { type: "boolean" }, + Validations: { + type: "array", + items: { type: "object", required: ["Key", "Value"], properties: { Key: { type: "string" }, Value: { type: "string" } } }, + }, + }, +}; + +/** + * A file as Bitween passes it to and from adapters. `data` is the content: text, or base64 for + * binary content. `badData` marks a bad response, a delivery the partner rejected — returned, not thrown. + */ +class ExchangeFile { + constructor({ data = "", filename = null, badData = false, contentType = null } = {}) { + this.data = data; + this.filename = filename; + this.badData = badData; + this.contentType = contentType; + } + + /** SHA-1 of `data` as lower-case hex, as .NET adapters write it. */ + get hash() { + return crypto.createHash("sha1").update(this.data ?? "", "utf8").digest("hex"); + } + + toWire() { + return { Filename: this.filename, Data: this.data ?? "", Hash: this.hash, BadData: this.badData, ContentType: this.contentType }; + } + + static fromWire(value) { + if (!value || typeof value !== "object") throw new TypeError("an ExchangeFile is a JSON object"); + // Hash is recomputed, never trusted; unknown properties are ignored. + return new ExchangeFile({ + data: value.Data ?? "", filename: value.Filename ?? null, badData: Boolean(value.BadData ?? false), + contentType: value.ContentType ?? null, + }); + } +} + +/** What a validator found: each failure as a key — often the field it concerns — and a message. */ +class ValidationResult { + constructor(validations = []) { + this.validations = validations; + } + + get success() { + return this.validations.length === 0; + } + + add(key, message) { + this.validations.push([key, message]); + return this; + } + + toWire() { + return { Success: this.success, Validations: this.validations.map(([Key, Value]) => ({ Key, Value })) }; + } + + static fromWire(value) { + return new ValidationResult(((value && value.Validations) || []).map((v) => [v.Key ?? "", v.Value ?? ""])); + } +} + +function asFile(value) { + if (value instanceof ExchangeFile) return value; + if (typeof value === "string") return new ExchangeFile({ data: value }); + if (value && typeof value === "object" && "data" in value) return new ExchangeFile(value); + throw new TypeError(`expected an ExchangeFile, got ${value === null ? "null" : typeof value}`); +} + +const ABSTRACT = Symbol("abstract"); + +function abstract(name) { + const fn = function () { throw new Error(`${name} is not implemented`); }; + fn[ABSTRACT] = true; + return fn; +} + +class Kind { + static contracts = { [CONTRACT]: CONTRACT_VERSION }; + static required = []; + + /** A declared kind without its methods fails when the adapter starts, not on first use. */ + __swCheck() { + const missing = this.constructor.required.filter((m) => this[m] && this[m][ABSTRACT]); + if (missing.length) + throw new TypeError(`${this.constructor.name} is a Bitween ${this.constructor.kinds[0]} but does not implement ${missing.join(", ")}`); + } +} + +const file = EXCHANGE_FILE_SCHEMA; + +/** Delivers a message and returns the partner's response. A rejection is returned with badData, not thrown. */ +class Handler extends Kind { + static kinds = ["handler"]; + static required = ["handle"]; + static commands = { + Handle: { method: "__swHandle", input: file, output: file, description: "Delivers a message and returns the partner's response." }, + }; + + async __swHandle(value) { + return asFile(await this.handle(ExchangeFile.fromWire(value))); + } +} +Handler.prototype.handle = abstract("handle"); + +/** Transforms a message into the shape the next step expects. */ +class Mapper extends Kind { + static kinds = ["mapper"]; + static required = ["map"]; + static commands = { + Handle: { method: "__swHandle", input: file, output: file, description: "Transforms a message into the shape the next step expects." }, + }; + + async __swHandle(value) { + return asFile(await this.map(ExchangeFile.fromWire(value))); + } +} +Mapper.prototype.map = abstract("map"); + +/** Checks a message before it is accepted. */ +class Validator extends Kind { + static kinds = ["validator"]; + static required = ["validate"]; + static commands = { + Validate: { method: "__swValidate", input: file, output: VALIDATION_RESULT_SCHEMA, description: "Checks a message before it is accepted." }, + }; + + async __swValidate(value) { + const result = await this.validate(ExchangeFile.fromWire(value)); + if (result === undefined || result === null) return new ValidationResult(); + if (result instanceof ValidationResult) return result; + // An array of [key, message] pairs, or an object of key -> message, reads naturally too. + return new ValidationResult(Array.isArray(result) ? result : Object.entries(result)); + } +} +Validator.prototype.validate = abstract("validate"); + +/** + * Fetches files from a source on a schedule. One session per run: initialize, listFiles, then for + * each file getFile and — once it is safely taken in — deleteFile, and finally finalize, which is + * also called after a failure. + */ +class Receiver extends Kind { + static kinds = ["receiver"]; + static required = ["listFiles", "getFile", "deleteFile"]; + static commands = { + Initialize: { method: "__swInitialize", description: "Starts a run." }, + ListFiles: { method: "__swListFiles", output: { type: "array", items: { type: "string" } }, description: "The ids of the files waiting." }, + GetFile: { method: "__swGetFile", input: "string", output: file, description: "One file, by an id ListFiles gave." }, + DeleteFile: { method: "__swDeleteFile", input: "string", description: "Removes a file from the source once it is safely taken in." }, + Finalize: { method: "__swFinalize", description: "Ends a run, after a failure too." }, + }; + + initialize() {} + finalize() {} + + async __swInitialize() { await this.initialize(); } + async __swListFiles() { return ((await this.listFiles()) || []).map(String); } + async __swGetFile(fileId) { return asFile(await this.getFile(fileId)); } + async __swDeleteFile(fileId) { await this.deleteFile(fileId); } + async __swFinalize() { await this.finalize(); } +} +Receiver.prototype.listFiles = abstract("listFiles"); +Receiver.prototype.getFile = abstract("getFile"); +Receiver.prototype.deleteFile = abstract("deleteFile"); + +module.exports = { CONTRACT, CONTRACT_VERSION, VERSION, ExchangeFile, Handler, Mapper, Receiver, ValidationResult, Validator }; diff --git a/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj b/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj index 2097f4c..87e368d 100644 --- a/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj +++ b/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj @@ -35,7 +35,7 @@ - + @@ -44,6 +44,10 @@ a build needs neither PyPI nor a network. Named by their path in the package. --> + + + + diff --git a/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs b/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs index 025842a..8d2ea7b 100644 --- a/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs +++ b/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs @@ -47,11 +47,14 @@ public static class Scaffolder public const string BitweenAdaptersPackageVersion = "10.0.59"; public static readonly IReadOnlyList Kinds = new[] { "handler", "mapper", "validator", "receiver" }; - public static readonly IReadOnlyList Languages = new[] { "dotnet", "python" }; + public static readonly IReadOnlyList Languages = new[] { "dotnet", "python", "node", "typescript" }; /// The Python SDK the templates name; serverless build vendors the copy it carries. public const string PythonSdkVersion = "10.1.0"; + /// The Node SDK the templates name; serverless build vendors the copy it carries. + public const string NodeSdkVersion = "10.1.0"; + public static ScaffoldResult Scaffold(ScaffoldRequest request) { var result = new ScaffoldResult(); @@ -74,7 +77,13 @@ public static ScaffoldResult Scaffold(ScaffoldRequest request) Directory.CreateDirectory(directory); result.ProjectDirectory = directory; - var files = request.Language == "python" ? PythonFiles(name, id, request.Kind) : DotnetFiles(name, id, request.Kind); + var files = request.Language switch + { + "python" => PythonFiles(name, id, request.Kind), + "node" => NodeFiles(name, id, request.Kind, typeScript: false), + "typescript" => NodeFiles(name, id, request.Kind, typeScript: true), + _ => DotnetFiles(name, id, request.Kind), + }; foreach (var (file, content) in files) { var path = Path.Combine(directory, file); @@ -397,6 +406,167 @@ def handle(self, file: ExchangeFile) -> ExchangeFile: """", }; + static IEnumerable<(string File, string Content)> NodeFiles(string name, string id, string kind, bool typeScript) + { + var entry = typeScript ? "main.ts" : "main.js"; + yield return ("adapter.json", $$""" + { + "id": "{{id}}", + "version": "0.1.0", + "displayName": "{{Spaced(name)}}", + "summary": "What this {{kind}} does, in one sentence, for the adapter list.", + "runtime": "node", + "entry": "{{entry}}" + } + """); + + yield return (entry, NodeMain(name, kind, typeScript)); + + // The SDKs are named for the editor and for running outside a build; serverless build + // vendors the copies it carries. Other dependencies go here too, and are installed into + // the package. + var package = new System.Text.Json.Nodes.JsonObject { ["name"] = id, ["private"] = true }; + if (typeScript) package["type"] = "module"; + package["engines"] = new System.Text.Json.Nodes.JsonObject { ["node"] = ">=22" }; + package["dependencies"] = new System.Text.Json.Nodes.JsonObject + { + ["@simplyworks/serverless"] = NodeSdkVersion, + ["@simplyworks/bitween"] = ">=" + BitweenAdaptersPackageVersion, + }; + yield return ("package.json", package.ToJsonString(new System.Text.Json.JsonSerializerOptions { WriteIndented = true })); + + if (typeScript) + yield return ("tsconfig.json", """ + { + // For your editor and tsc --noEmit. serverless build doesn't compile: Node strips the + // types, so only syntax that strips cleanly is allowed (no enums or namespaces). + "compilerOptions": { + "target": "es2023", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "erasableSyntaxOnly": true, + "verbatimModuleSyntax": true, + "allowImportingTsExtensions": true + } + } + """); + + yield return ("settings.example.json", """ + { + "BaseUrl": "https://partner.example.test", + "ApiKey": "put a test key here, and keep this file out of version control once it holds one" + } + """); + + yield return (".gitignore", """ + node_modules/ + bin/ + settings.json + """); + + yield return ("README.md", $$""" + # {{Spaced(name)}} + + A Bitween {{kind}} adapter in {{(typeScript ? "TypeScript" : "JavaScript")}}, for Node 22 or later. + + ```sh + serverless build # builds bin/serverless/{{id}}-0.1.0.zip + cp settings.example.json settings.json # then fill in real values + serverless test --settings settings.json # checks it against the Bitween contract + serverless publish bin/serverless/{{id}}-0.1.0.zip # with your storage flags + ``` + + Settings are declared in code with `expect`; `serverless build` writes them into the + manifest Bitween reads.{{(typeScript ? " The build strips the types with Node itself, so no compiler is needed; `tsc --noEmit` checks them." : "")}} + """); + } + + static string NodeMain(string name, string kind, bool ts) + { + var header = ts + ? $$""" + import { expect, run, valueOf } from "@simplyworks/serverless"; + import { ExchangeFile, {{Pascal(kind)}}{{(kind == "validator" ? ", ValidationResult" : "")}} } from "@simplyworks/bitween"; + """ + : $$""" + const { expect, run, valueOf } = require("@simplyworks/serverless"); + const { ExchangeFile, {{Pascal(kind)}}{{(kind == "validator" ? ", ValidationResult" : "")}} } = require("@simplyworks/bitween"); + """; + string T(string type) => ts ? type : ""; + var body = kind switch + { + "receiver" => $$""" + /** Fetches files on a schedule. Bitween calls initialize, listFiles, then getFile and deleteFile for each file, then finalize. */ + class {{name}} extends Receiver { + constructor() { + super(); + // Declare settings here; read them in the methods with valueOf. + expect("BaseUrl", { default: "https://partner.example.test", description: "Where files are fetched from." }); + expect("ApiKey", { secret: true, description: "The partner's key." }); + } + + listFiles(){{T(": string[]")}} { + return ["example-1"]; + } + + getFile(fileId{{T(": string")}}){{T(": ExchangeFile")}} { + return new ExchangeFile({ data: JSON.stringify({ id: fileId }), filename: `${fileId}.json` }); + } + + deleteFile(fileId{{T(": string")}}){{T(": void")}} {} + } + """, + "validator" => $$""" + /** Checks a message before Bitween accepts it. */ + class {{name}} extends Validator { + constructor() { + super(); + expect("MaxBytes", { default: "1000000", type: "number", description: "The largest message accepted." }); + } + + validate(file{{T(": ExchangeFile")}}){{T(": ValidationResult")}} { + const result = new ValidationResult(); + if (file.data.length > Number(valueOf("MaxBytes"))) result.add("Data", "The message is larger than allowed."); + return result; + } + } + """, + "mapper" => $$""" + /** Maps a message into the shape the next step expects. */ + class {{name}} extends Mapper { + map(file{{T(": ExchangeFile")}}){{T(": ExchangeFile")}} { + // Return the message in its new shape. + return new ExchangeFile({ data: file.data, filename: file.filename }); + } + } + """, + _ => $$""" + /** Delivers a message and returns the partner's response. */ + class {{name}} extends Handler { + constructor() { + super(); + // Declare settings here; read them in the methods with valueOf. + expect("BaseUrl", { default: "https://partner.example.test", description: "Where messages go." }); + expect("ApiKey", { secret: true, description: "The partner's key." }); + } + + handle(file{{T(": ExchangeFile")}}){{T(": ExchangeFile")}} { + // Send file.data to valueOf("BaseUrl"). A rejection is returned with badData: true, not thrown. + return new ExchangeFile({ data: file.data, filename: file.filename }); + } + } + """, + }; + // valueOf is used by every template but the mapper; keep the import list honest there. + if (kind == "mapper") header = header.Replace("expect, run, valueOf", "run"); + else if (kind == "receiver") header = header.Replace("expect, run, valueOf", "expect, run"); + return header + "\n\n" + body + "\n\nrun(" + name + ");\n"; + } + + static string Pascal(string kind) => char.ToUpperInvariant(kind[0]) + kind[1..]; + static string Spaced(string name) => Regex.Replace(name, "(?<=[a-z0-9])(?=[A-Z])", " "); } } From 8db67a90a0d575f37a46f190702162e5a36ca165 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 17:11:52 +0300 Subject: [PATCH 08/17] Publish and promote under the adapters folder a deployment uses AdapterRepository and the package publish took the adapters folder to be "adapters" always, while hosts are configured with their own. Both now take it, defaulting to "adapters", so Bitween can publish where its hosts read; the static members stay as they were. --- .../CatalogPublishingTests.cs | 51 +++++++++++++++++++ SW.Serverless.Tooling/AdapterRepository.cs | 40 ++++++++++----- SW.Serverless.Tooling/PackagePublisher.cs | 5 +- 3 files changed, 81 insertions(+), 15 deletions(-) diff --git a/SW.Serverless.Installer.UnitTests/CatalogPublishingTests.cs b/SW.Serverless.Installer.UnitTests/CatalogPublishingTests.cs index 35bd4b6..de66e39 100644 --- a/SW.Serverless.Installer.UnitTests/CatalogPublishingTests.cs +++ b/SW.Serverless.Installer.UnitTests/CatalogPublishingTests.cs @@ -716,4 +716,55 @@ public void Published_by_falls_back_from_flag_to_environment_to_user() Assert.AreEqual("actor", UploadOptionsResolver.ResolvePublishedBy(null, n => n == "GITHUB_ACTOR" ? "actor" : null)); Assert.AreEqual(Environment.UserName, UploadOptionsResolver.ResolvePublishedBy(null, _ => null)); } + + /// + /// A deployment whose hosts read adapters from another folder: publishing, locating and promoting + /// all happen there, and nothing lands under the default one. + /// + [TestMethod] + public async Task A_repository_and_a_publish_work_under_the_folder_they_are_given() + { + var project = Path.Combine(root, "pyproject"); + Directory.CreateDirectory(project); + File.WriteAllText(Path.Combine(project, "adapter.json"), """{ "id": "acme.py", "version": "1.0.0", "runtime": "python", "entry": "main.py" }"""); + File.WriteAllText(Path.Combine(project, "main.py"), """ + import simplyworks_serverless as sw + + class Echo: + @sw.command("Echo") + def echo(self, text: str) -> str: + return text + + sw.run(Echo) + """); + var built = await Tooling.Building.PackageBuilder.BuildAsync(new Tooling.Building.BuildRequest + { + ProjectDirectory = project, + OutputDirectory = Path.Combine(root, "pybuilt"), + }); + Assert.IsTrue(built.Succeeded, string.Join("; ", built.Problems)); + + var published = await PackagePublisher.PublishPackageAsync(store, new PublishPackageRequest + { + PackagePath = built.ZipPath, + Promote = false, + RemotePath = "custom", + }, output.Add); + Assert.AreEqual("1.0.0", published.Version); + + var keys = await Keys(); + CollectionAssert.Contains(keys, "custom-versions/acme.py/1.0.0"); + CollectionAssert.Contains(keys, "custom-catalog/acme.py.json"); + Assert.IsFalse(keys.Any(k => k.StartsWith("adapters")), string.Join(", ", keys)); + + var repository = new AdapterRepository(store, output.Add, "custom"); + Assert.AreEqual("custom", repository.RemotePath); + Assert.IsNull((await repository.LoadEntryAsync("acme.py")).Current, "published without being made current"); + CollectionAssert.AreEqual(new[] { "1.0.0" }, (await repository.LocateVersionsAsync("acme.py")).Keys.ToArray()); + + await repository.PromoteAsync("acme.py", "1.0.0", Path.Combine(root, "promote")); + Assert.AreEqual("1.0.0", (await repository.LoadEntryAsync("acme.py")).Current); + // A Python adapter is never put where hosts before manifests would run it with dotnet. + Assert.IsFalse((await Keys()).Contains("custom/acme.py")); + } } diff --git a/SW.Serverless.Tooling/AdapterRepository.cs b/SW.Serverless.Tooling/AdapterRepository.cs index c4cef5d..39fc574 100644 --- a/SW.Serverless.Tooling/AdapterRepository.cs +++ b/SW.Serverless.Tooling/AdapterRepository.cs @@ -71,14 +71,23 @@ public class AdapterRepository readonly ICloudFilesService files; readonly AdapterCatalogStore catalog; readonly Action log; + readonly string root; - public AdapterRepository(ICloudFilesService files, Action log = null) + /// + /// The adapters' folder in storage, as hosts are configured with it (AdapterRemotePath); + /// unless a deployment changed it. + /// + public AdapterRepository(ICloudFilesService files, Action log = null, string root = Root) { this.files = files ?? throw new ArgumentNullException(nameof(files)); - catalog = new AdapterCatalogStore(files, Root); + this.root = string.IsNullOrWhiteSpace(root) ? Root : root.Trim().TrimEnd('/'); + catalog = new AdapterCatalogStore(files, this.root); this.log = log ?? Console.WriteLine; } + /// The folder this repository publishes under. + public string RemotePath => root; + public AdapterCatalogStore Catalog => catalog; // ---------------------------------------------------------------- metadata @@ -112,6 +121,9 @@ public static Dictionary LegacyMetadata( public static string CurrentKey(string adapterId) => AdapterCatalogPaths.Current(Root, adapterId); + string CurrentKeyOf(string adapterId) => AdapterCatalogPaths.Current(root, adapterId); + string VersionKeyOf(string adapterId, string version) => AdapterCatalogPaths.Version(root, adapterId, version); + /// Where this installer writes a version: adapters-versions/{id}/{version}. public static string VersionKey(string adapterId, string version) => AdapterCatalogPaths.Version(Root, adapterId, version); @@ -124,18 +136,18 @@ public async Task> LocateVersionsAsync(string adapter { var located = new Dictionary(StringComparer.OrdinalIgnoreCase); - var root = AdapterCatalogPaths.VersionsRoot(Root, adapterId).TrimEnd('/'); - foreach (var version in InstallerLogic.ExistingVersions(root, - (await files.ListAsync($"{root}/")).Select(f => f.Key))) - if (AdapterCatalogPaths.IsVersion(version)) located[version] = VersionKey(adapterId, version); + var versionsRoot = AdapterCatalogPaths.VersionsRoot(root, adapterId).TrimEnd('/'); + foreach (var version in InstallerLogic.ExistingVersions(versionsRoot, + (await files.ListAsync($"{versionsRoot}/")).Select(f => f.Key))) + if (AdapterCatalogPaths.IsVersion(version)) located[version] = VersionKeyOf(adapterId, version); // Listed WITH the slash, so "foo" does not pick up "foobar"'s versions, nor the current // package adapters/foo itself. - var legacy = CurrentKey(adapterId); + var legacy = CurrentKeyOf(adapterId); foreach (var version in InstallerLogic.ExistingVersions(legacy, (await files.ListAsync($"{legacy}/")).Select(f => f.Key))) if (AdapterCatalogPaths.IsVersion(version)) - located.TryAdd(version, AdapterCatalogPaths.LegacyVersion(Root, adapterId, version)); + located.TryAdd(version, AdapterCatalogPaths.LegacyVersion(root, adapterId, version)); return located; } @@ -205,7 +217,7 @@ public async Task LoadEntryAsync(string adapterId) public async Task PublishVersionAsync(string adapterId, string version, string zipPath, PackageInfo package, bool promote, string publishedBy) { - var versionKey = VersionKey(adapterId, version); + var versionKey = VersionKeyOf(adapterId, version); // Immutable: Semver already refuses an explicit version that exists, and this catches // one published between resolving the number and uploading it. @@ -215,7 +227,7 @@ public async Task PublishVersionAsync(string adapterId, string version, string z // An id keeps its runtime: hosts that predate manifests run adapters/{id} with dotnet, // and would go on running the old .NET package under an id that had moved on. var runsOnDotnet = RunsOnDotnet(package.Manifest); - if (!runsOnDotnet && await ExistsAsync(CurrentKey(adapterId))) + if (!runsOnDotnet && await ExistsAsync(CurrentKeyOf(adapterId))) throw new SWException( $"'{adapterId}' is a .NET adapter, and older hosts would keep running that under its id. " + $"Publish the {package.Manifest!.Runtime} adapter under a new id."); @@ -229,7 +241,7 @@ public async Task PublishVersionAsync(string adapterId, string version, string z // adapters/{id} is what hosts before manifests list and run, always with dotnet. An // adapter in another runtime never goes there; newer hosts find its current version in // the catalog. - if (promote && runsOnDotnet) await UploadAsync(CurrentKey(adapterId), zipPath, metadata); + if (promote && runsOnDotnet) await UploadAsync(CurrentKeyOf(adapterId), zipPath, metadata); entry.Versions.Add(new AdapterVersionRecord { @@ -265,7 +277,7 @@ public async Task PublishUnversionedAsync(string adapterId, string zipPath, Pack var entry = await LoadEntryAsync(adapterId); var sha256 = InstallerLogic.Sha256Of(zipPath); - await UploadAsync(CurrentKey(adapterId), zipPath, + await UploadAsync(CurrentKeyOf(adapterId), zipPath, LegacyMetadata(package.EntryAssembly, package.Lifecycle, package.Kind, sha256, null)); MakeCurrent(entry, null, package.Manifest, sha256, package.IconDataUri); @@ -357,7 +369,7 @@ public async Task PromoteAsync(string adapterId, string version, string workDire foreach (var item in LegacyMetadata(entryAssembly, lifecycle, kind, sha256, version)) metadata[item.Key] = item.Value; - await UploadAsync(CurrentKey(adapterId), zipPath, metadata); + await UploadAsync(CurrentKeyOf(adapterId), zipPath, metadata); record.Sha256 ??= sha256; record.Manifest ??= manifest; @@ -427,7 +439,7 @@ public async Task ListVersionsAsync(string adapterId) } var listing = new VersionListing { FromCatalog = false }; - var current = await ExistsAsync(CurrentKey(adapterId)) ? await MetadataAsync(CurrentKey(adapterId)) : null; + var current = await ExistsAsync(CurrentKeyOf(adapterId)) ? await MetadataAsync(CurrentKeyOf(adapterId)) : null; var currentVersion = Value(current, "Version"); var currentSha = Value(current, "Sha256"); diff --git a/SW.Serverless.Tooling/PackagePublisher.cs b/SW.Serverless.Tooling/PackagePublisher.cs index d5b2b45..d89f1c0 100644 --- a/SW.Serverless.Tooling/PackagePublisher.cs +++ b/SW.Serverless.Tooling/PackagePublisher.cs @@ -43,6 +43,9 @@ public class PublishPackageRequest public bool Promote { get; set; } = true; public string ReleaseNotes { get; set; } public string PublishedBy { get; set; } + + /// The adapters' folder in storage, as hosts are configured with it; "adapters" unless set. + public string RemotePath { get; set; } = AdapterRepository.Root; } public class PublishResult @@ -190,7 +193,7 @@ public static async Task PublishPackageAsync(ICloudFilesService f if (string.IsNullOrWhiteSpace(mode)) throw new SWException("The package has no version: give one with -v, or set version in adapter.json."); - var repository = new AdapterRepository(files, log); + var repository = new AdapterRepository(files, log, request.RemotePath); var version = await repository.ResolveVersionAsync(adapterId, mode); manifest.Version = version; manifest.PublishedOn = DateTimeOffset.UtcNow; From 6a8f973a169944cd33df5d33470d0418693cfa66 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 18:09:15 +0300 Subject: [PATCH 09/17] Take Bitween out of SW-Serverless; the CLI becomes sw-serverless MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SW-Serverless knew one application by name: its CLI scaffolded Bitween's kinds, vendored Bitween's Python and Node packages into every build and carried Bitween's contract in its conformance kit, and its manifest had a minBitweenVersion. None of it belongs to a library any application can use. - The CLI is sw-serverless. init writes a generic adapter — two settings, two commands — in .NET, Python, JavaScript or TypeScript. - The kit carries no contract. One is handed to it with --contract or ConformanceOptions.Contracts, or registered once by an application's own tool (ContractDocument.Register, FromJson). - The build vendors only the SDK; an application hands it its own packages (BuildRequest.Packages), which requirements.txt and package.json may then name without being fetched. - The scaffolder takes an application's templates (Scaffold with Templates) and keeps its checks and layout. - compatibility.applications gives each application's minimum version; manifests written with minBitweenVersion stay readable through MinVersionOf("bitween"). - The Python and Node SDKs are sw-serverless (import sw_serverless) and @simplyworks/sw-serverless, 10.2.0. Python gains sw.implements to declare a contract and its kinds. - Tests use a sample "orders" contract of their own, and nothing reads ../Bitween-api. The release is 10.2.0: public members were removed. --- .github/workflows/nuget-publish.yml | 2 +- .github/workflows/pr.yml | 2 +- .../SW.Serverless.Compat.ManifestV1002.csproj | 2 +- .../ListingAndManifestTests.cs | 16 +- .../MultiLanguageManifestTests.cs | 8 +- .../PackageLayoutTests.cs | 2 +- .../SW.Serverless.CompatibilityTests.csproj | 2 +- .../Catalog/AdapterManifest.cs | 32 +- SW.Serverless.Contract/Protos/adapter.proto | 12 +- .../BuildTests.cs | 9 +- .../CatalogPublishingTests.cs | 8 +- .../CliCommandTests.cs | 183 ++--- .../ConformanceTests.cs | 85 ++- .../Contracts/order.schema.json | 11 + .../Contracts/orders-adapter-contract.v1.json | 42 ++ .../Contracts/receipt.schema.json | 11 + .../SW.Serverless.Installer.UnitTests.csproj | 9 +- SW.Serverless.Installer/AdapterCommands.cs | 26 +- SW.Serverless.Installer/Options.cs | 8 +- SW.Serverless.Installer/Program.cs | 2 +- .../SW.Serverless.Installer.csproj | 5 +- .../Telemetry/DashboardEventSink.cs | 2 +- .../Telemetry/DedupeWindow.cs | 2 +- SW.Serverless.Samples.Classic/Handler.cs | 8 +- .../ConsoleEventSink.cs | 6 +- .../Publisher/PublisherHandler.cs | 2 +- SW.Serverless.Sdk/AdapterContractAttribute.cs | 2 +- SW.Serverless.Tooling/AdapterRepository.cs | 12 +- SW.Serverless.Tooling/Building/NodeBuild.cs | 18 +- .../Building/PackageBuilder.cs | 31 +- SW.Serverless.Tooling/Building/PythonBuild.cs | 44 +- SW.Serverless.Tooling/CloudFilesFactory.cs | 2 +- .../Conformance/ConformanceRunner.cs | 6 +- .../Conformance/ContractDocument.cs | 45 +- .../bitween/bitween-adapter-contract.v1.json | 121 --- .../bitween/exchange-file.schema.json | 16 - .../node/@simplyworks/bitween/package.json | 13 - .../node/@simplyworks/bitween/src/index.d.ts | 46 -- .../node/@simplyworks/bitween/src/index.js | 203 ----- .../python/simplyworks_bitween/__init__.py | 236 ------ .../bitween/validation-result.schema.json | 24 - SW.Serverless.Tooling/LocalAdapterHost.cs | 2 +- SW.Serverless.Tooling/PackagePublisher.cs | 6 +- .../SW.Serverless.Tooling.csproj | 26 +- .../Scaffolding/Scaffolder.cs | 711 ++++++------------ .../Program.cs | 39 - .../Program.cs | 47 -- .../Program.cs | 4 +- .../Program.cs | 37 + ...erverless.UnitTests.OrdersProcessor.csproj | 0 .../Program.cs | 47 ++ ...W.Serverless.UnitTests.OrdersSource.csproj | 0 SW.Serverless.UnitTests/DescribeTests.cs | 4 +- SW.Serverless.UnitTests/GrpcClassicTests.cs | 10 +- SW.Serverless.UnitTests/NodeAdapterTests.cs | 99 +-- .../NodeAdapters/bitween_handler.js | 19 - .../NodeAdapters/bitween_receiver.js | 28 - .../NodeAdapters/bitween_validator.ts | 17 - .../NodeAdapters/classic.js | 4 +- .../NodeAdapters/orders_processor.js | 21 + .../NodeAdapters/orders_processor.ts | 23 + .../NodeAdapters/orders_source.js | 36 + .../NodeAdapters/resident.js | 2 +- SW.Serverless.UnitTests/PythonAdapterTests.cs | 100 +-- .../PythonAdapters/bitween_handler.py | 21 - .../PythonAdapters/bitween_receiver.py | 40 - .../PythonAdapters/bitween_validator.py | 20 - .../PythonAdapters/classic.py | 4 +- .../PythonAdapters/orders_processor.py | 18 + .../PythonAdapters/orders_source.py | 45 ++ .../PythonAdapters/resident.py | 2 +- SW.Serverless.UnitTests/ResourceLimitTests.cs | 2 +- SW.Serverless.sln | 4 +- SW.Serverless/Resident/IAdapterEventSink.cs | 2 +- SW.Serverless/Resident/IAdapterStateStore.cs | 2 +- .../Resident/IResidentAdapterHost.cs | 2 +- SW.Serverless/Resident/ResidentAdapterHost.cs | 2 +- .../Resident/ResidentAdapterInstance.cs | 4 +- sdk/node/README.md | 4 +- sdk/node/package.json | 4 +- sdk/node/src/index.js | 8 +- sdk/node/test/sdk.test.js | 4 +- sdk/python/README.md | 4 +- sdk/python/pyproject.toml | 4 +- .../__init__.py | 6 +- .../_adapter.py | 23 +- .../_hpack.py | 0 .../_http2.py | 2 +- .../_huffman.py | 0 .../_runner.py | 4 +- .../_types.py | 2 +- .../_wire.py | 0 sdk/python/tests/test_adapter.py | 20 +- sdk/python/tests/test_wire.py | 4 +- 94 files changed, 1018 insertions(+), 1837 deletions(-) create mode 100644 SW.Serverless.Installer.UnitTests/Contracts/order.schema.json create mode 100644 SW.Serverless.Installer.UnitTests/Contracts/orders-adapter-contract.v1.json create mode 100644 SW.Serverless.Installer.UnitTests/Contracts/receipt.schema.json delete mode 100644 SW.Serverless.Tooling/Contracts/bitween/bitween-adapter-contract.v1.json delete mode 100644 SW.Serverless.Tooling/Contracts/bitween/exchange-file.schema.json delete mode 100644 SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/package.json delete mode 100644 SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/src/index.d.ts delete mode 100644 SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/src/index.js delete mode 100644 SW.Serverless.Tooling/Contracts/bitween/python/simplyworks_bitween/__init__.py delete mode 100644 SW.Serverless.Tooling/Contracts/bitween/validation-result.schema.json delete mode 100644 SW.Serverless.UnitTests.BitweenHandler/Program.cs delete mode 100644 SW.Serverless.UnitTests.BitweenReceiver/Program.cs create mode 100644 SW.Serverless.UnitTests.OrdersProcessor/Program.cs rename SW.Serverless.UnitTests.BitweenHandler/SW.Serverless.UnitTests.BitweenHandler.csproj => SW.Serverless.UnitTests.OrdersProcessor/SW.Serverless.UnitTests.OrdersProcessor.csproj (100%) create mode 100644 SW.Serverless.UnitTests.OrdersSource/Program.cs rename SW.Serverless.UnitTests.BitweenReceiver/SW.Serverless.UnitTests.BitweenReceiver.csproj => SW.Serverless.UnitTests.OrdersSource/SW.Serverless.UnitTests.OrdersSource.csproj (100%) delete mode 100644 SW.Serverless.UnitTests/NodeAdapters/bitween_handler.js delete mode 100644 SW.Serverless.UnitTests/NodeAdapters/bitween_receiver.js delete mode 100644 SW.Serverless.UnitTests/NodeAdapters/bitween_validator.ts create mode 100644 SW.Serverless.UnitTests/NodeAdapters/orders_processor.js create mode 100644 SW.Serverless.UnitTests/NodeAdapters/orders_processor.ts create mode 100644 SW.Serverless.UnitTests/NodeAdapters/orders_source.js delete mode 100644 SW.Serverless.UnitTests/PythonAdapters/bitween_handler.py delete mode 100644 SW.Serverless.UnitTests/PythonAdapters/bitween_receiver.py delete mode 100644 SW.Serverless.UnitTests/PythonAdapters/bitween_validator.py create mode 100644 SW.Serverless.UnitTests/PythonAdapters/orders_processor.py create mode 100644 SW.Serverless.UnitTests/PythonAdapters/orders_source.py rename sdk/python/src/{simplyworks_serverless => sw_serverless}/__init__.py (80%) rename sdk/python/src/{simplyworks_serverless => sw_serverless}/_adapter.py (87%) rename sdk/python/src/{simplyworks_serverless => sw_serverless}/_hpack.py (100%) rename sdk/python/src/{simplyworks_serverless => sw_serverless}/_http2.py (99%) rename sdk/python/src/{simplyworks_serverless => sw_serverless}/_huffman.py (100%) rename sdk/python/src/{simplyworks_serverless => sw_serverless}/_runner.py (99%) rename sdk/python/src/{simplyworks_serverless => sw_serverless}/_types.py (98%) rename sdk/python/src/{simplyworks_serverless => sw_serverless}/_wire.py (100%) diff --git a/.github/workflows/nuget-publish.yml b/.github/workflows/nuget-publish.yml index 5e3b412..8b3b5b1 100644 --- a/.github/workflows/nuget-publish.yml +++ b/.github/workflows/nuget-publish.yml @@ -26,7 +26,7 @@ jobs: SW.Serverless.Sdk/SW.Serverless.Sdk.csproj SW.Serverless.Tooling/SW.Serverless.Tooling.csproj major-version: '10' - minor-version: '1' + minor-version: '2' dotnet-version: '10.0.x' run-tests: true test-projects: '**/*UnitTests/*.csproj' diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7f4ee7c..0f479ae 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -40,7 +40,7 @@ jobs: - name: Test installer run: dotnet test SW.Serverless.Installer.UnitTests/SW.Serverless.Installer.UnitTests.csproj -c Release --no-build - # Hosts on the published SimplyWorks.Serverless 10.0.x, older Bitween listings and adapters on + # Hosts on the published SimplyWorks.Serverless 10.0.x, older storage listings and adapters on # the published SDK, against what this branch's installer and host write and read. Outside the # *UnitTests glob the publish workflow uses, so it is run here explicitly. - name: Test backward compatibility diff --git a/SW.Serverless.Compat.ManifestV1002/SW.Serverless.Compat.ManifestV1002.csproj b/SW.Serverless.Compat.ManifestV1002/SW.Serverless.Compat.ManifestV1002.csproj index f498991..7e7504a 100644 --- a/SW.Serverless.Compat.ManifestV1002/SW.Serverless.Compat.ManifestV1002.csproj +++ b/SW.Serverless.Compat.ManifestV1002/SW.Serverless.Compat.ManifestV1002.csproj @@ -1,7 +1,7 @@ Exe diff --git a/SW.Serverless.CompatibilityTests/ListingAndManifestTests.cs b/SW.Serverless.CompatibilityTests/ListingAndManifestTests.cs index edd35ae..a44de1b 100644 --- a/SW.Serverless.CompatibilityTests/ListingAndManifestTests.cs +++ b/SW.Serverless.CompatibilityTests/ListingAndManifestTests.cs @@ -8,7 +8,7 @@ namespace SW.Serverless.CompatibilityTests; /// -/// (d) Bitween builds older than the catalog list adapters by listing storage. Whatever the current +/// (d) Applications older than the catalog list adapters by listing storage. Whatever the current /// installer writes must look to them like exactly the adapters that exist — no catalog file, no /// version folder, no stray id. /// @@ -17,14 +17,14 @@ public class OldListingTests { const string Sample = "SW.Serverless.Samples.Classic"; - /// Bitween's Semver.IsVersionNumber, as shipped. + /// The version test deployed listings use, as shipped. static bool IsVersionNumber(string text) => Regex.IsMatch(text, @"^\d+\.\d+\.\d+(-\S+)?$"); /// - /// The grouping in Bitween's AdapterListing and Search handlers, copied as shipped: non-empty + /// The grouping deployed listings use, copied as shipped: non-empty /// keys, a semver last segment grouped under the segment before it, anything else an adapter id. /// - static async Task>> OldBitweenListAsync(ICloudFilesService files, string prefix) + static async Task>> OldListingAsync(ICloudFilesService files, string prefix) { var index = "adapters".Length + 1; return (await files.ListAsync(prefix)) @@ -41,10 +41,10 @@ static async Task>> OldBitweenListAsync(ICloudFi } [TestMethod] - public async Task Old_bitween_sees_exactly_the_adapter_after_versions_promote_and_withdraw() + public async Task An_old_listing_sees_exactly_the_adapter_after_versions_promote_and_withdraw() { using var bucket = new Bucket(); - // Named by the old convention, so both of old Bitween's listings — by convention prefix, + // Named by the old convention, so both of the old listings — by convention prefix, // and everything under adapters/ — are exercised. const string id = "infolink6.handlers.compat"; @@ -57,9 +57,9 @@ public async Task Old_bitween_sees_exactly_the_adapter_after_versions_promote_an foreach (var prefix in new[] { "adapters/", "adapters/infolink6.handlers" }) { - var listed = await OldBitweenListAsync(bucket.Files, prefix); + var listed = await OldListingAsync(bucket.Files, prefix); CollectionAssert.AreEqual(new[] { id }, listed.Keys.ToList(), $"listing {prefix}"); - Assert.AreEqual(0, listed[id].Count, "no version is under adapters/ for old Bitween to offer"); + Assert.AreEqual(0, listed[id].Count, "no version is under adapters/ for an old listing to offer"); } CollectionAssert.AreEqual(new[] { $"adapters/{id}" }, await bucket.KeysAsync("adapters/"), diff --git a/SW.Serverless.CompatibilityTests/MultiLanguageManifestTests.cs b/SW.Serverless.CompatibilityTests/MultiLanguageManifestTests.cs index 7dee375..3e15ed3 100644 --- a/SW.Serverless.CompatibilityTests/MultiLanguageManifestTests.cs +++ b/SW.Serverless.CompatibilityTests/MultiLanguageManifestTests.cs @@ -8,7 +8,7 @@ namespace SW.Serverless.CompatibilityTests; /// /// The manifest fields multi-language adapters add — runtime version, platforms and their entries, /// contracts, the source the package carries — written by the current tools and read by the -/// manifest parser released Bitween uses (SimplyWorks.Serverless.Contract 10.0.2). That parser +/// first published manifest parser (SimplyWorks.Serverless.Contract 10.0.2), which deployed applications use. That parser /// must keep every one of them, find nothing wrong, and still see the fields it knows as before. /// [TestClass] @@ -27,7 +27,7 @@ public class MultiLanguageManifestTests Lifecycle = AdapterManifest.ClassicLifecycle, Platforms = new() { "linux-x64", "linux-arm64" }, Entries = new() { ["linux-x64"] = "x64/main.py", ["linux-arm64"] = "arm64/main.py" }, - Contracts = new() { ["bitween"] = 1 }, + Contracts = new() { ["orders"] = 1 }, Source = new AdapterSource { Files = new() { ["main.py"] = new string('a', 64), ["requirements.lock"] = new string('b', 64) }, @@ -72,7 +72,7 @@ public void The_new_fields_round_trip_through_the_current_parser() CollectionAssert.AreEqual(new[] { "linux-x64", "linux-arm64" }, again.Platforms); Assert.AreEqual("arm64/main.py", again.EntryFor("linux-arm64")); Assert.AreEqual("main.py", again.EntryFor("osx-arm64"), "a platform without its own entry uses the default"); - Assert.AreEqual(1, again.Contracts!["bitween"]); + Assert.AreEqual(1, again.Contracts!["orders"]); Assert.AreEqual(AdapterSource.DefaultPath, again.Source!.Path); Assert.AreEqual(2, again.Source.Files.Count); Assert.IsNull(again.Extensions, "nothing unknown to the current parser"); @@ -95,7 +95,7 @@ public void A_manifest_without_the_new_fields_is_read_as_a_dotnet_adapter_as_bef [DataRow("""{ "platforms": ["Linux x64"] }""", "platform")] [DataRow("""{ "platforms": ["linux-x64"], "entries": { "osx-arm64": "a" } }""", "doesn't list")] [DataRow("""{ "platforms": ["linux-x64"], "entries": { "linux-x64": "../../etc/passwd" } }""", "inside the package")] - [DataRow("""{ "contracts": { "bitween": 0 } }""", "contract")] + [DataRow("""{ "contracts": { "orders": 0 } }""", "contract")] [DataRow("""{ "source": { "files": { "main.py": "not-a-hash" } } }""", "SHA-256")] [DataRow("""{ "source": { "path": "/abs", "files": {} } }""", "source.path")] public void A_wrong_new_field_is_named_by_validation(string json, string expected) diff --git a/SW.Serverless.CompatibilityTests/PackageLayoutTests.cs b/SW.Serverless.CompatibilityTests/PackageLayoutTests.cs index e3d5815..d8ccc70 100644 --- a/SW.Serverless.CompatibilityTests/PackageLayoutTests.cs +++ b/SW.Serverless.CompatibilityTests/PackageLayoutTests.cs @@ -113,7 +113,7 @@ public async Task An_adapter_in_another_runtime_is_out_of_old_hosts_sight_and_ne await repository.PublishVersionAsync(id, "1.0.0", Package(Python(id)), package, promote: true, "compat"); await repository.PublishVersionAsync(id, "1.1.0", Package(Python(id)), package, promote: false, "compat"); - Assert.AreEqual(0, (await bucket.KeysAsync("adapters/")).Count, "nothing under adapters/, where old hosts and old Bitween look"); + Assert.AreEqual(0, (await bucket.KeysAsync("adapters/")).Count, "nothing under adapters/, where old hosts and old listings look"); Assert.AreEqual("1.0.0", (await repository.LoadEntryAsync(id)).Current); var old = await Compat.RunOldHostAsync(bucket, id, "--command", "Echo", "--input", "x"); diff --git a/SW.Serverless.CompatibilityTests/SW.Serverless.CompatibilityTests.csproj b/SW.Serverless.CompatibilityTests/SW.Serverless.CompatibilityTests.csproj index 8910b0a..b3d9c16 100644 --- a/SW.Serverless.CompatibilityTests/SW.Serverless.CompatibilityTests.csproj +++ b/SW.Serverless.CompatibilityTests/SW.Serverless.CompatibilityTests.csproj @@ -1,7 +1,7 @@ net10.0 diff --git a/SW.Serverless.Contract/Catalog/AdapterManifest.cs b/SW.Serverless.Contract/Catalog/AdapterManifest.cs index e17d36b..7093bfe 100644 --- a/SW.Serverless.Contract/Catalog/AdapterManifest.cs +++ b/SW.Serverless.Contract/Catalog/AdapterManifest.cs @@ -101,7 +101,7 @@ public class AdapterManifest /// public Dictionary Entries { get; set; } - /// The contracts it implements and their versions, e.g. bitween → 1. + /// The contracts it implements and their versions, e.g. orders → 1. public Dictionary Contracts { get; set; } /// The source the package was built from, when it carries it. @@ -210,6 +210,10 @@ public IReadOnlyList Validate() if (Compatibility?.MinHostVersion is { Length: > 0 } host && !VersionPattern.IsMatch(host) && !System.Version.TryParse(host, out _)) problems.Add($"compatibility.minHostVersion '{host}' is not a version."); + foreach (var (application, minimum) in Compatibility?.Applications ?? new Dictionary()) + if (string.IsNullOrWhiteSpace(application) || string.IsNullOrWhiteSpace(minimum) || + (!VersionPattern.IsMatch(minimum) && !System.Version.TryParse(minimum, out _))) + problems.Add($"compatibility.applications '{application}': '{minimum}' is not a version."); var seen = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (var property in Properties ?? new List()) @@ -287,8 +291,30 @@ public class AdapterCompatibility /// The lowest SW.Serverless host that may run it. The host refuses to install it below. public string MinHostVersion { get; set; } - /// The lowest Bitween that may use it. Bitween warns and refuses to bind it below. - public string MinBitweenVersion { get; set; } + /// + /// The lowest version of each application that may use it, by the application's name. Hosts + /// ignore it; each application reads its own and refuses an adapter it is too old for. + /// + public Dictionary Applications { get; set; } + + /// + /// The lowest version of that may use the adapter, or null. + /// Reads , and a manifest written before it with + /// "min<Application>Version", which is kept among . + /// + public string MinVersionOf(string application) + { + if (string.IsNullOrWhiteSpace(application)) return null; + if (Applications != null) + foreach (var (name, version) in Applications) + if (string.Equals(name, application, StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(version)) + return version; + if (Extensions != null) + foreach (var (name, value) in Extensions) + if (string.Equals(name, $"min{application}Version", StringComparison.OrdinalIgnoreCase) && value.ValueKind == JsonValueKind.String) + return value.GetString(); + return null; + } /// Fields written by a newer tool, kept so they are not lost on a round trip. [JsonExtensionData] diff --git a/SW.Serverless.Contract/Protos/adapter.proto b/SW.Serverless.Contract/Protos/adapter.proto index f692739..0da4de5 100644 --- a/SW.Serverless.Contract/Protos/adapter.proto +++ b/SW.Serverless.Contract/Protos/adapter.proto @@ -3,8 +3,8 @@ syntax = "proto3"; option csharp_namespace = "SW.Serverless.Contract"; package sw.serverless.v1; -// The ENVELOPE only. Command names and payload shapes belong to the host's own SDK -// (SimplyWorks.TraxisGateway.Sdk, SW.Bitween.Sdk, ...), never to this file — see the +// The ENVELOPE only. Command names and payload shapes belong to the host application (its +// own SDK or contract package), never to this file — see the // design doc, section 14.6. Payloads are opaque bytes here on purpose. service AdapterHost { @@ -51,7 +51,7 @@ message Invoke { // Configuration for THIS call, on top of the startup values the process was given. // - // An exclusive resident instance is shared: in Bitween one database connection serves every + // An exclusive resident instance is shared: one database connection can serve every // subscription bound to it, and each of those has its own settings — which statement to run, // which operation it is. Startup values cannot carry that, because they belong to the process // and the process belongs to all of them. This is the same reason session_id exists: a shared @@ -129,11 +129,11 @@ message Hello { // UI needs to ask for before running it. repeated SettingInfo settings = 9; - // The kinds of adapter it implements for a contract, as that contract names them — for - // Bitween: handler, mapper, validator, receiver. + // The kinds of adapter it implements for a contract, as that contract names them — for an + // orders contract, say: processor, source. repeated string kinds = 10; - // The contracts it implements and their versions, e.g. bitween -> 1. + // The contracts it implements and their versions, e.g. orders -> 1. map contracts = 11; } diff --git a/SW.Serverless.Installer.UnitTests/BuildTests.cs b/SW.Serverless.Installer.UnitTests/BuildTests.cs index 27fe5e0..4bfee78 100644 --- a/SW.Serverless.Installer.UnitTests/BuildTests.cs +++ b/SW.Serverless.Installer.UnitTests/BuildTests.cs @@ -88,7 +88,7 @@ static string RepositoryRoot() } /// - /// A copy of the BitweenHandler test adapter as an author's project: inside the repository, so + /// A copy of the OrdersProcessor test adapter as an author's project: inside the repository, so /// its reference to the SDK project still resolves, with an adapter.json and a few files the /// source rules must leave out. /// @@ -109,7 +109,7 @@ static string AuthorProject(string authorJson = null, IDictionary """); - File.Copy(Path.Combine(root, "SW.Serverless.UnitTests.BitweenHandler", "Program.cs"), Path.Combine(project, "Program.cs")); + File.Copy(Path.Combine(root, "SW.Serverless.UnitTests.OrdersProcessor", "Program.cs"), Path.Combine(project, "Program.cs")); File.WriteAllText(Path.Combine(project, AdapterManifest.FileName), authorJson ?? """{ "id": "acme.orders", "version": "1.2.0", "displayName": "Acme orders", "properties": [ { "name": "Mode", "type": "select", "options": ["working", "broken"] } ] }"""); File.WriteAllText(Path.Combine(project, ".env"), "API_KEY=never-carried"); @@ -148,8 +148,8 @@ public async Task A_build_writes_the_manifest_from_the_adapter_and_carries_its_s Assert.AreEqual("AcmeOrders.dll", manifest.Entry); Assert.AreEqual("10.1.0", manifest.SdkVersion); Assert.AreEqual(AdapterManifest.ClassicLifecycle, manifest.Lifecycle); - CollectionAssert.AreEqual(new[] { "handler" }, manifest.Kinds); - Assert.AreEqual(1, manifest.Contracts!["bitween"]); + CollectionAssert.AreEqual(new[] { "processor" }, manifest.Kinds); + Assert.AreEqual(1, manifest.Contracts!["orders"]); var mode = manifest.Properties.Single(p => p.Name == "Mode"); Assert.AreEqual("select", mode.Type, "the author's presentation is kept"); @@ -184,6 +184,7 @@ public async Task What_a_build_makes_passes_the_conformance_kit() { PackageDirectory = result.PackageDirectory, Settings = new Dictionary { ["ApiKey"] = "k" }, + Contracts = { ContractDocument.FromFile(Path.Combine(AppContext.BaseDirectory, "Contracts", "orders-adapter-contract.v1.json")) }, }); Assert.IsTrue(report.Passed, string.Join("; ", report.Checks.Where(c => c.Outcome == CheckOutcome.Failed))); } diff --git a/SW.Serverless.Installer.UnitTests/CatalogPublishingTests.cs b/SW.Serverless.Installer.UnitTests/CatalogPublishingTests.cs index de66e39..9a4ec8d 100644 --- a/SW.Serverless.Installer.UnitTests/CatalogPublishingTests.cs +++ b/SW.Serverless.Installer.UnitTests/CatalogPublishingTests.cs @@ -149,7 +149,7 @@ Task Entry(string id, ICloudFilesService? files = null) => static void AssertLegacyMetadata(IReadOnlyDictionary metadata, string? version) { foreach (var key in LegacyKeys) - Assert.IsTrue(metadata.ContainsKey(key), $"adapters/{{id}} is missing the {key} metadata an older host or Bitween reads"); + Assert.IsTrue(metadata.ContainsKey(key), $"adapters/{{id}} is missing the {key} metadata an older host or listing reads"); Assert.AreEqual(Classic + ".dll", metadata["EntryAssembly"]); Assert.AreEqual("dotnet", metadata["Lang"]); Assert.IsFalse(string.IsNullOrWhiteSpace(metadata["Hash"]), "every host requires Hash"); @@ -657,10 +657,10 @@ public async Task Version_numbers_count_versions_left_by_an_older_installer() Assert.AreEqual("1.2.1", (await Publish("legacy.adapter", "patch", promote: false)).Version); } - // ---------------------------------------------------------------- what older hosts and Bitween see + // ---------------------------------------------------------------- what older hosts and listings see /// - /// Older Bitween lists every key under adapters/ as an adapter, so after anything this installer + /// Older listings take every key under adapters/ for an adapter, so after anything this installer /// does, the only key there must be adapters/{id}. /// [TestMethod] @@ -728,7 +728,7 @@ public async Task A_repository_and_a_publish_work_under_the_folder_they_are_give Directory.CreateDirectory(project); File.WriteAllText(Path.Combine(project, "adapter.json"), """{ "id": "acme.py", "version": "1.0.0", "runtime": "python", "entry": "main.py" }"""); File.WriteAllText(Path.Combine(project, "main.py"), """ - import simplyworks_serverless as sw + import sw_serverless as sw class Echo: @sw.command("Echo") diff --git a/SW.Serverless.Installer.UnitTests/CliCommandTests.cs b/SW.Serverless.Installer.UnitTests/CliCommandTests.cs index b6eaf1f..2690349 100644 --- a/SW.Serverless.Installer.UnitTests/CliCommandTests.cs +++ b/SW.Serverless.Installer.UnitTests/CliCommandTests.cs @@ -58,18 +58,18 @@ public async Task Init_makes_a_project_and_refuses_what_it_can_t_make() { var work = WorkFolder(); - var (exit, output) = await Cli("init", "AcmeOrders", "--kind", "validator", "--dir", work); + var (exit, output) = await Cli("init", "AcmeOrders", "--dir", work); Assert.AreEqual(Program.Success, exit, output); var project = Path.Combine(work, "AcmeOrders"); foreach (var file in new[] { "AcmeOrders.csproj", "Program.cs", "adapter.json", "settings.example.json", ".gitignore", "README.md" }) Assert.IsTrue(File.Exists(Path.Combine(project, file)), file); Assert.AreEqual("acme.orders", AdapterManifest.Parse(File.ReadAllText(Path.Combine(project, "adapter.json"))).Id); - StringAssert.Contains(File.ReadAllText(Path.Combine(project, "Program.cs")), "IBitweenValidator"); + StringAssert.Contains(File.ReadAllText(Path.Combine(project, "Program.cs")), "public Task Greet(string name)"); Assert.AreEqual(Program.Failure, (await Cli("init", "AcmeOrders", "--dir", work)).Exit, "an existing project isn't overwritten"); var (goExit, goOutput) = await Cli("init", "GoOrders", "--lang", "go", "--dir", work); Assert.AreEqual(Program.Failure, goExit); - StringAssert.Contains(goOutput, "arrives with that language's SDK"); + StringAssert.Contains(goOutput, "isn't a language the SDK has"); } [TestMethod] @@ -86,7 +86,7 @@ public async Task Manifest_validate_passes_a_good_file_and_names_what_is_wrong_w StringAssert.Contains(output, "runtime '../sh'"); } - /// An author's project, as BuildTests makes one: the BitweenHandler adapter with an adapter.json. + /// An author's project, as BuildTests makes one: the sample orders processor with an adapter.json. static string AuthorProject() { var root = RepositoryRoot(); @@ -104,7 +104,7 @@ static string AuthorProject() """); - File.Copy(Path.Combine(root, "SW.Serverless.UnitTests.BitweenHandler", "Program.cs"), Path.Combine(project, "Program.cs")); + File.Copy(Path.Combine(root, "SW.Serverless.UnitTests.OrdersProcessor", "Program.cs"), Path.Combine(project, "Program.cs")); File.WriteAllText(Path.Combine(project, "adapter.json"), """{ "id": "acme.orders", "version": "1.2.0", "displayName": "Acme orders" }"""); return project; } @@ -122,14 +122,16 @@ public async Task Build_test_run_and_publish_take_a_project_into_storage() var zip = Path.Combine(project, "bin", "serverless", "acme.orders-1.2.0.zip"); Assert.IsTrue(File.Exists(zip), build.Output); - var test = await Cli("test", zip, "--settings", settings); + // The contract it declares is the application's, handed to the CLI with --contract. + var contract = Path.Combine(AppContext.BaseDirectory, "Contracts", "orders-adapter-contract.v1.json"); + var test = await Cli("test", zip, "--settings", settings, "--contract", contract); Assert.AreEqual(Program.Success, test.Exit, test.Output); - StringAssert.Contains(test.Output, "PASS bitween handler: Handle answers example 1"); + StringAssert.Contains(test.Output, "PASS orders processor: Process answers example 1"); StringAssert.Contains(test.Output, "Conforms."); - var run = await Cli("run", zip, "--settings", settings, "--call", "Handle", "--input", """{"Data":"{\"order\":1}"}"""); + var run = await Cli("run", zip, "--settings", settings, "--call", "Process", "--input", """{"OrderId":"SO-1"}"""); Assert.AreEqual(Program.Success, run.Exit, run.Output); - StringAssert.Contains(run.Output, "accepted"); + StringAssert.Contains(run.Output, "\"Accepted\":true"); var store = Path.Combine(Path.GetDirectoryName(project)!, "store"); var publish = await Cli("publish", zip, "-p", "local", "-b", "cli-tests", "-u", store); @@ -152,132 +154,57 @@ public async Task Build_test_run_and_publish_take_a_project_into_storage() } /// - /// What init writes builds and conforms. The templates reference the published SDK and Bitween - /// contract packages; until those are on NuGet the project is pointed at the projects themselves, - /// with Bitween-api beside this repository. + /// What init writes, in every language, builds, conforms and answers: a .NET adapter pointed at + /// this repository's SDK project, and Python and Node ones built with the SDK the CLI carries, so + /// no NuGet, PyPI or npm is needed. /// [DataTestMethod] - [DataRow("handler")] - [DataRow("receiver")] - [DataRow("validator")] - public async Task What_init_writes_builds_and_conforms(string kind) - { - var root = RepositoryRoot(); - var contracts = Path.GetFullPath(Path.Combine(root, "..", "Bitween-api", "SW.Bitween.Adapters", "SW.Bitween.Adapters.csproj")); - if (!File.Exists(contracts)) Assert.Inconclusive($"Bitween-api isn't beside this repository ({contracts})"); - - var work = WorkFolder(); - Assert.AreEqual(Program.Success, (await Cli("init", "Acme" + char.ToUpper(kind[0]) + kind[1..], "--kind", kind, "--dir", work)).Exit); - var project = Directory.GetDirectories(work).Single(); - var csproj = Directory.GetFiles(project, "*.csproj").Single(); - var text = File.ReadAllText(csproj) - .Replace($"", - $"") - .Replace($"", - $"") - .Replace("true", ""); - File.WriteAllText(csproj, text); - - var build = await Cli("build", project, "--no-source"); - Assert.AreEqual(Program.Success, build.Exit, build.Output); - - var settings = Path.Combine(work, "settings.json"); - File.WriteAllText(settings, """{ "ApiKey": "k" }"""); - var test = await Cli("test", Path.Combine(project, "bin", "serverless", "package"), "--settings", settings); - Assert.AreEqual(Program.Success, test.Exit, test.Output); - } - - /// - /// A Python adapter, from init to a package that conforms: built with the SDK and the Bitween - /// kinds vendored from the copies the CLI carries, so no PyPI and no network are needed. - /// - [DataTestMethod] - [DataRow("handler")] - [DataRow("mapper")] - [DataRow("receiver")] - [DataRow("validator")] - public async Task What_init_writes_in_python_builds_and_conforms(string kind) - { - var work = WorkFolder(); - var name = "Py" + char.ToUpper(kind[0]) + kind[1..]; - Assert.AreEqual(Program.Success, (await Cli("init", name, "--lang", "python", "--kind", kind, "--dir", work)).Exit); - var project = Path.Combine(work, name); - Assert.IsTrue(File.Exists(Path.Combine(project, "main.py"))); - - var build = await Cli("build", project); - Assert.AreEqual(Program.Success, build.Exit, build.Output); - - var package = Path.Combine(project, "bin", "serverless", "package"); - var manifest = SW.Serverless.Contract.Catalog.AdapterManifest.Parse(File.ReadAllText(Path.Combine(package, "adapter.json"))); - Assert.AreEqual("python", manifest.Runtime); - Assert.AreEqual(Tooling.Building.PythonBuild.EntryScript, manifest.Entry); - Assert.AreEqual("classic", manifest.Lifecycle); - Assert.AreEqual(2, manifest.Protocol.Min); - CollectionAssert.AreEqual(new[] { kind }, manifest.Kinds); - Assert.AreEqual(1, manifest.Contracts["bitween"]); - Assert.IsNull(manifest.Platforms, "nothing native: it runs anywhere"); - Assert.IsTrue(File.Exists(Path.Combine(package, "_vendor", "simplyworks_serverless", "__init__.py"))); - Assert.IsTrue(File.Exists(Path.Combine(package, "_vendor", "simplyworks_bitween", "__init__.py"))); - Assert.IsTrue(manifest.Source.Files.ContainsKey("main.py")); - Assert.IsTrue(File.Exists(Path.Combine(package, "source", "main.py"))); - - var settings = Path.Combine(work, "settings.json"); - File.WriteAllText(settings, """{ "ApiKey": "k" }"""); - // The project folder: built first, then checked. - var test = await Cli("test", project, "--settings", settings); - Assert.AreEqual(Program.Success, test.Exit, test.Output); - } - - /// - /// The CLI vendors its own copy of the Bitween kinds for Python, as it carries its own copy of - /// the contract; it must be the one Bitween-api maintains. - /// - [TestMethod] - public void The_python_bitween_kinds_the_cli_carries_are_bitween_s() - { - var root = RepositoryRoot(); - var original = Path.GetFullPath(Path.Combine(root, "..", "Bitween-api", "sdk", "python", "src", "simplyworks_bitween", "__init__.py")); - if (!File.Exists(original)) Assert.Inconclusive($"Bitween-api isn't beside this repository ({original})"); - var copy = Path.Combine(root, "SW.Serverless.Tooling", "Contracts", "bitween", "python", "simplyworks_bitween", "__init__.py"); - Assert.AreEqual(File.ReadAllText(original), File.ReadAllText(copy), - "copy Bitween-api/sdk/python/src/simplyworks_bitween into SW.Serverless.Tooling/Contracts/bitween/python"); - } - - /// - /// A JavaScript or TypeScript adapter, from init to a package that conforms: TypeScript's types - /// stripped by Node itself, the SDKs vendored into node_modules from the copies the CLI carries. - /// - [DataTestMethod] - [DataRow("node", "handler")] - [DataRow("node", "receiver")] - [DataRow("typescript", "mapper")] - [DataRow("typescript", "validator")] - public async Task What_init_writes_in_node_builds_and_conforms(string language, string kind) + [DataRow("dotnet")] + [DataRow("python")] + [DataRow("node")] + [DataRow("typescript")] + public async Task What_init_writes_builds_conforms_and_answers(string language) { if (language == "typescript" && !NodeStripsTypes()) Assert.Inconclusive("this machine's node can't strip TypeScript types; it takes Node 22.13 or later"); + + var root = RepositoryRoot(); var work = WorkFolder(); - var name = (language == "node" ? "Js" : "Ts") + char.ToUpper(kind[0]) + kind[1..]; - Assert.AreEqual(Program.Success, (await Cli("init", name, "--lang", language, "--kind", kind, "--dir", work)).Exit); + var name = "Greeter" + char.ToUpper(language[0]) + language[1..]; + Assert.AreEqual(Program.Success, (await Cli("init", name, "--lang", language, "--dir", work)).Exit); var project = Path.Combine(work, name); + if (language == "dotnet") + { + var csproj = Directory.GetFiles(project, "*.csproj").Single(); + File.WriteAllText(csproj, File.ReadAllText(csproj) + .Replace($"", + $"") + .Replace("true", "")); + } + var build = await Cli("build", project); Assert.AreEqual(Program.Success, build.Exit, build.Output); var package = Path.Combine(project, "bin", "serverless", "package"); - var manifest = SW.Serverless.Contract.Catalog.AdapterManifest.Parse(File.ReadAllText(Path.Combine(package, "adapter.json"))); - Assert.AreEqual("node", manifest.Runtime); - Assert.AreEqual("main.js", manifest.Entry); - Assert.AreEqual(language == "node" ? "javascript" : "typescript", manifest.Language); - CollectionAssert.AreEqual(new[] { kind }, manifest.Kinds); - Assert.IsTrue(File.Exists(Path.Combine(package, "node_modules", "@simplyworks", "serverless", "src", "index.js"))); - Assert.IsTrue(File.Exists(Path.Combine(package, "node_modules", "@simplyworks", "bitween", "src", "index.js"))); - Assert.IsTrue(manifest.Source.Files.ContainsKey(language == "node" ? "main.js" : "main.ts")); - - var settings = Path.Combine(work, "settings.json"); - File.WriteAllText(settings, """{ "ApiKey": "k" }"""); - var test = await Cli("test", project, "--settings", settings); + var manifest = AdapterManifest.Parse(File.ReadAllText(Path.Combine(package, "adapter.json"))); + Assert.AreEqual(Tooling.Scaffolding.Scaffolder.IdFrom(name), manifest.Id); + Assert.AreEqual(language switch { "python" => "python", "node" or "typescript" => "node", _ => "dotnet" }, manifest.Runtime); + CollectionAssert.AreEquivalent(new[] { "Greeting", "ApiKey" }, manifest.Properties.Select(p => p.Name).ToArray()); + Assert.IsTrue(manifest.Properties.Single(p => p.Name == "ApiKey").Secret); + Assert.IsNull(manifest.Contracts, "the generic adapter implements no contract"); + Assert.IsTrue(manifest.Source.Files.Count > 0, "it carries its source"); + if (language == "python") + Assert.IsTrue(File.Exists(Path.Combine(package, "_vendor", "sw_serverless", "__init__.py"))); + if (language is "node" or "typescript") + Assert.IsTrue(File.Exists(Path.Combine(package, "node_modules", "@simplyworks", "sw-serverless", "src", "index.js"))); + + var test = await Cli("test", project); Assert.AreEqual(Program.Success, test.Exit, test.Output); + + var run = await Cli("run", project, "--call", "Greet", "--input", "Ada"); + Assert.AreEqual(Program.Success, run.Exit, run.Output); + StringAssert.Contains(run.Output, "Hello, Ada!"); } static bool NodeStripsTypes() @@ -295,16 +222,4 @@ static bool NodeStripsTypes() return false; } } - - [TestMethod] - public void The_node_bitween_kinds_the_cli_carries_are_bitween_s() - { - var root = RepositoryRoot(); - var original = Path.GetFullPath(Path.Combine(root, "..", "Bitween-api", "sdk", "node")); - if (!Directory.Exists(original)) Assert.Inconclusive($"Bitween-api isn't beside this repository ({original})"); - var copy = Path.Combine(root, "SW.Serverless.Tooling", "Contracts", "bitween", "node", "@simplyworks", "bitween"); - foreach (var file in new[] { "package.json", "src/index.js", "src/index.d.ts" }) - Assert.AreEqual(File.ReadAllText(Path.Combine(original, file)), File.ReadAllText(Path.Combine(copy, file)), - $"copy Bitween-api/sdk/node/{file} into SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween"); - } } diff --git a/SW.Serverless.Installer.UnitTests/ConformanceTests.cs b/SW.Serverless.Installer.UnitTests/ConformanceTests.cs index a080e48..9db8c52 100644 --- a/SW.Serverless.Installer.UnitTests/ConformanceTests.cs +++ b/SW.Serverless.Installer.UnitTests/ConformanceTests.cs @@ -10,8 +10,9 @@ namespace SW.Serverless.Installer.UnitTests; /// -/// The conformance kit — what serverless test runs — against real adapters built beside the tests: -/// started as a host starts them, described, and called with the Bitween contract's examples. +/// The conformance kit — what sw-serverless test runs — against real adapters built beside the +/// tests: started as a host starts them, described, and called with the examples of the sample +/// "orders" contract in Contracts/, handed to the kit as any application's contract would be. /// [TestClass] public class ConformanceTests @@ -43,13 +44,15 @@ static string Package(string project, AdapterManifest manifest) return target; } - static AdapterManifest HandlerManifest(bool listApiKey = true) + static string ContractFile => Path.Combine(AppContext.BaseDirectory, "Contracts", "orders-adapter-contract.v1.json"); + + static AdapterManifest ProcessorManifest(bool listApiKey = true, string contract = "orders") { var manifest = new AdapterManifest { - Id = "test.bitween.handler", - Kinds = { "handler" }, - Contracts = new() { ["bitween"] = 1 }, + Id = "test.orders.processor", + Kinds = { "processor" }, + Contracts = new() { [contract] = 1 }, Properties = { new AdapterProperty { Name = "Endpoint", Default = "https://partner.example.test/orders", Description = "Where orders go." }, @@ -61,7 +64,7 @@ static AdapterManifest HandlerManifest(bool listApiKey = true) } static async Task RunAsync(string packageDirectory, IDictionary settings = null, - bool allowDelete = false) + bool allowDelete = false, bool withContract = true) { var report = await new ConformanceRunner().RunAsync(new ConformanceOptions { @@ -69,6 +72,7 @@ static async Task RunAsync(string packageDirectory, IDictiona Settings = settings ?? new Dictionary { ["ApiKey"] = "test-key" }, AllowDelete = allowDelete, CommandTimeoutSeconds = 30, + Contracts = withContract ? new List { ContractDocument.FromFile(ContractFile) } : new List(), }); Console.WriteLine(string.Join(Environment.NewLine, report.Checks.Select(c => $"{c.Outcome,-7} {c.Name} {c.Detail}"))); return report; @@ -78,54 +82,54 @@ static ConformanceCheck Check(ConformanceReport report, string name) => report.Checks.Single(c => c.Name == name); [TestMethod] - public async Task A_conforming_handler_passes_every_check() + public async Task A_conforming_processor_passes_every_check() { - var report = await RunAsync(Package("SW.Serverless.UnitTests.BitweenHandler", HandlerManifest())); + var report = await RunAsync(Package("SW.Serverless.UnitTests.OrdersProcessor", ProcessorManifest())); Assert.IsTrue(report.Passed, string.Join("; ", report.Checks.Where(c => c.Outcome == CheckOutcome.Failed))); foreach (var name in new[] { "manifest", "describe", "settings match the manifest", "starts", - "bitween handler: methods", "bitween handler: Handle answers example 1", "an unknown command is refused", + "orders processor: methods", "orders processor: Process answers example 1", "an unknown command is refused", }) Assert.AreEqual(CheckOutcome.Passed, Check(report, name).Outcome, name); } [TestMethod] - public async Task A_handler_answering_without_data_fails_the_contract() + public async Task A_processor_answering_without_what_the_schema_requires_fails_the_contract() { - var report = await RunAsync(Package("SW.Serverless.UnitTests.BitweenHandler", HandlerManifest()), + var report = await RunAsync(Package("SW.Serverless.UnitTests.OrdersProcessor", ProcessorManifest()), new Dictionary { ["ApiKey"] = "k", ["Mode"] = "broken" }); - var check = Check(report, "bitween handler: Handle answers example 1"); + var check = Check(report, "orders processor: Process answers example 1"); Assert.AreEqual(CheckOutcome.Failed, check.Outcome); - StringAssert.Contains(check.Detail, "isn't a valid ExchangeFile"); + StringAssert.Contains(check.Detail, "isn't a valid Receipt"); Assert.IsFalse(report.Passed); } [TestMethod] public async Task Settings_the_manifest_leaves_out_are_named() { - var report = await RunAsync(Package("SW.Serverless.UnitTests.BitweenHandler", HandlerManifest(listApiKey: false))); + var report = await RunAsync(Package("SW.Serverless.UnitTests.OrdersProcessor", ProcessorManifest(listApiKey: false))); var check = Check(report, "settings match the manifest"); Assert.AreEqual(CheckOutcome.Failed, check.Outcome); StringAssert.Contains(check.Detail, "'ApiKey' is declared by the adapter but missing from the manifest"); } - static (string Package, string Folder) Receiver() + static (string Package, string Folder) Source() { var folder = Path.Combine(Path.GetTempPath(), "swsl-conformance-tests", "source-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(folder); File.WriteAllText(Path.Combine(folder, "a.json"), "{\"n\":1}"); File.WriteAllText(Path.Combine(folder, "b.json"), "{\"n\":2}"); - var package = Package("SW.Serverless.UnitTests.BitweenReceiver", new AdapterManifest + var package = Package("SW.Serverless.UnitTests.OrdersSource", new AdapterManifest { - Id = "test.bitween.receiver", - Kinds = { "receiver" }, - Contracts = new() { ["bitween"] = 1 }, - // Run as Bitween runs receivers, classically, over gRPC. + Id = "test.orders.source", + Kinds = { "source" }, + Contracts = new() { ["orders"] = 1 }, + // Run classically, one session at a time, over gRPC. Protocol = new AdapterProtocolRange { Min = 2, Max = 2 }, Properties = { new AdapterProperty { Name = "Folder", Required = true } }, }); @@ -133,29 +137,52 @@ public async Task Settings_the_manifest_leaves_out_are_named() } [TestMethod] - public async Task A_receiver_runs_its_session_in_order_and_leaves_the_source_alone_unless_allowed() + public async Task A_session_kind_runs_in_order_and_leaves_the_source_alone_unless_allowed() { - var (package, folder) = Receiver(); + var (package, folder) = Source(); var report = await RunAsync(package, new Dictionary { ["Folder"] = folder }); Assert.IsTrue(report.Passed, string.Join("; ", report.Checks.Where(c => c.Outcome == CheckOutcome.Failed))); - Assert.AreEqual(CheckOutcome.Passed, Check(report, "bitween receiver: ListFiles").Outcome); - Assert.AreEqual(CheckOutcome.Passed, Check(report, "bitween receiver: GetFile").Outcome); - Assert.AreEqual(CheckOutcome.Skipped, Check(report, "bitween receiver: DeleteFile").Outcome); + Assert.AreEqual(CheckOutcome.Passed, Check(report, "orders source: List").Outcome); + Assert.AreEqual(CheckOutcome.Passed, Check(report, "orders source: Fetch").Outcome); + Assert.AreEqual(CheckOutcome.Skipped, Check(report, "orders source: Remove").Outcome); Assert.AreEqual(2, Directory.GetFiles(folder).Length, "nothing was deleted"); } [TestMethod] - public async Task A_receiver_s_delete_runs_when_allowed() + public async Task A_destructive_method_runs_when_allowed() { - var (package, folder) = Receiver(); + var (package, folder) = Source(); var report = await RunAsync(package, new Dictionary { ["Folder"] = folder }, allowDelete: true); - Assert.AreEqual(CheckOutcome.Passed, Check(report, "bitween receiver: DeleteFile").Outcome); + Assert.AreEqual(CheckOutcome.Passed, Check(report, "orders source: Remove").Outcome); CollectionAssert.AreEqual(new[] { "b.json" }, Directory.GetFiles(folder).Select(Path.GetFileName).ToArray(), "the first file listed was deleted"); } + [TestMethod] + public async Task A_contract_the_kit_isn_t_given_is_named_with_how_to_give_it() + { + var report = await RunAsync(Package("SW.Serverless.UnitTests.OrdersProcessor", ProcessorManifest(contract: "unheard-of")), + withContract: false); + + var check = Check(report, "contract unheard-of v1"); + Assert.AreEqual(CheckOutcome.Failed, check.Outcome); + StringAssert.Contains(check.Detail, "--contract"); + } + + [TestMethod] + public async Task A_registered_contract_is_checked_without_being_handed_over() + { + // What an application's own CLI does with its contract, once, at start. + ContractDocument.Register(ContractDocument.FromJson(File.ReadAllText(ContractFile), + file => File.ReadAllText(Path.Combine(Path.GetDirectoryName(ContractFile)!, file)))); + + var report = await RunAsync(Package("SW.Serverless.UnitTests.OrdersProcessor", ProcessorManifest()), withContract: false); + + Assert.AreEqual(CheckOutcome.Passed, Check(report, "orders processor: Process answers example 1").Outcome); + } + [TestMethod] public async Task An_adapter_on_the_old_SDK_is_told_it_can_t_describe_itself() { diff --git a/SW.Serverless.Installer.UnitTests/Contracts/order.schema.json b/SW.Serverless.Installer.UnitTests/Contracts/order.schema.json new file mode 100644 index 0000000..c28f059 --- /dev/null +++ b/SW.Serverless.Installer.UnitTests/Contracts/order.schema.json @@ -0,0 +1,11 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Order", + "type": "object", + "required": ["OrderId"], + "properties": { + "OrderId": { "type": "string" }, + "Lines": { "type": "integer" } + }, + "additionalProperties": true +} diff --git a/SW.Serverless.Installer.UnitTests/Contracts/orders-adapter-contract.v1.json b/SW.Serverless.Installer.UnitTests/Contracts/orders-adapter-contract.v1.json new file mode 100644 index 0000000..8afdd3d --- /dev/null +++ b/SW.Serverless.Installer.UnitTests/Contracts/orders-adapter-contract.v1.json @@ -0,0 +1,42 @@ +{ + "contract": "orders", + "version": 1, + "description": "A sample contract for the tests: what an order-processing application calls on its adapters. Shaped like any application's contract, so the kit's handling of kinds, payloads, sessions and destructive methods is tested without depending on a real one.", + "encoding": { + "string": "A string argument or result is the raw UTF-8 text, not a JSON string.", + "object": "Any other argument or result is JSON, with property names exactly as its schema gives them.", + "none": "A method with no argument receives an empty payload; one with no result returns an empty payload." + }, + "types": { + "Order": { "schema": "order.schema.json" }, + "Receipt": { "schema": "receipt.schema.json" }, + "OrderId": { "encoding": "string", "description": "An id a source gave in List, passed back exactly as given." }, + "OrderIdList": { "schema": { "type": "array", "items": { "type": "string" } } } + }, + "kinds": { + "processor": { + "description": "Takes an order and says whether it was accepted.", + "methods": [ + { + "name": "Process", + "input": "Order", + "output": "Receipt", + "examples": [ + { "OrderId": "SO-1001", "Lines": 2 } + ] + } + ] + }, + "source": { + "description": "Where orders come from, read in one session per run.", + "session": "Open, List, then for each order Fetch and — once it is safely taken — Remove, and finally Close, which is also called after a failure.", + "methods": [ + { "name": "Open", "input": null, "output": null }, + { "name": "List", "input": null, "output": "OrderIdList" }, + { "name": "Fetch", "input": "OrderId", "output": "Order" }, + { "name": "Remove", "input": "OrderId", "output": null, "destructive": true }, + { "name": "Close", "input": null, "output": null } + ] + } + } +} diff --git a/SW.Serverless.Installer.UnitTests/Contracts/receipt.schema.json b/SW.Serverless.Installer.UnitTests/Contracts/receipt.schema.json new file mode 100644 index 0000000..28aa09f --- /dev/null +++ b/SW.Serverless.Installer.UnitTests/Contracts/receipt.schema.json @@ -0,0 +1,11 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Receipt", + "type": "object", + "required": ["Accepted"], + "properties": { + "Accepted": { "type": "boolean" }, + "Reference": { "type": ["string", "null"] } + }, + "additionalProperties": true +} diff --git a/SW.Serverless.Installer.UnitTests/SW.Serverless.Installer.UnitTests.csproj b/SW.Serverless.Installer.UnitTests/SW.Serverless.Installer.UnitTests.csproj index 74d418e..fbde970 100644 --- a/SW.Serverless.Installer.UnitTests/SW.Serverless.Installer.UnitTests.csproj +++ b/SW.Serverless.Installer.UnitTests/SW.Serverless.Installer.UnitTests.csproj @@ -22,9 +22,9 @@ - - @@ -34,4 +34,9 @@ ReferenceOutputAssembly="false" /> + + + + + diff --git a/SW.Serverless.Installer/AdapterCommands.cs b/SW.Serverless.Installer/AdapterCommands.cs index fbe3028..4836777 100644 --- a/SW.Serverless.Installer/AdapterCommands.cs +++ b/SW.Serverless.Installer/AdapterCommands.cs @@ -15,18 +15,15 @@ namespace SW.Serverless.Installer { - /// serverless init <name> + /// sw-serverless init <name> public class InitCliOptions { [Value(0, Required = true, MetaName = "name", HelpText = "The adapter's name, e.g. AcmeOrders: its folder, project and class.")] public string Name { get; set; } - [Option("lang", Default = "dotnet", HelpText = "Its language: dotnet, python, node (JavaScript) or typescript. Go arrives with its SDK.")] + [Option("lang", Default = "dotnet", HelpText = "Its language: dotnet, python, node (JavaScript) or typescript.")] public string Language { get; set; } - [Option("kind", Default = "handler", HelpText = "handler, mapper, validator or receiver.")] - public string Kind { get; set; } - [Option("id", HelpText = "Its id in storage; derived from the name when not given (AcmeOrders -> acme.orders).")] public string Id { get; set; } @@ -34,7 +31,7 @@ public class InitCliOptions public string Directory { get; set; } } - /// serverless build [project] + /// sw-serverless build [project] public class BuildCliOptions { [Value(0, MetaName = "project", Default = ".", HelpText = "The adapter's project folder, with its adapter.json.")] @@ -53,7 +50,7 @@ public class BuildCliOptions public bool DryRun { get; set; } } - /// serverless test [package] + /// sw-serverless test [package] public class TestCliOptions { [Value(0, MetaName = "package", Default = ".", @@ -73,7 +70,7 @@ public class TestCliOptions public int Timeout { get; set; } } - /// serverless run [package] --call Command + /// sw-serverless run [package] --call Command public class RunCliOptions { [Value(0, MetaName = "package", Default = ".", HelpText = "A package zip, a package folder, or a project folder (built first).")] @@ -92,7 +89,7 @@ public class RunCliOptions public int Timeout { get; set; } } - /// serverless manifest validate [path] + /// sw-serverless manifest validate [path] public class ManifestCliOptions { [Value(0, Required = true, MetaName = "action", HelpText = "validate")] @@ -102,10 +99,10 @@ public class ManifestCliOptions public string Path { get; set; } } - /// serverless publish <package> + /// sw-serverless publish <package> public class PublishPackageCliOptions : StorageCliOptions { - [Value(0, Required = true, MetaName = "package", HelpText = "A package zip from serverless build.")] + [Value(0, Required = true, MetaName = "package", HelpText = "A package zip from sw-serverless build.")] public string Package { get; set; } [Option('v', "version", HelpText = "The version: explicit, or major, minor or patch to bump. The manifest's version unless set.")] @@ -134,7 +131,6 @@ public static Task Init(InitCliOptions opts) Name = opts.Name, Id = opts.Id, Language = opts.Language, - Kind = opts.Kind, ParentDirectory = opts.Directory, }); if (!result.Succeeded) @@ -143,9 +139,9 @@ public static Task Init(InitCliOptions opts) return Task.FromResult(Failure); } - Console.WriteLine($"Made a {opts.Kind} adapter in {result.ProjectDirectory}:"); + Console.WriteLine($"Made an adapter in {result.ProjectDirectory}:"); foreach (var file in result.Files) Console.WriteLine($" {file}"); - Console.WriteLine("Next: serverless build, then serverless test --settings settings.json"); + Console.WriteLine("Next: sw-serverless build, then sw-serverless test --settings settings.json"); return Task.FromResult(Success); } @@ -314,7 +310,7 @@ static bool IsUnbuiltScript(string folder) if (string.Equals(runtime, AdapterManifest.PythonRuntime, StringComparison.OrdinalIgnoreCase)) return !File.Exists(System.IO.Path.Combine(folder, PythonBuild.EntryScript)); if (string.Equals(runtime, AdapterManifest.NodeRuntime, StringComparison.OrdinalIgnoreCase)) - return !File.Exists(System.IO.Path.Combine(folder, "node_modules", "@simplyworks", "serverless", "package.json")); + return !File.Exists(System.IO.Path.Combine(folder, "node_modules", "@simplyworks", "sw-serverless", "package.json")); return false; } catch (JsonException) diff --git a/SW.Serverless.Installer/Options.cs b/SW.Serverless.Installer/Options.cs index 4f533de..d98d647 100644 --- a/SW.Serverless.Installer/Options.cs +++ b/SW.Serverless.Installer/Options.cs @@ -54,7 +54,7 @@ public class CliOptions : StorageCliOptions public string Version { get; set; } [Option("no-promote", - HelpText = "With -v: upload the version without making it the one that runs. Promote it later with 'serverless promote'.")] + HelpText = "With -v: upload the version without making it the one that runs. Promote it later with 'sw-serverless promote'.")] public bool NoPromote { get; set; } [Option("notes", HelpText = "Release notes for this version (Markdown). Overrides releaseNotes in adapter.json.")] @@ -76,7 +76,7 @@ public class CliOptions : StorageCliOptions public string AdapterId { get; set; } } - /// serverless promote <id> <version> + /// sw-serverless promote <id> <version> public class PromoteCliOptions : StorageCliOptions { [Value(0, Required = true, MetaName = "adapter-id", HelpText = "Adapter Id.")] @@ -86,14 +86,14 @@ public class PromoteCliOptions : StorageCliOptions public string Version { get; set; } } - /// serverless versions <id> + /// sw-serverless versions <id> public class VersionsCliOptions : StorageCliOptions { [Value(0, Required = true, MetaName = "adapter-id", HelpText = "Adapter Id.")] public string AdapterId { get; set; } } - /// serverless withdraw <id> <version> + /// sw-serverless withdraw <id> <version> public class WithdrawCliOptions : StorageCliOptions { [Value(0, Required = true, MetaName = "adapter-id", HelpText = "Adapter Id.")] diff --git a/SW.Serverless.Installer/Program.cs b/SW.Serverless.Installer/Program.cs index cb31f36..42dc048 100644 --- a/SW.Serverless.Installer/Program.cs +++ b/SW.Serverless.Installer/Program.cs @@ -41,7 +41,7 @@ public static async Task RunAsync(string[] args, Func envir }); // A command is recognised before the publish parser sees anything, so every existing - // "serverless " line parses exactly as before. A project file that happens + // "sw-serverless " line parses exactly as before. A project file that happens // to be called "promote" is still a project. if (args.Length > 0 && Commands.Contains(args[0]) && !File.Exists(args[0])) { diff --git a/SW.Serverless.Installer/SW.Serverless.Installer.csproj b/SW.Serverless.Installer/SW.Serverless.Installer.csproj index 3f95b76..ab91d5f 100644 --- a/SW.Serverless.Installer/SW.Serverless.Installer.csproj +++ b/SW.Serverless.Installer/SW.Serverless.Installer.csproj @@ -3,7 +3,7 @@ Exe net10.0 - serverless + sw-serverless latest @@ -13,8 +13,7 @@ - + diff --git a/SW.Serverless.SampleWeb/Telemetry/DashboardEventSink.cs b/SW.Serverless.SampleWeb/Telemetry/DashboardEventSink.cs index 93237f1..e9e298b 100644 --- a/SW.Serverless.SampleWeb/Telemetry/DashboardEventSink.cs +++ b/SW.Serverless.SampleWeb/Telemetry/DashboardEventSink.cs @@ -9,7 +9,7 @@ namespace SW.Serverless.SampleWeb.Telemetry { /// - /// Stands in for a real ingest path. Bitween would persist an Xchange and return its id here; + /// Stands in for a real ingest path. A real host would persist the message and return its id here; /// the adapter does not acknowledge its broker until this returns Accepted. /// public class DashboardEventSink : IAdapterEventSink diff --git a/SW.Serverless.SampleWeb/Telemetry/DedupeWindow.cs b/SW.Serverless.SampleWeb/Telemetry/DedupeWindow.cs index 5be6cff..80e0492 100644 --- a/SW.Serverless.SampleWeb/Telemetry/DedupeWindow.cs +++ b/SW.Serverless.SampleWeb/Telemetry/DedupeWindow.cs @@ -5,7 +5,7 @@ namespace SW.Serverless.SampleWeb.Telemetry { /// - /// A dedupe table with a retention window, which is the shape a real one has too. Bitween keeps + /// A dedupe table with a retention window, which is the shape a real one has too. A real host keeps /// this in the database with a pruning job; a sample keeps it in memory — but not for ever, or a /// long-lived host with a steady stream of distinct keys simply grows until it dies. /// diff --git a/SW.Serverless.Samples.Classic/Handler.cs b/SW.Serverless.Samples.Classic/Handler.cs index ad3ebee..653744b 100644 --- a/SW.Serverless.Samples.Classic/Handler.cs +++ b/SW.Serverless.Samples.Classic/Handler.cs @@ -8,8 +8,8 @@ namespace SW.Serverless.Samples.Classic { /// - /// A conventional per-invocation adapter — the shape the ~190 published Traxis and Bitween - /// adapters already use. Commands are public Task / Task<T> methods discovered by name; + /// A conventional per-invocation adapter — the shape most published adapters + /// already use. Commands are public Task / Task<T> methods discovered by name; /// configuration comes from startup values; logging goes through AdapterLogger. /// [AdapterKind("handler")] @@ -147,8 +147,8 @@ public async Task Slow(int seconds) } // ------------------------------------------------------------------ contracts - // In a real deployment these live in the HOST's SDK — SimplyWorks.TraxisGateway.Sdk, - // SW.Bitween.Sdk — never in SW.Serverless, which only ever sees opaque payloads. + // In a real deployment these live in the host application's own SDK or contract package — + // never in SW.Serverless, which only ever sees opaque payloads. public class Order { diff --git a/SW.Serverless.Samples.Host/ConsoleEventSink.cs b/SW.Serverless.Samples.Host/ConsoleEventSink.cs index 7b34692..d90471c 100644 --- a/SW.Serverless.Samples.Host/ConsoleEventSink.cs +++ b/SW.Serverless.Samples.Host/ConsoleEventSink.cs @@ -8,8 +8,8 @@ namespace SW.Serverless.Samples.Host { /// - /// Stands in for Bitween's ingest path. In the real host this persists an Xchange, writes the - /// payload to cloud storage, commits, and returns the Xchange id — and only then does the + /// Stands in for an application's ingest path. A real host persists the message, writes the + /// payload to cloud storage, commits, and returns its id — and only then does the /// adapter acknowledge its broker. /// public class ConsoleEventSink : IAdapterEventSink @@ -57,7 +57,7 @@ static string Preview(byte[] payload, int chars) /// Keys expire rather than accumulating for the process lifetime — this host runs /// indefinitely, and a steady stream of distinct keys would otherwise grow until it dies. /// The window has to cover the source's redelivery horizon; a real host would keep this in - /// a database with a pruning job, which is exactly what Bitween does. + /// a database with a pruning job, as integration platforms do. /// static class Seen { diff --git a/SW.Serverless.Samples.RabbitMq/Publisher/PublisherHandler.cs b/SW.Serverless.Samples.RabbitMq/Publisher/PublisherHandler.cs index a6c45a9..20eb161 100644 --- a/SW.Serverless.Samples.RabbitMq/Publisher/PublisherHandler.cs +++ b/SW.Serverless.Samples.RabbitMq/Publisher/PublisherHandler.cs @@ -14,7 +14,7 @@ namespace SW.Serverless.Samples.RabbitMq.Publisher /// messages a second, enough to make throughput and backpressure visible rather than /// theoretical. /// - /// This is the direction Bitween does not have yet even on the internal gateway. The adapter + /// The adapter /// owns the connection and the confirm handling; the host only says what to send. /// public class PublisherHandler : RabbitAdapterBase diff --git a/SW.Serverless.Sdk/AdapterContractAttribute.cs b/SW.Serverless.Sdk/AdapterContractAttribute.cs index c5b729b..7c7fd7d 100644 --- a/SW.Serverless.Sdk/AdapterContractAttribute.cs +++ b/SW.Serverless.Sdk/AdapterContractAttribute.cs @@ -3,7 +3,7 @@ namespace SW.Serverless.Sdk { /// - /// A contract this adapter implements, and its version — for Bitween, "bitween" 1. Reported + /// A contract this adapter implements, and its version — e.g. "orders" 1. Reported /// in the adapter's description and handshake, and checked by serverless test. /// [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] diff --git a/SW.Serverless.Tooling/AdapterRepository.cs b/SW.Serverless.Tooling/AdapterRepository.cs index 39fc574..6865272 100644 --- a/SW.Serverless.Tooling/AdapterRepository.cs +++ b/SW.Serverless.Tooling/AdapterRepository.cs @@ -23,7 +23,7 @@ public class PackageInfo public string IconDataUri { get; set; } } - /// One row of serverless versions. + /// One row of sw-serverless versions. public class VersionRow { public string Version { get; set; } @@ -46,13 +46,13 @@ public class VersionListing /// /// The adapters layout in storage, and the only code in the installer that writes to it. /// - /// The layout is a contract with ten production deployments running hosts and Bitween builds - /// older than the catalog, so it is held to three rules: + /// The layout is a contract with production deployments running hosts — and applications that + /// list adapters from storage — older than the catalog, so it is held to three rules: /// /// adapters/{id} always holds whatever is current, with the full metadata an old /// host needs to run it (EntryAssembly and Hash at least). - /// Nothing but adapters/{id} is ever written under adapters/ — older Bitween - /// lists every key there as an adapter. Versions an older installer put at + /// Nothing but adapters/{id} is ever written under adapters/ — older + /// listings take every key there for an adapter. Versions an older installer put at /// adapters/{id}/{version} are read, never written. /// Everything new lives beside it: versions under adapters-versions/, the catalog /// under adapters-catalog/. Nothing old reads either. @@ -258,7 +258,7 @@ public async Task PublishVersionAsync(string adapterId, string version, string z log(promote ? $"Version {version} of '{adapterId}' is published and current." : $"Version {version} of '{adapterId}' is published; '{adapterId}' still runs " + - $"{(entry.Current ?? "its unversioned package")}. Run 'serverless promote {adapterId} {version}' to switch."); + $"{(entry.Current ?? "its unversioned package")}. Run 'sw-serverless promote {adapterId} {version}' to switch."); } /// diff --git a/SW.Serverless.Tooling/Building/NodeBuild.cs b/SW.Serverless.Tooling/Building/NodeBuild.cs index d0f9c4f..fcbd4ed 100644 --- a/SW.Serverless.Tooling/Building/NodeBuild.cs +++ b/SW.Serverless.Tooling/Building/NodeBuild.cs @@ -13,11 +13,11 @@ namespace SW.Serverless.Tooling.Building { /// - /// serverless build for a JavaScript or TypeScript adapter. Node runs JavaScript from its source, + /// sw-serverless build for a JavaScript or TypeScript adapter. Node runs JavaScript from its source, /// so the package is the adapter's files as they are, TypeScript among them with its types /// stripped by Node itself — no compiler, no packages — and a node_modules beside them: the SDK - /// and the Bitween kinds from copies this tool carries, and whatever package.json depends on, - /// through npm. + /// from the copy this tool carries, any packages the request hands it, and whatever + /// package.json depends on, through npm. /// /// /// A dependency with a native addon is built for the machine npm runs on, which is not something @@ -29,7 +29,8 @@ public static class NodeBuild public const string DefaultEntry = "main.js"; public const string DefaultRuntimeVersion = ">=22"; - static readonly string[] OwnPackages = { "@simplyworks/serverless", "@simplyworks/bitween" }; + /// The SDK, which the build vendors itself; package.json naming it is not sent to npm. + static readonly string[] OwnPackages = { "@simplyworks/sw-serverless" }; static readonly string[] TypeScript = { ".ts", ".mts", ".cts" }; internal static async Task BuildAsync(BuildRequest request, string project, AdapterManifest author, BuildResult result) @@ -69,13 +70,14 @@ internal static async Task BuildAsync(BuildRequest request, string project, Adap var platforms = await InstallDependenciesAsync(request, project, author, packageDirectory, result); if (!result.Succeeded) return; WriteOwnPackages(modules); + PythonBuild.WritePackages(request, AdapterManifest.NodeRuntime, modules); request.Log("Asking the adapter to describe itself..."); var (description, problem) = await LocalAdapterHost.DescribeAsync(Path.Combine(packageDirectory, entry), AdapterManifest.NodeRuntime, request.Runtimes); if (description == null) { - result.Problems.Add($"{problem}. A Node adapter describes itself through run() from @simplyworks/serverless; make sure {entry} calls it"); + result.Problems.Add($"{problem}. A Node adapter describes itself through run() from @simplyworks/sw-serverless; make sure {entry} calls it"); return; } result.Warnings.AddRange(description.Warnings); @@ -93,7 +95,7 @@ internal static async Task BuildAsync(BuildRequest request, string project, Adap { manifest.Source = new AdapterSource { - BuildCommand = "serverless build", + BuildCommand = "sw-serverless build", Lockfiles = source.Keys.Where(PackageBuilder.IsLockfile).OrderBy(k => k).ToList(), }; var sourceDirectory = Path.Combine(packageDirectory, AdapterSource.DefaultPath); @@ -184,6 +186,8 @@ static async Task> InstallDependenciesAsync(BuildRequest request, s var dependencies = document?["dependencies"]?.AsObject(); if (dependencies == null) return authored; foreach (var own in OwnPackages) dependencies.Remove(own); + foreach (var provided in request.Packages.Where(p => string.Equals(p.Runtime, AdapterManifest.NodeRuntime, StringComparison.OrdinalIgnoreCase))) + dependencies.Remove(provided.Name); document.Remove("devDependencies"); if (dependencies.Count == 0) return authored; @@ -233,7 +237,7 @@ static async Task> InstallDependenciesAsync(BuildRequest request, s } } - /// The SDK and the Bitween kinds, as this tool carries them, under . + /// The SDK, as this tool carries it, under . public static void WriteOwnPackages(string modules) { var assembly = typeof(NodeBuild).Assembly; diff --git a/SW.Serverless.Tooling/Building/PackageBuilder.cs b/SW.Serverless.Tooling/Building/PackageBuilder.cs index afb684b..fea437e 100644 --- a/SW.Serverless.Tooling/Building/PackageBuilder.cs +++ b/SW.Serverless.Tooling/Building/PackageBuilder.cs @@ -29,6 +29,29 @@ public class BuildRequest public Runtimes.AdapterRuntimeOptions Runtimes { get; set; } = new(); public Action Log { get; set; } = _ => { }; + + /// + /// Packages to vendor beside the SDK, from files rather than PyPI or npm: what an + /// application's own CLI adds — the types of its contract, say. Ignored for .NET, whose + /// packages come from NuGet. + /// + public IList Packages { get; set; } = new List(); + } + + /// A package the build writes into a Python or Node package itself, rather than fetching. + public class VendoredPackage + { + /// python or node. + public string Runtime { get; set; } + + /// + /// Its name as requirements.txt or package.json would give it — acme-orders, + /// @acme/orders — so naming it there doesn't send the build to fetch it. + /// + public string Name { get; set; } + + /// Its files, keyed by path under _vendor/ (Python) or node_modules/ (Node). + public IDictionary Files { get; set; } = new Dictionary(); } public class BuildResult @@ -46,7 +69,7 @@ public class BuildResult } /// - /// What serverless build does, for an adapter in any language its SDK supports: builds it, + /// What sw-serverless build does, for an adapter in any language its SDK supports: builds it, /// asks it to describe itself, writes its manifest from that and the author's adapter.json, /// carries its source under the source rules, and zips the result into a package any host runs. /// @@ -67,7 +90,7 @@ public static async Task BuildAsync(BuildRequest request) var authorPath = Path.Combine(project, AdapterManifest.FileName); if (!File.Exists(authorPath)) { - result.Problems.Add($"there is no {AdapterManifest.FileName} in {project}; serverless init writes one"); + result.Problems.Add($"there is no {AdapterManifest.FileName} in {project}; sw-serverless init writes one"); return result; } var authorJson = await File.ReadAllTextAsync(authorPath); @@ -133,8 +156,8 @@ public static async Task BuildAsync(BuildRequest request) AdapterManifest.DotnetRuntime, request.Runtimes); if (description == null) { - result.Problems.Add($"{problem}. serverless build needs SimplyWorks.Serverless.Sdk 10.1.0 or later; " + - "an adapter on an older SDK is published with serverless as before"); + result.Problems.Add($"{problem}. sw-serverless build needs SimplyWorks.Serverless.Sdk 10.1.0 or later; " + + "an adapter on an older SDK is published with sw-serverless as before"); return result; } result.Warnings.AddRange(description.Warnings); diff --git a/SW.Serverless.Tooling/Building/PythonBuild.cs b/SW.Serverless.Tooling/Building/PythonBuild.cs index d3ff6b5..a23b398 100644 --- a/SW.Serverless.Tooling/Building/PythonBuild.cs +++ b/SW.Serverless.Tooling/Building/PythonBuild.cs @@ -14,9 +14,10 @@ namespace SW.Serverless.Tooling.Building { /// - /// serverless build for a Python adapter. Python runs from its source, so the package is the + /// sw-serverless build for a Python adapter. Python runs from its source, so the package is the /// adapter's files as they are, with what they import vendored under : - /// the SDK and the Bitween kinds from copies this tool carries — no PyPI, no network — and + /// the SDK from the copy this tool carries, and any packages the request hands it — no PyPI, no + /// network — and /// whatever requirements.txt names, through pip. A small entry script puts the vendored code on /// the path and runs the adapter's own entry. /// @@ -37,7 +38,8 @@ public static class PythonBuild public static readonly IReadOnlyList DefaultNativePlatforms = new[] { "linux-x64", "linux-arm64" }; /// The SDKs the build vendors itself; requirements.txt naming them is not sent to pip. - static readonly string[] OwnPackages = { "simplyworks-serverless", "simplyworks_serverless", "simplyworks-bitween", "simplyworks_bitween" }; + /// The SDK, which the build vendors itself; requirements.txt naming it is not sent to pip. + static readonly string[] OwnPackages = { "sw-serverless", "sw_serverless" }; // pip's platform tags for each platform a manifest can name. static readonly Dictionary PipPlatforms = new(StringComparer.OrdinalIgnoreCase) @@ -83,6 +85,7 @@ internal static async Task BuildAsync(BuildRequest request, string project, Adap var vendor = Path.Combine(packageDirectory, VendorFolder); WriteOwnPackages(vendor); + WritePackages(request, AdapterManifest.PythonRuntime, vendor); var (platforms, describeOnly) = await VendorRequirementsAsync(request, project, author, vendor, result); if (!result.Succeeded) return; @@ -94,7 +97,7 @@ internal static async Task BuildAsync(BuildRequest request, string project, Adap AdapterManifest.PythonRuntime, request.Runtimes); if (description == null) { - result.Problems.Add($"{problem}. A Python adapter describes itself through simplyworks_serverless.run(); make sure {entry} calls it"); + result.Problems.Add($"{problem}. A Python adapter describes itself through sw_serverless.run(); make sure {entry} calls it"); return; } result.Warnings.AddRange(description.Warnings); @@ -115,7 +118,7 @@ internal static async Task BuildAsync(BuildRequest request, string project, Adap { manifest.Source = new AdapterSource { - BuildCommand = "serverless build", + BuildCommand = "sw-serverless build", Lockfiles = source.Keys.Where(k => PackageBuilder.IsLockfile(k)).OrderBy(k => k).ToList(), }; var sourceDirectory = Path.Combine(packageDirectory, AdapterSource.DefaultPath); @@ -147,7 +150,21 @@ internal static async Task BuildAsync(BuildRequest request, string project, Adap result.ZipPath = zip; } - /// The SDK and the Bitween kinds, as this tool carries them, under . + /// The packages the request hands the build for , under . + internal static void WritePackages(BuildRequest request, string runtime, string root) + { + foreach (var package in request.Packages.Where(p => string.Equals(p.Runtime, runtime, StringComparison.OrdinalIgnoreCase))) + foreach (var (path, bytes) in package.Files) + { + var target = Path.GetFullPath(Path.Combine(root, path.Replace('\\', '/'))); + if (!target.StartsWith(Path.GetFullPath(root) + Path.DirectorySeparatorChar, StringComparison.Ordinal)) + throw new ArgumentException($"{package.Name} names {path}, outside the folder packages go in."); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.WriteAllBytes(target, bytes); + } + } + + /// The SDK, as this tool carries it, under . public static void WriteOwnPackages(string vendor) { var assembly = typeof(PythonBuild).Assembly; @@ -175,7 +192,7 @@ public static void WriteOwnPackages(string vendor) if (!File.Exists(requirementsFile)) return (author.Platforms is { Count: > 0 } ? author.Platforms : null, null); var lines = (await File.ReadAllLinesAsync(requirementsFile)) - .Where(l => !IsOwnPackage(l)) + .Where(l => !IsProvided(l, request)) .ToList(); if (!lines.Any(l => !string.IsNullOrWhiteSpace(l) && !l.TrimStart().StartsWith('#'))) return (author.Platforms is { Count: > 0 } ? author.Platforms : null, null); @@ -257,12 +274,17 @@ static IEnumerable WheelOptions(string platform) => new[] { "--only-binary=:all:", "--python-version", TargetPythonVersion, "--implementation", "cp" } .Concat(PipPlatforms[platform].SelectMany(tag => new[] { "--platform", tag })); - static bool IsOwnPackage(string line) + /// Whether a requirements line names the SDK or a package the request vendors, which pip isn't asked for. + static bool IsProvided(string line, BuildRequest request) { - var name = Regex.Match(line.Trim(), @"^[A-Za-z0-9_.\-]+").Value; - return OwnPackages.Contains(name, StringComparer.OrdinalIgnoreCase); + var name = Normalize(Regex.Match(line.Trim(), @"^[A-Za-z0-9_.\-]+").Value); + return name.Length > 0 && (OwnPackages.Any(o => Normalize(o) == name) || + request.Packages.Any(p => string.Equals(p.Runtime, AdapterManifest.PythonRuntime, StringComparison.OrdinalIgnoreCase) && Normalize(p.Name) == name)); } + // PyPI treats -, _ and . alike, and case as nothing. + static string Normalize(string name) => Regex.Replace(name ?? "", "[-_.]+", "-").ToLowerInvariant(); + static async Task<(bool Ok, string Output)> PipAsync(BuildRequest request, string[] arguments) { var start = new ProcessStartInfo(request.Runtimes.PythonExecutable) @@ -288,7 +310,7 @@ static string LastLines(string output) => /// when requirements were vendored per platform — and runs the adapter's own entry as __main__. /// internal static string Bootstrap(string entry) => $$""" - # Written by serverless build. Puts the vendored packages on the path and runs {{entry}}. + # Written by sw-serverless build. Puts the vendored packages on the path and runs {{entry}}. import os, platform, runpy, sys here = os.path.dirname(os.path.abspath(__file__)) diff --git a/SW.Serverless.Tooling/CloudFilesFactory.cs b/SW.Serverless.Tooling/CloudFilesFactory.cs index 910014f..452a837 100644 --- a/SW.Serverless.Tooling/CloudFilesFactory.cs +++ b/SW.Serverless.Tooling/CloudFilesFactory.cs @@ -94,7 +94,7 @@ public static ICloudFilesService Create(ServerlessUploadOptions options) break; case "local": - // The filesystem provider, for a developer running Bitween against a local + // The filesystem provider, for a developer running a host against a local // store. Publishing to it is the same command with a different -p, rather than // the hand-built zip and .meta.json it used to take. services.AddLocalTestsCloudFiles(o => diff --git a/SW.Serverless.Tooling/Conformance/ConformanceRunner.cs b/SW.Serverless.Tooling/Conformance/ConformanceRunner.cs index 87203ae..f84bd97 100644 --- a/SW.Serverless.Tooling/Conformance/ConformanceRunner.cs +++ b/SW.Serverless.Tooling/Conformance/ConformanceRunner.cs @@ -12,7 +12,7 @@ namespace SW.Serverless.Tooling.Conformance /// /// Runs an adapter package the way a host does — installed from storage, started on its runtime, /// called by name — and checks it against its manifest, its own description and every contract - /// it declares. What serverless test runs, for an adapter in any language. + /// it declares. What sw-serverless test runs, for an adapter in any language. /// public class ConformanceRunner { @@ -41,7 +41,7 @@ async Task RunCoreAsync(ConformanceOptions options, ConformanceReport report, st var manifestPath = Path.Combine(options.PackageDirectory, AdapterManifest.FileName); if (!File.Exists(manifestPath)) { - report.Fail("manifest", $"there is no {AdapterManifest.FileName} in {options.PackageDirectory}; serverless build writes it"); + report.Fail("manifest", $"there is no {AdapterManifest.FileName} in {options.PackageDirectory}; sw-serverless build writes it"); return; } @@ -132,7 +132,7 @@ static void CheckSettings(AdapterManifest manifest, AdapterSelfDescription descr if (!code.Secret && (code.Default ?? "") != (file.Default ?? "")) differences.Add($"'{name}' defaults to '{code.Default}' in the adapter but '{file.Default}' in the manifest"); } - if (differences.Count > 0) report.Fail("settings match the manifest", string.Join("; ", differences) + " — serverless build rewrites the manifest from the adapter"); + if (differences.Count > 0) report.Fail("settings match the manifest", string.Join("; ", differences) + " — sw-serverless build rewrites the manifest from the adapter"); else report.Pass("settings match the manifest", $"{declared.Count} settings"); } diff --git a/SW.Serverless.Tooling/Conformance/ContractDocument.cs b/SW.Serverless.Tooling/Conformance/ContractDocument.cs index 3088d57..aa6e6ad 100644 --- a/SW.Serverless.Tooling/Conformance/ContractDocument.cs +++ b/SW.Serverless.Tooling/Conformance/ContractDocument.cs @@ -57,21 +57,36 @@ public async Task SchemaOfAsync(string type) }; } - /// The contracts the kit knows by name, carried with it. - public static ContractDocument Known(string name, int version) + static readonly List registered = new(); + + /// + /// Makes a contract known by name, so the kit checks every adapter that declares it without + /// being handed it each time — what an application's own CLI does with its contract. + /// + public static void Register(ContractDocument contract) { - // Found by file name: the folder in a resource's name is written with whichever - // separator the machine that built it uses. - var file = $"{name}-adapter-contract.v{version}.json"; - var resource = typeof(ContractDocument).Assembly.GetManifestResourceNames() - .FirstOrDefault(r => r.StartsWith("SW.Serverless.Tooling.Contracts.", StringComparison.Ordinal) && - r.EndsWith(file, StringComparison.Ordinal)); - if (resource == null) return null; + if (contract == null) throw new ArgumentNullException(nameof(contract)); + lock (registered) + { + registered.RemoveAll(c => string.Equals(c.Name, contract.Name, StringComparison.OrdinalIgnoreCase) && c.Version == contract.Version); + registered.Add(contract); + } + } - var prefix = resource[..^file.Length]; - return new ContractDocument(JObject.Parse(ReadResource(resource)), sibling => ReadResource(prefix + sibling)); + /// A registered contract, or null. The kit carries none of its own. + public static ContractDocument Known(string name, int version) + { + lock (registered) + return registered.FirstOrDefault(c => string.Equals(c.Name, name, StringComparison.OrdinalIgnoreCase) && c.Version == version); } + /// + /// A contract from its JSON, with reading the schema files it + /// names — from embedded resources, say. + /// + public static ContractDocument FromJson(string json, Func readSibling) => + new(JObject.Parse(json), readSibling ?? (file => throw new FileNotFoundException($"The contract names {file}, and nothing was given to read it."))); + /// A contract from a file, with its schemas beside it. public static ContractDocument FromFile(string path) { @@ -79,14 +94,6 @@ public static ContractDocument FromFile(string path) return new ContractDocument(JObject.Parse(File.ReadAllText(path)), file => File.ReadAllText(Path.Combine(directory, file))); } - - static string ReadResource(string name) - { - using var stream = typeof(ContractDocument).Assembly.GetManifestResourceStream(name) - ?? throw new FileNotFoundException(name); - using var reader = new StreamReader(stream); - return reader.ReadToEnd(); - } } public record ContractMethod(string Name, string Input, string Output, IReadOnlyList Examples, bool Destructive); diff --git a/SW.Serverless.Tooling/Contracts/bitween/bitween-adapter-contract.v1.json b/SW.Serverless.Tooling/Contracts/bitween/bitween-adapter-contract.v1.json deleted file mode 100644 index a56d1b4..0000000 --- a/SW.Serverless.Tooling/Contracts/bitween/bitween-adapter-contract.v1.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "contract": "bitween", - "version": 1, - "description": "What Bitween calls on each kind of adapter, and what it passes. Bitween calls adapters by method name over the SW-Serverless protocol; the names below are exact and case-sensitive.", - "encoding": { - "string": "A string argument or result is the raw UTF-8 text, not a JSON string.", - "object": "Any other argument or result is JSON, with property names exactly as its schema gives them.", - "none": "A method with no argument receives an empty payload; one with no result returns an empty payload." - }, - "types": { - "ExchangeFile": { - "schema": "exchange-file.schema.json" - }, - "ValidationResult": { - "schema": "validation-result.schema.json" - }, - "FileId": { - "encoding": "string", - "description": "An id a receiver gave in ListFiles, passed back exactly as given." - }, - "FileIdList": { - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "errors": "A call fails when the adapter raises an error; Bitween records the error's type and message on the exchange, and retry policies decide what happens next. A handler whose partner rejected the message does not fail: it returns the partner's answer with BadData true.", - "kinds": { - "handler": { - "description": "Delivers a message and returns the partner's response.", - "methods": [ - { - "name": "Handle", - "input": "ExchangeFile", - "output": "ExchangeFile", - "examples": [ - { - "Data": "{\"orderId\":\"SO-1001\",\"lines\":2}", - "Filename": "order-SO-1001.json", - "BadData": false, - "ContentType": "application/json" - } - ] - } - ] - }, - "mapper": { - "description": "Transforms a message into the shape the next step expects.", - "methods": [ - { - "name": "Handle", - "input": "ExchangeFile", - "output": "ExchangeFile", - "examples": [ - { - "Data": "{\"orderId\":\"SO-1001\",\"lines\":2}", - "Filename": "order-SO-1001.json", - "BadData": false, - "ContentType": "application/json" - } - ] - } - ] - }, - "validator": { - "description": "Checks a message before it is accepted.", - "methods": [ - { - "name": "Validate", - "input": "ExchangeFile", - "output": "ValidationResult", - "examples": [ - { - "Data": "{\"orderId\":\"SO-1001\",\"lines\":2}", - "Filename": "order-SO-1001.json", - "BadData": false, - "ContentType": "application/json" - } - ] - } - ] - }, - "receiver": { - "description": "Fetches files from a source on a schedule.", - "session": "One session per run. Calls come in this order: Initialize, ListFiles, then for each file GetFile and, once it is safely taken in, DeleteFile, and finally Finalize, which is also called after a failure.", - "methods": [ - { - "name": "Initialize", - "input": null, - "output": null - }, - { - "name": "ListFiles", - "input": null, - "output": "FileIdList" - }, - { - "name": "GetFile", - "input": "FileId", - "output": "ExchangeFile" - }, - { - "name": "DeleteFile", - "input": "FileId", - "output": null, - "destructive": true - }, - { - "name": "Finalize", - "input": null, - "output": null - } - ] - } - }, - "examples": "Example inputs for each method that takes one. serverless test calls the adapter with them, using the settings it is given, and checks each answer against the method's output type.", - "destructive": "A destructive method changes the source an adapter reads: serverless test calls it only when told it may." -} diff --git a/SW.Serverless.Tooling/Contracts/bitween/exchange-file.schema.json b/SW.Serverless.Tooling/Contracts/bitween/exchange-file.schema.json deleted file mode 100644 index 52a5fa0..0000000 --- a/SW.Serverless.Tooling/Contracts/bitween/exchange-file.schema.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://bitween.systems/contract/v1/exchange-file.schema.json", - "title": "ExchangeFile", - "description": "A file as Bitween passes it to and from adapters. Property names are exactly as shown; unknown properties are allowed and ignored.", - "type": "object", - "required": ["Data"], - "properties": { - "Filename": { "type": ["string", "null"], "description": "The file's name, when it has one." }, - "Data": { "type": "string", "description": "The content: text, or base64 for binary content." }, - "Hash": { "type": ["string", "null"], "description": "SHA-1 of Data as lower-case hex. Written by .NET adapters; always recomputed from Data when read, never trusted." }, - "BadData": { "type": "boolean", "default": false, "description": "Whether this is a bad response: a delivery the partner rejected." }, - "ContentType": { "type": ["string", "null"], "description": "The content's media type, when known." } - }, - "additionalProperties": true -} diff --git a/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/package.json b/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/package.json deleted file mode 100644 index b9a3a4a..0000000 --- a/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "@simplyworks/bitween", - "version": "10.0.59", - "description": "The Bitween adapter contract for JavaScript and TypeScript: handlers, mappers, validators and receivers.", - "main": "src/index.js", - "types": "src/index.d.ts", - "files": ["src"], - "engines": { "node": ">=22" }, - "peerDependencies": { "@simplyworks/serverless": ">=10.1.0" }, - "scripts": { "test": "node --test" }, - "license": "MIT", - "repository": { "type": "git", "url": "https://github.com/simplify9/Bitween-api", "directory": "sdk/node" } -} diff --git a/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/src/index.d.ts b/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/src/index.d.ts deleted file mode 100644 index 7adaabc..0000000 --- a/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/src/index.d.ts +++ /dev/null @@ -1,46 +0,0 @@ -export declare const CONTRACT: "bitween"; -export declare const CONTRACT_VERSION: 1; - -export declare class ExchangeFile { - constructor(init?: { data?: string; filename?: string | null; badData?: boolean; contentType?: string | null }); - /** The content: text, or base64 for binary content. */ - data: string; - filename: string | null; - /** A bad response: a delivery the partner rejected. Returned, not thrown. */ - badData: boolean; - contentType: string | null; - /** SHA-1 of data as lower-case hex. */ - readonly hash: string; - toWire(): Record; - static fromWire(value: unknown): ExchangeFile; -} - -export declare class ValidationResult { - constructor(validations?: [string, string][]); - validations: [string, string][]; - readonly success: boolean; - add(key: string, message: string): this; - toWire(): Record; -} - -type Awaitable = T | Promise; - -export declare abstract class Handler { - abstract handle(file: ExchangeFile): Awaitable; -} - -export declare abstract class Mapper { - abstract map(file: ExchangeFile): Awaitable; -} - -export declare abstract class Validator { - abstract validate(file: ExchangeFile): Awaitable | null | void>; -} - -export declare abstract class Receiver { - initialize(): Awaitable; - abstract listFiles(): Awaitable; - abstract getFile(fileId: string): Awaitable; - abstract deleteFile(fileId: string): Awaitable; - finalize(): Awaitable; -} diff --git a/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/src/index.js b/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/src/index.js deleted file mode 100644 index 35d1e8d..0000000 --- a/SW.Serverless.Tooling/Contracts/bitween/node/@simplyworks/bitween/src/index.js +++ /dev/null @@ -1,203 +0,0 @@ -"use strict"; -/** - * The Bitween adapter contract for JavaScript and TypeScript: the kinds of adapter Bitween runs, and - * what it passes. Extend a kind and implement its methods; the wire names, the encoding and the kind - * and contract declarations are taken care of. The contract itself is bitween-adapter-contract.v1.json - * in SW.Bitween.Adapters; this is its JavaScript form. - * - * const sw = require("@simplyworks/serverless"); - * const { ExchangeFile, Handler } = require("@simplyworks/bitween"); - * - * class Orders extends Handler { - * constructor() { super(); sw.expect("Url", { description: "Where orders go" }); } - * handle(file) { return new ExchangeFile({ data: file.data, filename: file.filename }); } - * } - * - * sw.run(Orders); - */ - -const crypto = require("node:crypto"); - -const CONTRACT = "bitween"; -const CONTRACT_VERSION = 1; -const VERSION = "10.0.59"; - -const EXCHANGE_FILE_SCHEMA = { - title: "ExchangeFile", type: "object", required: ["Data"], additionalProperties: true, - properties: { - Filename: { type: ["string", "null"] }, Data: { type: "string" }, Hash: { type: ["string", "null"] }, - BadData: { type: "boolean", default: false }, ContentType: { type: ["string", "null"] }, - }, -}; - -const VALIDATION_RESULT_SCHEMA = { - title: "ValidationResult", type: "object", required: ["Validations"], additionalProperties: true, - properties: { - Success: { type: "boolean" }, - Validations: { - type: "array", - items: { type: "object", required: ["Key", "Value"], properties: { Key: { type: "string" }, Value: { type: "string" } } }, - }, - }, -}; - -/** - * A file as Bitween passes it to and from adapters. `data` is the content: text, or base64 for - * binary content. `badData` marks a bad response, a delivery the partner rejected — returned, not thrown. - */ -class ExchangeFile { - constructor({ data = "", filename = null, badData = false, contentType = null } = {}) { - this.data = data; - this.filename = filename; - this.badData = badData; - this.contentType = contentType; - } - - /** SHA-1 of `data` as lower-case hex, as .NET adapters write it. */ - get hash() { - return crypto.createHash("sha1").update(this.data ?? "", "utf8").digest("hex"); - } - - toWire() { - return { Filename: this.filename, Data: this.data ?? "", Hash: this.hash, BadData: this.badData, ContentType: this.contentType }; - } - - static fromWire(value) { - if (!value || typeof value !== "object") throw new TypeError("an ExchangeFile is a JSON object"); - // Hash is recomputed, never trusted; unknown properties are ignored. - return new ExchangeFile({ - data: value.Data ?? "", filename: value.Filename ?? null, badData: Boolean(value.BadData ?? false), - contentType: value.ContentType ?? null, - }); - } -} - -/** What a validator found: each failure as a key — often the field it concerns — and a message. */ -class ValidationResult { - constructor(validations = []) { - this.validations = validations; - } - - get success() { - return this.validations.length === 0; - } - - add(key, message) { - this.validations.push([key, message]); - return this; - } - - toWire() { - return { Success: this.success, Validations: this.validations.map(([Key, Value]) => ({ Key, Value })) }; - } - - static fromWire(value) { - return new ValidationResult(((value && value.Validations) || []).map((v) => [v.Key ?? "", v.Value ?? ""])); - } -} - -function asFile(value) { - if (value instanceof ExchangeFile) return value; - if (typeof value === "string") return new ExchangeFile({ data: value }); - if (value && typeof value === "object" && "data" in value) return new ExchangeFile(value); - throw new TypeError(`expected an ExchangeFile, got ${value === null ? "null" : typeof value}`); -} - -const ABSTRACT = Symbol("abstract"); - -function abstract(name) { - const fn = function () { throw new Error(`${name} is not implemented`); }; - fn[ABSTRACT] = true; - return fn; -} - -class Kind { - static contracts = { [CONTRACT]: CONTRACT_VERSION }; - static required = []; - - /** A declared kind without its methods fails when the adapter starts, not on first use. */ - __swCheck() { - const missing = this.constructor.required.filter((m) => this[m] && this[m][ABSTRACT]); - if (missing.length) - throw new TypeError(`${this.constructor.name} is a Bitween ${this.constructor.kinds[0]} but does not implement ${missing.join(", ")}`); - } -} - -const file = EXCHANGE_FILE_SCHEMA; - -/** Delivers a message and returns the partner's response. A rejection is returned with badData, not thrown. */ -class Handler extends Kind { - static kinds = ["handler"]; - static required = ["handle"]; - static commands = { - Handle: { method: "__swHandle", input: file, output: file, description: "Delivers a message and returns the partner's response." }, - }; - - async __swHandle(value) { - return asFile(await this.handle(ExchangeFile.fromWire(value))); - } -} -Handler.prototype.handle = abstract("handle"); - -/** Transforms a message into the shape the next step expects. */ -class Mapper extends Kind { - static kinds = ["mapper"]; - static required = ["map"]; - static commands = { - Handle: { method: "__swHandle", input: file, output: file, description: "Transforms a message into the shape the next step expects." }, - }; - - async __swHandle(value) { - return asFile(await this.map(ExchangeFile.fromWire(value))); - } -} -Mapper.prototype.map = abstract("map"); - -/** Checks a message before it is accepted. */ -class Validator extends Kind { - static kinds = ["validator"]; - static required = ["validate"]; - static commands = { - Validate: { method: "__swValidate", input: file, output: VALIDATION_RESULT_SCHEMA, description: "Checks a message before it is accepted." }, - }; - - async __swValidate(value) { - const result = await this.validate(ExchangeFile.fromWire(value)); - if (result === undefined || result === null) return new ValidationResult(); - if (result instanceof ValidationResult) return result; - // An array of [key, message] pairs, or an object of key -> message, reads naturally too. - return new ValidationResult(Array.isArray(result) ? result : Object.entries(result)); - } -} -Validator.prototype.validate = abstract("validate"); - -/** - * Fetches files from a source on a schedule. One session per run: initialize, listFiles, then for - * each file getFile and — once it is safely taken in — deleteFile, and finally finalize, which is - * also called after a failure. - */ -class Receiver extends Kind { - static kinds = ["receiver"]; - static required = ["listFiles", "getFile", "deleteFile"]; - static commands = { - Initialize: { method: "__swInitialize", description: "Starts a run." }, - ListFiles: { method: "__swListFiles", output: { type: "array", items: { type: "string" } }, description: "The ids of the files waiting." }, - GetFile: { method: "__swGetFile", input: "string", output: file, description: "One file, by an id ListFiles gave." }, - DeleteFile: { method: "__swDeleteFile", input: "string", description: "Removes a file from the source once it is safely taken in." }, - Finalize: { method: "__swFinalize", description: "Ends a run, after a failure too." }, - }; - - initialize() {} - finalize() {} - - async __swInitialize() { await this.initialize(); } - async __swListFiles() { return ((await this.listFiles()) || []).map(String); } - async __swGetFile(fileId) { return asFile(await this.getFile(fileId)); } - async __swDeleteFile(fileId) { await this.deleteFile(fileId); } - async __swFinalize() { await this.finalize(); } -} -Receiver.prototype.listFiles = abstract("listFiles"); -Receiver.prototype.getFile = abstract("getFile"); -Receiver.prototype.deleteFile = abstract("deleteFile"); - -module.exports = { CONTRACT, CONTRACT_VERSION, VERSION, ExchangeFile, Handler, Mapper, Receiver, ValidationResult, Validator }; diff --git a/SW.Serverless.Tooling/Contracts/bitween/python/simplyworks_bitween/__init__.py b/SW.Serverless.Tooling/Contracts/bitween/python/simplyworks_bitween/__init__.py deleted file mode 100644 index 43a32fc..0000000 --- a/SW.Serverless.Tooling/Contracts/bitween/python/simplyworks_bitween/__init__.py +++ /dev/null @@ -1,236 +0,0 @@ -"""The Bitween adapter contract for Python: the kinds of adapter Bitween runs, and what it passes. - -Subclass a kind and implement its methods; the wire names, the encoding and the kind and contract -declarations are taken care of. The contract itself is ``bitween-adapter-contract.v1.json`` in -SW.Bitween.Adapters; this is its Python form, as ``SimplyWorks.Bitween.Adapters`` is its .NET one. - - import simplyworks_serverless as sw -from simplyworks_serverless._runner import _call - from simplyworks_bitween import ExchangeFile, Handler - - class Orders(Handler): - def __init__(self): - sw.expect("Url", description="Where orders go") - - def handle(self, file: ExchangeFile) -> ExchangeFile: - ... - - if __name__ == "__main__": - sw.run(Orders) -""" - -import hashlib -from dataclasses import dataclass, field - -import simplyworks_serverless as sw -from simplyworks_serverless._runner import _call - -CONTRACT = "bitween" -CONTRACT_VERSION = 1 - -__version__ = "10.0.59" - - -@dataclass -class ExchangeFile: - """A file as Bitween passes it to and from adapters. - - ``data`` is the content: text, or base64 for binary content. ``bad_data`` marks a bad response, - a delivery the partner rejected — returned, not raised. - """ - - data: str = "" - filename: str | None = None - bad_data: bool = False - content_type: str | None = None - - @property - def hash(self): - """SHA-1 of ``data`` as lower-case hex, as .NET adapters write it.""" - return hashlib.sha1((self.data or "").encode("utf-8")).hexdigest() - - def to_wire(self): - return {"Filename": self.filename, "Data": self.data or "", "Hash": self.hash, - "BadData": self.bad_data, "ContentType": self.content_type} - - @classmethod - def from_wire(cls, value): - if not isinstance(value, dict): - raise ValueError("an ExchangeFile is a JSON object") - # Hash is recomputed, never trusted; unknown properties are ignored. - return cls(data=value.get("Data") or "", filename=value.get("Filename"), - bad_data=bool(value.get("BadData", False)), content_type=value.get("ContentType")) - - @staticmethod - def __sw_schema__(): - return { - "title": "ExchangeFile", "type": "object", "required": ["Data"], "additionalProperties": True, - "properties": { - "Filename": {"type": ["string", "null"]}, "Data": {"type": "string"}, - "Hash": {"type": ["string", "null"]}, "BadData": {"type": "boolean", "default": False}, - "ContentType": {"type": ["string", "null"]}, - }, - } - - -@dataclass -class ValidationResult: - """What a validator found: each failure as a key — often the field it concerns — and a message. - No failures means valid.""" - - validations: list[tuple[str, str]] = field(default_factory=list) - - @property - def success(self): - return not self.validations - - def add(self, key, message): - self.validations.append((key, message)) - return self - - def to_wire(self): - return {"Success": self.success, "Validations": [{"Key": k, "Value": v} for k, v in self.validations]} - - @classmethod - def from_wire(cls, value): - return cls([(v.get("Key", ""), v.get("Value", "")) for v in (value or {}).get("Validations") or []]) - - @staticmethod - def __sw_schema__(): - return { - "title": "ValidationResult", "type": "object", "required": ["Validations"], "additionalProperties": True, - "properties": { - "Success": {"type": "boolean"}, - "Validations": {"type": "array", "items": { - "type": "object", "required": ["Key", "Value"], - "properties": {"Key": {"type": "string"}, "Value": {"type": "string"}}}}, - }, - } - - -def _as_file(value): - if isinstance(value, ExchangeFile): - return value - if isinstance(value, str): - return ExchangeFile(data=value) - raise TypeError(f"expected an ExchangeFile, got {type(value).__name__}") - - -class _Kind: - __sw_contracts__ = {CONTRACT: CONTRACT_VERSION} - _required = () - - def __sw_check__(self): - # A declared kind without its methods fails when the adapter starts, not on first use. - missing = [m for m in self._required if getattr(getattr(type(self), m), "__sw_abstract__", False)] - if missing: - raise TypeError(f"{type(self).__name__} is a Bitween {self.__sw_kinds__[0]} " - f"but does not implement {', '.join(missing)}") - - -def _abstract(fn): - fn.__sw_abstract__ = True - return fn - - -class Handler(_Kind): - """Delivers a message and returns the partner's response. A rejected delivery is returned with - ``bad_data=True``, not raised.""" - - __sw_kinds__ = ["handler"] - _required = ("handle",) - - @_abstract - def handle(self, file: ExchangeFile) -> ExchangeFile: - raise NotImplementedError - - @sw.command("Handle", description="Delivers a message and returns the partner's response.") - async def _sw_handle(self, file: ExchangeFile) -> ExchangeFile: - return _as_file(await _call(self.handle, file)) - - -class Mapper(_Kind): - """Transforms a message into the shape the next step expects.""" - - __sw_kinds__ = ["mapper"] - _required = ("map",) - - @_abstract - def map(self, file: ExchangeFile) -> ExchangeFile: - raise NotImplementedError - - @sw.command("Handle", description="Transforms a message into the shape the next step expects.") - async def _sw_handle(self, file: ExchangeFile) -> ExchangeFile: - return _as_file(await _call(self.map, file)) - - -class Validator(_Kind): - """Checks a message before it is accepted.""" - - __sw_kinds__ = ["validator"] - _required = ("validate",) - - @_abstract - def validate(self, file: ExchangeFile) -> ValidationResult: - raise NotImplementedError - - @sw.command("Validate", description="Checks a message before it is accepted.") - async def _sw_validate(self, file: ExchangeFile) -> ValidationResult: - result = await _call(self.validate, file) - if result is None: - return ValidationResult() - if isinstance(result, ValidationResult): - return result - # A list of (key, message) pairs, or a dict of key -> message, reads naturally too. - return ValidationResult(list(result.items()) if isinstance(result, dict) else [tuple(r) for r in result]) - - -class Receiver(_Kind): - """Fetches files from a source on a schedule. One session per run: ``initialize``, ``list_files``, - then for each file ``get_file`` and — once it is safely taken in — ``delete_file``, and finally - ``finalize``, which is also called after a failure.""" - - __sw_kinds__ = ["receiver"] - _required = ("list_files", "get_file", "delete_file") - - def initialize(self): - pass - - @_abstract - def list_files(self) -> list[str]: - raise NotImplementedError - - @_abstract - def get_file(self, file_id: str) -> ExchangeFile: - raise NotImplementedError - - @_abstract - def delete_file(self, file_id: str) -> None: - raise NotImplementedError - - def finalize(self): - pass - - @sw.command("Initialize", description="Starts a run.") - async def _sw_initialize(self) -> None: - await _call(self.initialize) - - @sw.command("ListFiles", description="The ids of the files waiting.") - async def _sw_list_files(self) -> list[str]: - return [str(f) for f in (await _call(self.list_files) or [])] - - @sw.command("GetFile", description="One file, by an id ListFiles gave.") - async def _sw_get_file(self, file_id: str) -> ExchangeFile: - return _as_file(await _call(self.get_file, file_id)) - - @sw.command("DeleteFile", description="Removes a file from the source once it is safely taken in.") - async def _sw_delete_file(self, file_id: str) -> None: - await _call(self.delete_file, file_id) - - @sw.command("Finalize", description="Ends a run, after a failure too.") - async def _sw_finalize(self) -> None: - await _call(self.finalize) - - -__all__ = ["CONTRACT", "CONTRACT_VERSION", "ExchangeFile", "Handler", "Mapper", "Receiver", "ValidationResult", - "Validator"] diff --git a/SW.Serverless.Tooling/Contracts/bitween/validation-result.schema.json b/SW.Serverless.Tooling/Contracts/bitween/validation-result.schema.json deleted file mode 100644 index 97d1a83..0000000 --- a/SW.Serverless.Tooling/Contracts/bitween/validation-result.schema.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://bitween.systems/contract/v1/validation-result.schema.json", - "title": "ValidationResult", - "description": "What a validator found. Property names are exactly as shown; unknown properties are allowed and ignored.", - "type": "object", - "required": ["Validations"], - "properties": { - "Success": { "type": "boolean", "description": "True when Validations is empty. Bitween derives it from Validations." }, - "Validations": { - "type": "array", - "description": "Each failure: a code, often the field it concerns, and a message.", - "items": { - "type": "object", - "required": ["Key", "Value"], - "properties": { - "Key": { "type": "string" }, - "Value": { "type": "string" } - } - } - } - }, - "additionalProperties": true -} diff --git a/SW.Serverless.Tooling/LocalAdapterHost.cs b/SW.Serverless.Tooling/LocalAdapterHost.cs index 511ff3e..90cbb34 100644 --- a/SW.Serverless.Tooling/LocalAdapterHost.cs +++ b/SW.Serverless.Tooling/LocalAdapterHost.cs @@ -21,7 +21,7 @@ namespace SW.Serverless.Tooling /// /// An adapter package run on this machine exactly as a host runs one: put in a temporary store, /// installed from it, started on its runtime, classic or resident, and called by command name. - /// What serverless run and serverless test use. + /// What sw-serverless run and sw-serverless test use. /// public sealed class LocalAdapterHost : IAsyncDisposable { diff --git a/SW.Serverless.Tooling/PackagePublisher.cs b/SW.Serverless.Tooling/PackagePublisher.cs index d89f1c0..571b22d 100644 --- a/SW.Serverless.Tooling/PackagePublisher.cs +++ b/SW.Serverless.Tooling/PackagePublisher.cs @@ -32,7 +32,7 @@ public class PublishRequest public string PublishedBy { get; set; } } - /// A package serverless build made — in any language — to publish as it is. + /// A package sw-serverless build made — in any language — to publish as it is. public class PublishPackageRequest { public string PackagePath { get; set; } @@ -161,7 +161,7 @@ await repository.PublishVersionAsync(adapterId, version, zipPath, package, reque } /// - /// Publishes a package serverless build made: its manifest is already complete, so this only + /// Publishes a package sw-serverless build made: its manifest is already complete, so this only /// settles the version, stamps it into the manifest and uploads. Always versioned; an adapter /// in another runtime goes only where hosts that can run it look. /// @@ -182,7 +182,7 @@ public static async Task PublishPackageAsync(ICloudFilesService f System.IO.Compression.ZipFile.ExtractToDirectory(request.PackagePath, folder); var manifestPath = Path.Combine(folder, AdapterManifest.FileName); if (!File.Exists(manifestPath)) - throw new SWException($"{request.PackagePath} has no {AdapterManifest.FileName}; build it with serverless build."); + throw new SWException($"{request.PackagePath} has no {AdapterManifest.FileName}; build it with sw-serverless build."); var manifest = AdapterManifest.Parse(await File.ReadAllTextAsync(manifestPath)); var adapterId = manifest.Id?.ToLowerInvariant(); diff --git a/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj b/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj index 87e368d..1665497 100644 --- a/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj +++ b/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj @@ -1,7 +1,7 @@ net10.0 @@ -25,29 +25,19 @@ - + - - - - - - - - - - - - + + + + diff --git a/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs b/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs index 8d2ea7b..a0b3a64 100644 --- a/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs +++ b/SW.Serverless.Tooling/Scaffolding/Scaffolder.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text.Json.Nodes; using System.Text.RegularExpressions; namespace SW.Serverless.Tooling.Scaffolding @@ -14,10 +15,14 @@ public class ScaffoldRequest /// Its id in storage; derived from the name when not given. public string Id { get; set; } + /// dotnet, python, node (JavaScript) or typescript. public string Language { get; set; } = "dotnet"; - /// handler, mapper, validator or receiver. - public string Kind { get; set; } = "handler"; + /// + /// What a template set means by it — an application's own kinds, say. The templates here have + /// none, and ignore it. + /// + public string Kind { get; set; } /// The folder the project folder is made in. public string ParentDirectory { get; set; } = "."; @@ -32,39 +37,42 @@ public class ScaffoldResult } /// - /// What serverless init writes: a working adapter of one kind for the Bitween contract, ready to - /// build, test and publish. Also what a code editor starts a new adapter from. + /// What sw-serverless init writes: a working adapter with two settings and two commands, in any of + /// the SDK's languages, ready to build, test, run and publish. An application with adapters of + /// its own — kinds, a contract — passes its own templates to + /// and keeps the same checks and layout. /// public static class Scaffolder { - /// The SDK the templates reference; the first with --describe and the Bitween contract types. + /// The .NET SDK the templates reference. public const string SdkPackageVersion = "10.1.0"; - /// - /// The Bitween contract package the templates reference: the first Bitween release that - /// publishes it. NuGet reads it as a minimum, so it resolves to the first one there is. - /// - public const string BitweenAdaptersPackageVersion = "10.0.59"; + /// The Python SDK the templates name; the build vendors the copy it carries. + public const string PythonSdkVersion = "10.2.0"; + + /// The Node SDK the templates name; the build vendors the copy it carries. + public const string NodeSdkVersion = "10.2.0"; - public static readonly IReadOnlyList Kinds = new[] { "handler", "mapper", "validator", "receiver" }; public static readonly IReadOnlyList Languages = new[] { "dotnet", "python", "node", "typescript" }; - /// The Python SDK the templates name; serverless build vendors the copy it carries. - public const string PythonSdkVersion = "10.1.0"; + /// A template set: the files of a new adapter, by path, for its name, id, language and kind. + public delegate IEnumerable<(string File, string Content)> Templates(string name, string id, string language, string kind); - /// The Node SDK the templates name; serverless build vendors the copy it carries. - public const string NodeSdkVersion = "10.1.0"; + /// Writes the generic adapter. + public static ScaffoldResult Scaffold(ScaffoldRequest request) => Scaffold(request, Generic); - public static ScaffoldResult Scaffold(ScaffoldRequest request) + /// + /// Writes a new adapter from , after the checks every one needs: + /// a usable name and id, a language the SDK has, and an empty folder to write in. + /// + public static ScaffoldResult Scaffold(ScaffoldRequest request, Templates templates) { var result = new ScaffoldResult(); var name = request.Name?.Trim(); if (string.IsNullOrEmpty(name) || !Regex.IsMatch(name, "^[A-Za-z][A-Za-z0-9_]*$")) result.Problems.Add($"'{request.Name}' isn't a name a project and a class can take: letters, digits and _, starting with a letter"); - if (!Kinds.Contains(request.Kind)) - result.Problems.Add($"'{request.Kind}' isn't a kind: use {string.Join(", ", Kinds)}"); if (!Languages.Contains(request.Language)) - result.Problems.Add($"a '{request.Language}' template arrives with that language's SDK; this release scaffolds {string.Join(", ", Languages)}"); + result.Problems.Add($"'{request.Language}' isn't a language the SDK has: use {string.Join(", ", Languages)}"); var id = string.IsNullOrWhiteSpace(request.Id) ? IdFrom(name ?? "") : request.Id.Trim(); if (!InstallerLogic.IsValidAdapterId(id)) @@ -75,18 +83,13 @@ public static ScaffoldResult Scaffold(ScaffoldRequest request) result.Problems.Add($"{directory} already exists and isn't empty"); if (!result.Succeeded) return result; + var files = templates(name, id, request.Language, request.Kind).ToList(); Directory.CreateDirectory(directory); result.ProjectDirectory = directory; - var files = request.Language switch - { - "python" => PythonFiles(name, id, request.Kind), - "node" => NodeFiles(name, id, request.Kind, typeScript: false), - "typescript" => NodeFiles(name, id, request.Kind, typeScript: true), - _ => DotnetFiles(name, id, request.Kind), - }; foreach (var (file, content) in files) { var path = Path.Combine(directory, file); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); // Ending in a newline, as text files should: a line appended later stays its own line. var text = content.Replace("\r\n", "\n"); File.WriteAllText(path, text.EndsWith('\n') ? text : text + "\n"); @@ -99,474 +102,248 @@ public static ScaffoldResult Scaffold(ScaffoldRequest request) public static string IdFrom(string name) => Regex.Replace(name, "(?<=[a-z0-9])(?=[A-Z])", ".").Replace('_', '.').ToLowerInvariant(); - static IEnumerable<(string File, string Content)> DotnetFiles(string name, string id, string kind) - { - yield return ($"{name}.csproj", $$""" - - - - Exe - net10.0 - enable - enable - - true - - - - - - - - - """); - - yield return ("adapter.json", $$""" - { - "id": "{{id}}", - "version": "0.1.0", - "displayName": "{{Spaced(name)}}", - "summary": "What this {{kind}} does, in one sentence, for the adapter list.", - "lifecycle": "classic" - } - """); - - yield return ("Program.cs", Program(name, kind)); + /// AcmeOrders → Acme Orders + public static string Spaced(string name) => Regex.Replace(name, "(?<=[a-z0-9])(?=[A-Z])", " "); - yield return ("settings.example.json", """ - { - "BaseUrl": "https://partner.example.test", - "ApiKey": "put a test key here, and keep this file out of version control once it holds one" - } - """); - - yield return (".gitignore", """ - bin/ - obj/ - settings.json - """); - - yield return ("README.md", $$""" - # {{Spaced(name)}} - - A Bitween {{kind}} adapter. - - ```sh - serverless build # builds bin/serverless/{{id}}-0.1.0.zip - cp settings.example.json settings.json # then fill in real values - serverless test --settings settings.json # checks it against the Bitween contract - serverless publish bin/serverless/{{id}}-0.1.0.zip # with your storage flags - ``` - - Settings are declared in code with `Runner.Expect`; `serverless build` writes them into - the manifest Bitween reads. - """); - } - - static string Program(string name, string kind) => kind switch + /// The package.json a Node adapter starts with, naming the SDK and any . + public static string NodePackageJson(string id, bool typeScript, IDictionary dependencies = null) { - "receiver" => $$""" - using SW.Bitween.Adapters; - using SW.Serverless.Sdk; - - namespace {{name}}; - - /// Fetches files on a schedule. Bitween calls Initialize, ListFiles, then GetFile and DeleteFile for each file, then Finalize. - [AdapterKind("receiver")] - [AdapterContract("bitween", 1)] - public class Receiver : IBitweenReceiver - { - public Receiver() - { - // Declare settings here; read them in the methods, never in the constructor. - Runner.Expect("BaseUrl", "https://partner.example.test", description: "Where files are fetched from."); - Runner.Expect("ApiKey", isPrivate: true, description: "The partner's key."); - } - - public Task Initialize() => Task.CompletedTask; - - public Task> ListFiles() => - Task.FromResult>(new[] { "example-1" }); - - public Task GetFile(string fileId) => - Task.FromResult(new ExchangeFile("{\"id\":\"" + fileId + "\"}", fileId + ".json")); - - public Task DeleteFile(string fileId) => Task.CompletedTask; - - public Task Finalize() => Task.CompletedTask; - } - - static class Program - { - static Task Main() => Runner.Run(new Receiver()); - } - """, - "validator" => $$""" - using SW.Bitween.Adapters; - using SW.Serverless.Sdk; - - namespace {{name}}; - - /// Checks a message before Bitween accepts it. - [AdapterKind("validator")] - [AdapterContract("bitween", 1)] - public class Validator : IBitweenValidator - { - public Validator() - { - Runner.Expect("MaxBytes", "1000000", description: "The largest message accepted."); - } + var package = new JsonObject { ["name"] = id, ["private"] = true }; + if (typeScript) package["type"] = "module"; + package["engines"] = new JsonObject { ["node"] = ">=22" }; + var deps = new JsonObject { ["@simplyworks/sw-serverless"] = NodeSdkVersion }; + foreach (var (name, version) in dependencies ?? new Dictionary()) deps[name] = version; + package["dependencies"] = deps; + return package.ToJsonString(new System.Text.Json.JsonSerializerOptions { WriteIndented = true }); + } - public Task Validate(ExchangeFile file) - { - var result = new ValidationResult(); - if (file.Data.Length > Runner.StartupValueOf("MaxBytes")) - result.AddError("Data", "The message is larger than allowed."); - return Task.FromResult(result); - } - } + /// The tsconfig.json a TypeScript adapter starts with: what Node can strip, checked by tsc --noEmit. + public const string TsConfig = """ + { + // For your editor and tsc --noEmit. The build doesn't compile: Node strips the types, so + // only syntax that strips cleanly is allowed (no enums or namespaces). + "compilerOptions": { + "target": "es2023", + "module": "nodenext", + "moduleResolution": "nodenext", + "strict": true, + "noEmit": true, + "erasableSyntaxOnly": true, + "verbatimModuleSyntax": true, + "allowImportingTsExtensions": true + } + } + """; - static class Program - { - static Task Main() => Runner.Run(new Validator()); - } - """, - var handlerOrMapper => $$""" - using SW.Bitween.Adapters; - using SW.Serverless.Sdk; + /// What a template's README says about the language, after "An adapter". + public static string InLanguage(string language) => language switch + { + "python" => " in Python 3.12 or later", + "typescript" => " in TypeScript, for Node 22 or later", + "node" => " in JavaScript, for Node 22 or later", + _ => " in .NET", + }; - namespace {{name}}; + // ------------------------------------------------------------------ the generic adapter - /// {{(handlerOrMapper == "mapper" ? "Maps a message into the shape the next step expects." : "Delivers a message and returns the partner's response.")}} - [AdapterKind("{{handlerOrMapper}}")] - [AdapterContract("bitween", 1)] - public class {{(handlerOrMapper == "mapper" ? "Mapper : IBitweenMapper" : "Handler : IBitweenHandler")}} - { - public {{(handlerOrMapper == "mapper" ? "Mapper" : "Handler")}}() + static IEnumerable<(string File, string Content)> Generic(string name, string id, string language, string kind) + { + var entry = language switch { "python" => "main.py", "node" => "main.js", "typescript" => "main.ts", _ => null }; + var runtime = language switch { "python" => "python", "node" or "typescript" => "node", _ => null }; + yield return ("adapter.json", runtime == null + ? $$""" { - // Declare settings here; read them in the methods, never in the constructor. - Runner.Expect("BaseUrl", "https://partner.example.test", description: "Where messages go."); - Runner.Expect("ApiKey", isPrivate: true, description: "The partner's key."); + "id": "{{id}}", + "version": "0.1.0", + "displayName": "{{Spaced(name)}}", + "summary": "What this adapter does, in one sentence.", + "lifecycle": "classic" } - - public Task Handle(ExchangeFile file) + """ + : $$""" { - // {{(handlerOrMapper == "mapper" ? "Return the message in its new shape." : "Send file.Data to the partner. A rejection is returned with BadData set, not thrown.")}} - return Task.FromResult(new ExchangeFile(file.Data, file.Filename)); + "id": "{{id}}", + "version": "0.1.0", + "displayName": "{{Spaced(name)}}", + "summary": "What this adapter does, in one sentence.", + "runtime": "{{runtime}}", + "entry": "{{entry}}" } - } - - static class Program - { - static Task Main() => Runner.Run(new {{(handlerOrMapper == "mapper" ? "Mapper" : "Handler")}}()); - } - """, - }; - - static IEnumerable<(string File, string Content)> PythonFiles(string name, string id, string kind) - { - yield return ("adapter.json", $$""" - { - "id": "{{id}}", - "version": "0.1.0", - "displayName": "{{Spaced(name)}}", - "summary": "What this {{kind}} does, in one sentence, for the adapter list.", - "runtime": "python", - "entry": "main.py" - } - """); - - yield return ("main.py", PythonMain(name, kind)); - - yield return ("requirements.txt", $$""" - # What the adapter imports, pinned, one per line; serverless build vendors them into the - # package. The two SDKs are vendored by serverless build itself: they're listed here for - # your editor and for running the tests outside a build. - simplyworks-serverless=={{PythonSdkVersion}} - simplyworks-bitween>={{BitweenAdaptersPackageVersion}} - """); - - yield return ("settings.example.json", """ - { - "BaseUrl": "https://partner.example.test", - "ApiKey": "put a test key here, and keep this file out of version control once it holds one" - } - """); - - yield return (".gitignore", """ - __pycache__/ - .venv/ - bin/ - settings.json - """); - - yield return ("README.md", $$""" - # {{Spaced(name)}} - - A Bitween {{kind}} adapter in Python (3.12 or later). - - ```sh - serverless build # builds bin/serverless/{{id}}-0.1.0.zip - cp settings.example.json settings.json # then fill in real values - serverless test --settings settings.json # checks it against the Bitween contract - serverless publish bin/serverless/{{id}}-0.1.0.zip # with your storage flags - ``` - - Settings are declared in code with `sw.expect`; `serverless build` writes them into the - manifest Bitween reads. Dependencies go in `requirements.txt`, pinned. - """); - } - - static string PythonMain(string name, string kind) => kind switch - { - "receiver" => $$"""" - import simplyworks_serverless as sw - from simplyworks_bitween import ExchangeFile, Receiver - - - class {{name}}(Receiver): - """Fetches files on a schedule. Bitween calls initialize, list_files, then get_file and - delete_file for each file, then finalize.""" - - def __init__(self): - # Declare settings here; read them in the methods with sw.value_of. - sw.expect("BaseUrl", "https://partner.example.test", description="Where files are fetched from.") - sw.expect("ApiKey", secret=True, description="The partner's key.") - - def list_files(self) -> list[str]: - return ["example-1"] - - def get_file(self, file_id: str) -> ExchangeFile: - return ExchangeFile(data='{"id": "%s"}' % file_id, filename=file_id + ".json") - - def delete_file(self, file_id: str) -> None: - pass - - - if __name__ == "__main__": - sw.run({{name}}) - """", - "validator" => $$"""" - import simplyworks_serverless as sw - from simplyworks_bitween import ExchangeFile, ValidationResult, Validator - - - class {{name}}(Validator): - """Checks a message before Bitween accepts it.""" - - def __init__(self): - sw.expect("MaxBytes", "1000000", type="number", description="The largest message accepted.") - - def validate(self, file: ExchangeFile) -> ValidationResult: - result = ValidationResult() - if len(file.data) > int(sw.value_of("MaxBytes")): - result.add("Data", "The message is larger than allowed.") - return result - - - if __name__ == "__main__": - sw.run({{name}}) - """", - "mapper" => $$"""" - import simplyworks_serverless as sw - from simplyworks_bitween import ExchangeFile, Mapper - - - class {{name}}(Mapper): - """Maps a message into the shape the next step expects.""" - - def map(self, file: ExchangeFile) -> ExchangeFile: - # Return the message in its new shape. - return ExchangeFile(data=file.data, filename=file.filename) - - - if __name__ == "__main__": - sw.run({{name}}) - """", - _ => $$"""" - import simplyworks_serverless as sw - from simplyworks_bitween import ExchangeFile, Handler - - - class {{name}}(Handler): - """Delivers a message and returns the partner's response.""" - - def __init__(self): - # Declare settings here; read them in the methods with sw.value_of. - sw.expect("BaseUrl", "https://partner.example.test", description="Where messages go.") - sw.expect("ApiKey", secret=True, description="The partner's key.") - - def handle(self, file: ExchangeFile) -> ExchangeFile: - # Send file.data to the partner. A rejection is returned with bad_data=True, not raised. - return ExchangeFile(data=file.data, filename=file.filename) - - - if __name__ == "__main__": - sw.run({{name}}) - """", - }; - - static IEnumerable<(string File, string Content)> NodeFiles(string name, string id, string kind, bool typeScript) - { - var entry = typeScript ? "main.ts" : "main.js"; - yield return ("adapter.json", $$""" - { - "id": "{{id}}", - "version": "0.1.0", - "displayName": "{{Spaced(name)}}", - "summary": "What this {{kind}} does, in one sentence, for the adapter list.", - "runtime": "node", - "entry": "{{entry}}" - } - """); - - yield return (entry, NodeMain(name, kind, typeScript)); + """); - // The SDKs are named for the editor and for running outside a build; serverless build - // vendors the copies it carries. Other dependencies go here too, and are installed into - // the package. - var package = new System.Text.Json.Nodes.JsonObject { ["name"] = id, ["private"] = true }; - if (typeScript) package["type"] = "module"; - package["engines"] = new System.Text.Json.Nodes.JsonObject { ["node"] = ">=22" }; - package["dependencies"] = new System.Text.Json.Nodes.JsonObject + switch (language) { - ["@simplyworks/serverless"] = NodeSdkVersion, - ["@simplyworks/bitween"] = ">=" + BitweenAdaptersPackageVersion, - }; - yield return ("package.json", package.ToJsonString(new System.Text.Json.JsonSerializerOptions { WriteIndented = true })); - - if (typeScript) - yield return ("tsconfig.json", """ - { - // For your editor and tsc --noEmit. serverless build doesn't compile: Node strips the - // types, so only syntax that strips cleanly is allowed (no enums or namespaces). - "compilerOptions": { - "target": "es2023", - "module": "nodenext", - "moduleResolution": "nodenext", - "strict": true, - "noEmit": true, - "erasableSyntaxOnly": true, - "verbatimModuleSyntax": true, - "allowImportingTsExtensions": true - } - } - """); + case "python": + yield return ("main.py", $$"""" + import sw_serverless as sw + + + class {{name}}: + """Greets whoever it is asked to, the way its settings say.""" + + def __init__(self): + # Declare settings here; read them in the commands with sw.value_of. + sw.expect("Greeting", "Hello", description="What to say before the name.") + sw.expect("ApiKey", secret=True, required=False, description="A key, to show how a secret is declared.") + + @sw.command("Greet", description="Greets someone by name.") + def greet(self, name: str) -> str: + return f"{sw.value_of('Greeting')}, {name}!" + + @sw.command("Count", description="Counts the words in a text.") + def count(self, text: str) -> int: + return len(text.split()) + + + if __name__ == "__main__": + sw.run({{name}}) + """"); + yield return ("requirements.txt", $$""" + # What the adapter imports, pinned, one per line; sw-serverless build vendors them into + # the package. The SDK is vendored by the build itself; it's listed for your editor. + sw-serverless=={{PythonSdkVersion}} + """); + yield return (".gitignore", "__pycache__/\n.venv/\nbin/\nsettings.json\n"); + break; + + case "node": + case "typescript": + var ts = language == "typescript"; + yield return (entry, ts + ? $$""" + import { expect, run, valueOf } from "@simplyworks/sw-serverless"; + + /** Greets whoever it is asked to, the way its settings say. */ + class {{name}} { + static commands = { + Greet: { method: "greet", input: "string", output: "string", description: "Greets someone by name." }, + Count: { method: "count", input: "string", output: "json", description: "Counts the words in a text." }, + }; + + constructor() { + // Declare settings here; read them in the commands with valueOf. + expect("Greeting", { default: "Hello", description: "What to say before the name." }); + expect("ApiKey", { secret: true, required: false, description: "A key, to show how a secret is declared." }); + } + + greet(name: string): string { + return `${valueOf("Greeting")}, ${name}!`; + } + + count(text: string): number { + return text.split(/\s+/).filter(Boolean).length; + } + } + + run({{name}}); + """ + : $$""" + const { expect, run, valueOf } = require("@simplyworks/sw-serverless"); + + /** Greets whoever it is asked to, the way its settings say. */ + class {{name}} { + static commands = { + Greet: { method: "greet", input: "string", output: "string", description: "Greets someone by name." }, + Count: { method: "count", input: "string", output: "json", description: "Counts the words in a text." }, + }; + + constructor() { + // Declare settings here; read them in the commands with valueOf. + expect("Greeting", { default: "Hello", description: "What to say before the name." }); + expect("ApiKey", { secret: true, required: false, description: "A key, to show how a secret is declared." }); + } + + greet(name) { + return `${valueOf("Greeting")}, ${name}!`; + } + + count(text) { + return text.split(/\s+/).filter(Boolean).length; + } + } + + run({{name}}); + """); + yield return ("package.json", NodePackageJson(id, ts)); + if (ts) yield return ("tsconfig.json", TsConfig); + yield return (".gitignore", "node_modules/\nbin/\nsettings.json\n"); + break; + + default: + yield return ($"{name}.csproj", $$""" + + + + Exe + net10.0 + enable + enable + + true + + + + + + + + """); + yield return ("Program.cs", $$""" + using SW.Serverless.Sdk; + + namespace {{name}}; + + /// Greets whoever it is asked to, the way its settings say. + public class Adapter + { + public Adapter() + { + // Declare settings here; read them in the commands, never in the constructor. + Runner.Expect("Greeting", "Hello", description: "What to say before the name."); + Runner.Expect("ApiKey", optional: true, isPrivate: true, description: "A key, to show how a secret is declared."); + } + + [AdapterCommand(Description = "Greets someone by name.")] + public Task Greet(string name) => + Task.FromResult($"{Runner.StartupValueOf("Greeting")}, {name}!"); + + [AdapterCommand(Description = "Counts the words in a text.")] + public Task Count(string text) => + Task.FromResult(text.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries).Length); + } + + static class Program + { + static Task Main() => Runner.Run(new Adapter()); + } + """); + yield return (".gitignore", "bin/\nobj/\nsettings.json\n"); + break; + } yield return ("settings.example.json", """ { - "BaseUrl": "https://partner.example.test", + "Greeting": "Hello", "ApiKey": "put a test key here, and keep this file out of version control once it holds one" } """); - yield return (".gitignore", """ - node_modules/ - bin/ - settings.json - """); - yield return ("README.md", $$""" # {{Spaced(name)}} - A Bitween {{kind}} adapter in {{(typeScript ? "TypeScript" : "JavaScript")}}, for Node 22 or later. + An SW-Serverless adapter{{InLanguage(language)}}. ```sh - serverless build # builds bin/serverless/{{id}}-0.1.0.zip - cp settings.example.json settings.json # then fill in real values - serverless test --settings settings.json # checks it against the Bitween contract - serverless publish bin/serverless/{{id}}-0.1.0.zip # with your storage flags + sw-serverless build # builds bin/serverless/{{id}}-0.1.0.zip + cp settings.example.json settings.json # then fill in real values + sw-serverless test --settings settings.json # starts it as a host would and checks it + sw-serverless run --settings settings.json --call Greet --input Ada + sw-serverless publish bin/serverless/{{id}}-0.1.0.zip # with your storage flags ``` - Settings are declared in code with `expect`; `serverless build` writes them into the - manifest Bitween reads.{{(typeScript ? " The build strips the types with Node itself, so no compiler is needed; `tsc --noEmit` checks them." : "")}} + Settings are declared in code; `sw-serverless build` writes them into the package's + manifest, so a host knows what to ask for without starting the adapter. """); } - - static string NodeMain(string name, string kind, bool ts) - { - var header = ts - ? $$""" - import { expect, run, valueOf } from "@simplyworks/serverless"; - import { ExchangeFile, {{Pascal(kind)}}{{(kind == "validator" ? ", ValidationResult" : "")}} } from "@simplyworks/bitween"; - """ - : $$""" - const { expect, run, valueOf } = require("@simplyworks/serverless"); - const { ExchangeFile, {{Pascal(kind)}}{{(kind == "validator" ? ", ValidationResult" : "")}} } = require("@simplyworks/bitween"); - """; - string T(string type) => ts ? type : ""; - var body = kind switch - { - "receiver" => $$""" - /** Fetches files on a schedule. Bitween calls initialize, listFiles, then getFile and deleteFile for each file, then finalize. */ - class {{name}} extends Receiver { - constructor() { - super(); - // Declare settings here; read them in the methods with valueOf. - expect("BaseUrl", { default: "https://partner.example.test", description: "Where files are fetched from." }); - expect("ApiKey", { secret: true, description: "The partner's key." }); - } - - listFiles(){{T(": string[]")}} { - return ["example-1"]; - } - - getFile(fileId{{T(": string")}}){{T(": ExchangeFile")}} { - return new ExchangeFile({ data: JSON.stringify({ id: fileId }), filename: `${fileId}.json` }); - } - - deleteFile(fileId{{T(": string")}}){{T(": void")}} {} - } - """, - "validator" => $$""" - /** Checks a message before Bitween accepts it. */ - class {{name}} extends Validator { - constructor() { - super(); - expect("MaxBytes", { default: "1000000", type: "number", description: "The largest message accepted." }); - } - - validate(file{{T(": ExchangeFile")}}){{T(": ValidationResult")}} { - const result = new ValidationResult(); - if (file.data.length > Number(valueOf("MaxBytes"))) result.add("Data", "The message is larger than allowed."); - return result; - } - } - """, - "mapper" => $$""" - /** Maps a message into the shape the next step expects. */ - class {{name}} extends Mapper { - map(file{{T(": ExchangeFile")}}){{T(": ExchangeFile")}} { - // Return the message in its new shape. - return new ExchangeFile({ data: file.data, filename: file.filename }); - } - } - """, - _ => $$""" - /** Delivers a message and returns the partner's response. */ - class {{name}} extends Handler { - constructor() { - super(); - // Declare settings here; read them in the methods with valueOf. - expect("BaseUrl", { default: "https://partner.example.test", description: "Where messages go." }); - expect("ApiKey", { secret: true, description: "The partner's key." }); - } - - handle(file{{T(": ExchangeFile")}}){{T(": ExchangeFile")}} { - // Send file.data to valueOf("BaseUrl"). A rejection is returned with badData: true, not thrown. - return new ExchangeFile({ data: file.data, filename: file.filename }); - } - } - """, - }; - // valueOf is used by every template but the mapper; keep the import list honest there. - if (kind == "mapper") header = header.Replace("expect, run, valueOf", "run"); - else if (kind == "receiver") header = header.Replace("expect, run, valueOf", "expect, run"); - return header + "\n\n" + body + "\n\nrun(" + name + ");\n"; - } - - static string Pascal(string kind) => char.ToUpperInvariant(kind[0]) + kind[1..]; - - static string Spaced(string name) => Regex.Replace(name, "(?<=[a-z0-9])(?=[A-Z])", " "); } } diff --git a/SW.Serverless.UnitTests.BitweenHandler/Program.cs b/SW.Serverless.UnitTests.BitweenHandler/Program.cs deleted file mode 100644 index 8396520..0000000 --- a/SW.Serverless.UnitTests.BitweenHandler/Program.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System.Threading.Tasks; -using SW.Serverless.Sdk; - -namespace SW.Serverless.UnitTests.BitweenHandler; - -/// An exchange file as Bitween sends it. -public class ExchangeFile -{ - public string Data { get; set; } - public string Filename { get; set; } - public bool BadData { get; set; } - public string ContentType { get; set; } -} - -/// -/// A Bitween handler on the classic text protocol: delivers by echoing the order back. With Mode -/// set to "broken" it answers without the Data every exchange file must carry. -/// -[AdapterKind("handler")] -[AdapterContract("bitween", 1)] -public class Handler -{ - public Handler() - { - Runner.Expect("Endpoint", "https://partner.example.test/orders", description: "Where orders go."); - Runner.Expect("Mode", "working"); - Runner.Expect("ApiKey", isPrivate: true); - } - - public Task Handle(ExchangeFile file) => - Task.FromResult(Runner.StartupValueOf("Mode") == "broken" - ? new { Filename = "answer.json" } - : new ExchangeFile { Data = "{\"accepted\":true}", Filename = "answer.json", ContentType = "application/json" }); -} - -static class Program -{ - static Task Main() => Runner.Run(new Handler()); -} diff --git a/SW.Serverless.UnitTests.BitweenReceiver/Program.cs b/SW.Serverless.UnitTests.BitweenReceiver/Program.cs deleted file mode 100644 index 6e15511..0000000 --- a/SW.Serverless.UnitTests.BitweenReceiver/Program.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using SW.Serverless.Sdk; - -namespace SW.Serverless.UnitTests.BitweenReceiver; - -public class ExchangeFile -{ - public string Data { get; set; } - public string Filename { get; set; } -} - -/// -/// A Bitween receiver over gRPC that reads the files in a folder and deletes one when told to — -/// so a test can see whether DeleteFile was called. -/// -[AdapterKind("receiver")] -[AdapterContract("bitween", 1)] -public class Handler -{ - public Handler() => Runner.Expect("Folder"); - - string Folder => Runner.StartupValueOf("Folder"); - - public Task Initialize() => Task.CompletedTask; - - public Task> ListFiles() => - Task.FromResult>(Directory.GetFiles(Folder).Select(Path.GetFileName).OrderBy(n => n).ToList()); - - public Task GetFile(string fileId) => - Task.FromResult(new ExchangeFile { Data = File.ReadAllText(Path.Combine(Folder, fileId)), Filename = fileId }); - - public Task DeleteFile(string fileId) - { - File.Delete(Path.Combine(Folder, fileId)); - return Task.CompletedTask; - } - - public Task Finalize() => Task.CompletedTask; -} - -static class Program -{ - static Task Main() => Runner.RunResident(new Handler()); -} diff --git a/SW.Serverless.UnitTests.GrpcClassicAdapter/Program.cs b/SW.Serverless.UnitTests.GrpcClassicAdapter/Program.cs index ff4694f..52a0189 100644 --- a/SW.Serverless.UnitTests.GrpcClassicAdapter/Program.cs +++ b/SW.Serverless.UnitTests.GrpcClassicAdapter/Program.cs @@ -3,8 +3,8 @@ namespace SW.Serverless.UnitTests.GrpcClassicAdapter; -[AdapterKind("handler")] -[AdapterContract("bitween", 1)] +[AdapterKind("processor")] +[AdapterContract("orders", 1)] public class Handler { public Handler() diff --git a/SW.Serverless.UnitTests.OrdersProcessor/Program.cs b/SW.Serverless.UnitTests.OrdersProcessor/Program.cs new file mode 100644 index 0000000..c70babb --- /dev/null +++ b/SW.Serverless.UnitTests.OrdersProcessor/Program.cs @@ -0,0 +1,37 @@ +using System.Threading.Tasks; +using SW.Serverless.Sdk; + +namespace SW.Serverless.UnitTests.OrdersProcessor; + +/// An order, as the tests' sample "orders" contract sends it. +public class Order +{ + public string OrderId { get; set; } + public int Lines { get; set; } +} + +/// +/// A processor for the sample "orders" contract, on the classic text protocol: accepts every order. +/// With Mode set to "broken" it answers without the Accepted every receipt must carry. +/// +[AdapterKind("processor")] +[AdapterContract("orders", 1)] +public class Processor +{ + public Processor() + { + Runner.Expect("Endpoint", "https://partner.example.test/orders", description: "Where orders go."); + Runner.Expect("Mode", "working"); + Runner.Expect("ApiKey", isPrivate: true); + } + + public Task Process(Order order) => + Task.FromResult(Runner.StartupValueOf("Mode") == "broken" + ? new { Reference = "R-" + order.OrderId } + : new { Accepted = true, Reference = "R-" + order.OrderId }); +} + +static class Program +{ + static Task Main() => Runner.Run(new Processor()); +} diff --git a/SW.Serverless.UnitTests.BitweenHandler/SW.Serverless.UnitTests.BitweenHandler.csproj b/SW.Serverless.UnitTests.OrdersProcessor/SW.Serverless.UnitTests.OrdersProcessor.csproj similarity index 100% rename from SW.Serverless.UnitTests.BitweenHandler/SW.Serverless.UnitTests.BitweenHandler.csproj rename to SW.Serverless.UnitTests.OrdersProcessor/SW.Serverless.UnitTests.OrdersProcessor.csproj diff --git a/SW.Serverless.UnitTests.OrdersSource/Program.cs b/SW.Serverless.UnitTests.OrdersSource/Program.cs new file mode 100644 index 0000000..14d03b7 --- /dev/null +++ b/SW.Serverless.UnitTests.OrdersSource/Program.cs @@ -0,0 +1,47 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using SW.Serverless.Sdk; + +namespace SW.Serverless.UnitTests.OrdersSource; + +public class Order +{ + public string OrderId { get; set; } + public int Lines { get; set; } +} + +/// +/// A source for the sample "orders" contract, over gRPC: reads the order files in a folder and +/// removes one when told to — so a test can see whether Remove was called. +/// +[AdapterKind("source")] +[AdapterContract("orders", 1)] +public class Source +{ + public Source() => Runner.Expect("Folder"); + + string Folder => Runner.StartupValueOf("Folder"); + + public Task Open() => Task.CompletedTask; + + public Task> List() => + Task.FromResult>(Directory.GetFiles(Folder).Select(Path.GetFileName).OrderBy(n => n).ToList()); + + public Task Fetch(string orderId) => + Task.FromResult(new Order { OrderId = orderId, Lines = File.ReadAllText(Path.Combine(Folder, orderId)).Length }); + + public Task Remove(string orderId) + { + File.Delete(Path.Combine(Folder, orderId)); + return Task.CompletedTask; + } + + public Task Close() => Task.CompletedTask; +} + +static class Program +{ + static Task Main() => Runner.RunResident(new Source()); +} diff --git a/SW.Serverless.UnitTests.BitweenReceiver/SW.Serverless.UnitTests.BitweenReceiver.csproj b/SW.Serverless.UnitTests.OrdersSource/SW.Serverless.UnitTests.OrdersSource.csproj similarity index 100% rename from SW.Serverless.UnitTests.BitweenReceiver/SW.Serverless.UnitTests.BitweenReceiver.csproj rename to SW.Serverless.UnitTests.OrdersSource/SW.Serverless.UnitTests.OrdersSource.csproj diff --git a/SW.Serverless.UnitTests/DescribeTests.cs b/SW.Serverless.UnitTests/DescribeTests.cs index 77273c1..18cd5ce 100644 --- a/SW.Serverless.UnitTests/DescribeTests.cs +++ b/SW.Serverless.UnitTests/DescribeTests.cs @@ -80,8 +80,8 @@ public void An_adapter_describes_the_contracts_it_implements() { var d = Describe("SW.Serverless.UnitTests.GrpcClassicAdapter"); - Assert.AreEqual(1, d.Contracts["bitween"]); - CollectionAssert.AreEqual(new[] { "handler" }, d.Kinds); + Assert.AreEqual(1, d.Contracts["orders"]); + CollectionAssert.AreEqual(new[] { "processor" }, d.Kinds); Assert.AreEqual("hello ", d.Settings.Single(s => s.Name == "Prefix").Default); Assert.AreEqual(0, d.Warnings.Count, string.Join("; ", d.Warnings)); } diff --git a/SW.Serverless.UnitTests/GrpcClassicTests.cs b/SW.Serverless.UnitTests/GrpcClassicTests.cs index 64814a7..1169f88 100644 --- a/SW.Serverless.UnitTests/GrpcClassicTests.cs +++ b/SW.Serverless.UnitTests/GrpcClassicTests.cs @@ -17,7 +17,7 @@ namespace SW.Serverless.UnitTests { /// - /// A classic session — start, call, dispose, as Bitween runs handlers — with an adapter that + /// A classic session — start, call, dispose, as an application runs one call — with an adapter that /// speaks gRPC: what every adapter in a language other than .NET does, and a .NET adapter /// whose manifest opts in with protocol 2. IServerlessService is unchanged for the caller; the /// session runs on a resident instance of its own. @@ -104,8 +104,8 @@ static IEnumerable RunningKeys() => [TestMethod] public async Task A_session_without_a_correlation_id_or_with_null_values_still_starts() { - // A protobuf map can't hold a null: Bitween's gateway runs validators with no correlation - // id, and those sessions failed before the adapter was ready. + // A protobuf map can't hold a null: a session started with no correlation id, or a + // setting left null, failed before the adapter was ready. var service = Service(); await service.StartAsync(AdapterId, null, new Dictionary { ["Prefix"] = "hi ", ["Unset"] = null }); try @@ -184,8 +184,8 @@ public async Task Its_handshake_names_its_language_kinds_and_contracts() var described = host.Services.GetRequiredService().Describe() .Single(h => !before.Contains(h.InstanceKey)); Assert.AreEqual("dotnet", described.SdkLanguage); - CollectionAssert.AreEqual(new[] { "handler" }, described.Kinds.ToList()); - Assert.AreEqual(1, described.Contracts["bitween"]); + CollectionAssert.AreEqual(new[] { "processor" }, described.Kinds.ToList()); + Assert.AreEqual(1, described.Contracts["orders"]); Assert.AreEqual("10.1.0", described.SdkVersion); } finally diff --git a/SW.Serverless.UnitTests/NodeAdapterTests.cs b/SW.Serverless.UnitTests/NodeAdapterTests.cs index dfd7ee1..d79258c 100644 --- a/SW.Serverless.UnitTests/NodeAdapterTests.cs +++ b/SW.Serverless.UnitTests/NodeAdapterTests.cs @@ -18,9 +18,9 @@ namespace SW.Serverless.UnitTests { /// - /// Adapters written in JavaScript and TypeScript with @simplyworks/serverless, built by serverless - /// build and run by the real host: classic sessions as Bitween runs handlers, resident instances, - /// and the Bitween kinds written with @simplyworks/bitween — the validator in TypeScript. + /// Adapters written in JavaScript and TypeScript with @simplyworks/sw-serverless, built by + /// sw-serverless build and run by the real host: classic sessions, resident instances, and + /// adapters implementing the tests' sample "orders" contract — one of them in TypeScript. /// [TestClass] public class NodeAdapterTests @@ -32,9 +32,9 @@ static readonly (string Id, string Script, string Lifecycle)[] Adapters = { ("test.node.classic", "classic.js", "classic"), ("test.node.resident", "resident.js", "resident"), - ("test.node.handler", "bitween_handler.js", "classic"), - ("test.node.receiver", "bitween_receiver.js", "classic"), - ("test.node.validator", "bitween_validator.ts", "classic"), + ("test.node.processor", "orders_processor.js", "classic"), + ("test.node.source", "orders_source.js", "classic"), + ("test.node.tsprocessor", "orders_processor.ts", "classic"), }; [ClassInitialize] @@ -235,78 +235,55 @@ public async Task A_resident_node_adapter_starts_reports_status_publishes_keeps_ Assert.IsFalse(residents.Describe().Any(d => d.InstanceKey == "nd-resident")); } - [TestMethod] - public async Task A_bitween_handler_in_javascript_takes_and_returns_exchange_files_as_dotnet_ones() - { - var answer = await InSession("test.node.handler", new Dictionary { ["Partner"] = "acme" }, s => - s.InvokeAsync("Handle", new { Data = "{\"orderId\":\"SO-1\"}", Filename = "order.json", BadData = false })); - - Assert.AreEqual("answer.json", (string)answer["Filename"]); - Assert.IsFalse((bool)answer["BadData"]); - var data = JObject.Parse((string)answer["Data"]); - Assert.AreEqual("acme", (string)data["to"]); - Assert.AreEqual("SO-1", (string)data["orderId"]); - Assert.AreEqual("order.json", (string)data["from"]); - // Hash is what .NET's ExchangeFile computes: SHA-1 of Data, lower-case hex. - using var sha1 = System.Security.Cryptography.SHA1.Create(); - Assert.AreEqual(Convert.ToHexString(sha1.ComputeHash(Encoding.UTF8.GetBytes((string)answer["Data"]))).ToLowerInvariant(), - (string)answer["Hash"]); - - var rejected = await InSession("test.node.handler", new Dictionary { ["Partner"] = "acme" }, s => - s.InvokeAsync("Handle", new { Data = "{\"reject\":true}" })); - Assert.IsTrue((bool)rejected["BadData"], "a rejected delivery is returned, not raised"); - } - - [TestMethod] - public async Task A_bitween_validator_in_typescript_reports_each_failure() + [DataTestMethod] + [DataRow("test.node.processor")] + [DataRow("test.node.tsprocessor")] + public async Task An_orders_processor_in_javascript_or_typescript_takes_and_returns_json_objects(string adapter) { - NodePackage.RequireTypeStripping(); - var result = await InSession("test.node.validator", null, s => - s.InvokeAsync("Validate", new { Data = "{\"lines\":[]}" })); - - Assert.IsFalse((bool)result["Success"]); - CollectionAssert.AreEqual(new[] { "orderId", "lines" }, - result["Validations"]!.Select(v => (string)v["Key"]).ToArray()); - - var valid = await InSession("test.node.validator", null, s => - s.InvokeAsync("Validate", new { Data = "{\"orderId\":\"SO-1\",\"lines\":[1]}" })); - Assert.IsTrue((bool)valid["Success"]); + if (adapter.Contains("ts")) NodePackage.RequireTypeStripping(); + var receipt = await InSession(adapter, new Dictionary { ["Partner"] = "acme" }, s => + s.InvokeAsync("Process", new { OrderId = "SO-1", Lines = 2 })); + Assert.IsTrue((bool)receipt["Accepted"]); + Assert.AreEqual("acme:SO-1", (string)receipt["Reference"]); + + var refused = await InSession(adapter, new Dictionary { ["Partner"] = "acme" }, s => + s.InvokeAsync("Process", new { OrderId = "SO-2" })); + Assert.IsFalse((bool)refused["Accepted"]); } [TestMethod] - public async Task A_bitween_receiver_in_javascript_runs_a_session_in_the_contract_s_order() + public async Task An_orders_source_in_javascript_runs_a_session_in_the_contract_s_order() { - var root = Path.Combine(workDirectory, "receiver"); + var root = Path.Combine(workDirectory, "source"); var folder = Path.Combine(root, "inbox"); Directory.CreateDirectory(folder); - File.WriteAllText(Path.Combine(folder, "a.json"), "{\"n\":1}"); - File.WriteAllText(Path.Combine(folder, "b.json"), "{\"n\":2}"); + File.WriteAllText(Path.Combine(folder, "a"), "1"); + File.WriteAllText(Path.Combine(folder, "b"), "22"); - var taken = await InSession("test.node.receiver", new Dictionary { ["Folder"] = folder }, async s => + var taken = await InSession("test.node.source", new Dictionary { ["Folder"] = folder }, async s => { var got = new List(); - await s.InvokeAsync("Initialize", null); - foreach (var id in await s.InvokeAsync("ListFiles", null)) + await s.InvokeAsync("Open", null); + foreach (var id in await s.InvokeAsync("List", null)) { - var file = await s.InvokeAsync("GetFile", id); - got.Add((string)file["Filename"] + "=" + (string)file["Data"]); - await s.InvokeAsync("DeleteFile", id); + var order = await s.InvokeAsync("Fetch", id); + got.Add((string)order["OrderId"] + "=" + (int)order["Lines"]); + await s.InvokeAsync("Remove", id); } - await s.InvokeAsync("Finalize", null); + await s.InvokeAsync("Close", null); return got; }); - CollectionAssert.AreEqual(new[] { "a.json={\"n\":1}", "b.json={\"n\":2}" }, taken); + CollectionAssert.AreEqual(new[] { "a=1", "b=2" }, taken); Assert.AreEqual(0, Directory.GetFiles(folder).Length); - Assert.AreEqual("Initialize,ListFiles,GetFile,DeleteFile,GetFile,DeleteFile,Finalize", - File.ReadAllText(Path.Combine(root, "calls.txt"))); + Assert.AreEqual("Open,List,Fetch,Remove,Fetch,Remove,Close", File.ReadAllText(Path.Combine(root, "calls.txt"))); } [TestMethod] public async Task A_node_adapter_describes_itself_for_the_manifest_and_typescript_runs_as_javascript() { NodePackage.RequireTypeStripping(); - var (zip, entry) = await NodePackage.BuildAsync(workDirectory, "test.node.describe", "bitween_validator.ts"); + var (zip, entry) = await NodePackage.BuildAsync(workDirectory, "test.node.describe", "orders_processor.ts"); Assert.AreEqual("main.js", entry, "the TypeScript entry runs as the JavaScript Node strips it to"); using var archive = ZipFile.OpenRead(zip); @@ -314,16 +291,16 @@ public async Task A_node_adapter_describes_itself_for_the_manifest_and_typescrip Assert.AreEqual("node", manifest.Runtime); Assert.AreEqual("typescript", manifest.Language); Assert.AreEqual(2, manifest.Protocol.Min); - CollectionAssert.AreEqual(new[] { "validator" }, manifest.Kinds); - Assert.AreEqual(1, manifest.Contracts["bitween"]); - Assert.IsNotNull(archive.GetEntry("node_modules/@simplyworks/serverless/src/index.js")); - Assert.IsNotNull(archive.GetEntry("node_modules/@simplyworks/bitween/src/index.js")); + CollectionAssert.AreEqual(new[] { "processor" }, manifest.Kinds); + Assert.AreEqual(1, manifest.Contracts["orders"]); + Assert.AreEqual("Partner", manifest.Properties.Single().Name); + Assert.IsNotNull(archive.GetEntry("node_modules/@simplyworks/sw-serverless/src/index.js")); Assert.IsNull(archive.GetEntry("main.ts"), "the package runs JavaScript"); Assert.IsNotNull(archive.GetEntry("source/main.ts"), "the source is what was written"); } } - /// A Node adapter's project — the script as main.js or main.ts, and its adapter.json — built by serverless build. + /// A Node adapter's project — the script as main.js or main.ts, and its adapter.json — built by sw-serverless build. static class NodePackage { static readonly Lazy canStripTypes = new(() => diff --git a/SW.Serverless.UnitTests/NodeAdapters/bitween_handler.js b/SW.Serverless.UnitTests/NodeAdapters/bitween_handler.js deleted file mode 100644 index 4e924ba..0000000 --- a/SW.Serverless.UnitTests/NodeAdapters/bitween_handler.js +++ /dev/null @@ -1,19 +0,0 @@ -// A Bitween handler written with @simplyworks/bitween. -const sw = require("@simplyworks/serverless"); -const { ExchangeFile, Handler } = require("@simplyworks/bitween"); - -class Orders extends Handler { - constructor() { - super(); - sw.expect("Partner", { description: "Who receives the orders" }); - } - - handle(file) { - const order = JSON.parse(file.data); - if (order.reject) return new ExchangeFile({ data: '{"error":"rejected"}', badData: true, contentType: "application/json" }); - const answer = { to: sw.valueOf("Partner"), orderId: order.orderId, from: file.filename }; - return new ExchangeFile({ data: JSON.stringify(answer), filename: "answer.json", contentType: "application/json" }); - } -} - -sw.run(Orders); diff --git a/SW.Serverless.UnitTests/NodeAdapters/bitween_receiver.js b/SW.Serverless.UnitTests/NodeAdapters/bitween_receiver.js deleted file mode 100644 index f914647..0000000 --- a/SW.Serverless.UnitTests/NodeAdapters/bitween_receiver.js +++ /dev/null @@ -1,28 +0,0 @@ -// A Bitween receiver over a folder, written with @simplyworks/bitween. -const fs = require("node:fs"); -const path = require("node:path"); -const sw = require("@simplyworks/serverless"); -const { ExchangeFile, Receiver } = require("@simplyworks/bitween"); - -class Folder extends Receiver { - constructor() { - super(); - sw.expect("Folder"); - this.calls = []; - } - - get folder() { return sw.valueOf("Folder"); } - initialize() { this.calls.push("Initialize"); } - listFiles() { this.calls.push("ListFiles"); return fs.readdirSync(this.folder).sort(); } - getFile(id) { - this.calls.push("GetFile"); - return new ExchangeFile({ data: fs.readFileSync(path.join(this.folder, id), "utf8"), filename: id }); - } - deleteFile(id) { this.calls.push("DeleteFile"); fs.rmSync(path.join(this.folder, id)); } - finalize() { - this.calls.push("Finalize"); - fs.writeFileSync(path.join(this.folder, "..", "calls.txt"), this.calls.join(",")); - } -} - -sw.run(Folder); diff --git a/SW.Serverless.UnitTests/NodeAdapters/bitween_validator.ts b/SW.Serverless.UnitTests/NodeAdapters/bitween_validator.ts deleted file mode 100644 index ff34fb0..0000000 --- a/SW.Serverless.UnitTests/NodeAdapters/bitween_validator.ts +++ /dev/null @@ -1,17 +0,0 @@ -// A Bitween validator in TypeScript: run as Node runs it once serverless build strips the types. -const sw = require("@simplyworks/serverless"); -const { ExchangeFile, ValidationResult, Validator } = require("@simplyworks/bitween"); - -interface Order { orderId?: string; lines?: unknown[] } - -class Orders extends Validator { - validate(file: typeof ExchangeFile.prototype): typeof ValidationResult.prototype { - const order: Order = JSON.parse(file.data); - const result = new ValidationResult(); - if (!order.orderId) result.add("orderId", "An order needs an id."); - if (!order.lines || order.lines.length === 0) result.add("lines", "An order needs at least one line."); - return result; - } -} - -sw.run(Orders); diff --git a/SW.Serverless.UnitTests/NodeAdapters/classic.js b/SW.Serverless.UnitTests/NodeAdapters/classic.js index 0edee5c..c0898b8 100644 --- a/SW.Serverless.UnitTests/NodeAdapters/classic.js +++ b/SW.Serverless.UnitTests/NodeAdapters/classic.js @@ -1,5 +1,5 @@ -// A classic adapter in JavaScript, called as Bitween calls handlers: one session, one call at a time. -const sw = require("@simplyworks/serverless"); +// A classic adapter in JavaScript, called one session, one call at a time. +const sw = require("@simplyworks/sw-serverless"); class Classic { static commands = { diff --git a/SW.Serverless.UnitTests/NodeAdapters/orders_processor.js b/SW.Serverless.UnitTests/NodeAdapters/orders_processor.js new file mode 100644 index 0000000..25d45fe --- /dev/null +++ b/SW.Serverless.UnitTests/NodeAdapters/orders_processor.js @@ -0,0 +1,21 @@ +// A processor for the tests' sample "orders" contract, in JavaScript. +const sw = require("@simplyworks/sw-serverless"); + +class Processor { + static kinds = ["processor"]; + static contracts = { orders: 1 }; + static commands = { + Process: { method: "process", input: "json", output: "json", description: "Takes an order and says whether it was accepted." }, + }; + + constructor() { + sw.expect("Partner", { description: "Who receives the orders" }); + } + + process(order) { + if (!order.Lines) return { Accepted: false, Reference: null }; + return { Accepted: true, Reference: `${sw.valueOf("Partner")}:${order.OrderId}` }; + } +} + +sw.run(Processor); diff --git a/SW.Serverless.UnitTests/NodeAdapters/orders_processor.ts b/SW.Serverless.UnitTests/NodeAdapters/orders_processor.ts new file mode 100644 index 0000000..bdef170 --- /dev/null +++ b/SW.Serverless.UnitTests/NodeAdapters/orders_processor.ts @@ -0,0 +1,23 @@ +// A processor for the tests' sample "orders" contract, in TypeScript: run as Node runs it once the build strips the types. +const sw = require("@simplyworks/sw-serverless"); + +interface Order { OrderId: string; Lines?: number } +interface Receipt { Accepted: boolean; Reference: string | null } + +class Processor { + static kinds = ["processor"]; + static contracts = { orders: 1 }; + static commands = { + Process: { method: "process", input: "json", output: "json" }, + }; + + constructor() { + sw.expect("Partner", { description: "Who receives the orders" }); + } + + process(order: Order): Receipt { + return order.Lines ? { Accepted: true, Reference: `${sw.valueOf("Partner")}:${order.OrderId}` } : { Accepted: false, Reference: null }; + } +} + +sw.run(Processor); diff --git a/SW.Serverless.UnitTests/NodeAdapters/orders_source.js b/SW.Serverless.UnitTests/NodeAdapters/orders_source.js new file mode 100644 index 0000000..a50ca8c --- /dev/null +++ b/SW.Serverless.UnitTests/NodeAdapters/orders_source.js @@ -0,0 +1,36 @@ +// A source for the tests' sample "orders" contract, in JavaScript: the order files in a folder. +const fs = require("node:fs"); +const path = require("node:path"); +const sw = require("@simplyworks/sw-serverless"); + +class Source { + static kinds = ["source"]; + static contracts = { orders: 1 }; + static commands = { + Open: { method: "open" }, + List: { method: "list", output: "json" }, + Fetch: { method: "fetch", input: "string", output: "json" }, + Remove: { method: "remove", input: "string" }, + Close: { method: "close" }, + }; + + constructor() { + sw.expect("Folder"); + this.calls = []; + } + + get folder() { return sw.valueOf("Folder"); } + open() { this.calls.push("Open"); } + list() { this.calls.push("List"); return fs.readdirSync(this.folder).sort(); } + fetch(id) { + this.calls.push("Fetch"); + return { OrderId: id, Lines: fs.readFileSync(path.join(this.folder, id), "utf8").length }; + } + remove(id) { this.calls.push("Remove"); fs.rmSync(path.join(this.folder, id)); } + close() { + this.calls.push("Close"); + fs.writeFileSync(path.join(this.folder, "..", "calls.txt"), this.calls.join(",")); + } +} + +sw.run(Source); diff --git a/SW.Serverless.UnitTests/NodeAdapters/resident.js b/SW.Serverless.UnitTests/NodeAdapters/resident.js index cc63134..597edb0 100644 --- a/SW.Serverless.UnitTests/NodeAdapters/resident.js +++ b/SW.Serverless.UnitTests/NodeAdapters/resident.js @@ -1,5 +1,5 @@ // A resident adapter in JavaScript: started, kept running, asked for its status, reset and stopped. -const sw = require("@simplyworks/serverless"); +const sw = require("@simplyworks/sw-serverless"); class Resident { static commands = { diff --git a/SW.Serverless.UnitTests/PythonAdapterTests.cs b/SW.Serverless.UnitTests/PythonAdapterTests.cs index 96a7b31..2b05d0f 100644 --- a/SW.Serverless.UnitTests/PythonAdapterTests.cs +++ b/SW.Serverless.UnitTests/PythonAdapterTests.cs @@ -18,10 +18,10 @@ namespace SW.Serverless.UnitTests { /// - /// Adapters written in Python with simplyworks-serverless, run by the real host: classic - /// sessions as Bitween runs handlers, resident instances, and the Bitween kinds written with - /// simplyworks-bitween. The SDK has no dependencies, so a package is the adapter's files and - /// the SDK's, vendored beside them — what serverless build makes. + /// Adapters written in Python with sw-serverless, run by the real host: classic sessions, one + /// call at a time, resident instances, and adapters implementing the tests' sample "orders" + /// contract. The SDK has no dependencies, so a package is the adapter's files and the SDK's, + /// vendored beside them — what sw-serverless build makes. /// [TestClass] public class PythonAdapterTests @@ -33,9 +33,8 @@ static readonly (string Id, string Script, string Lifecycle)[] Adapters = { ("test.python.classic", "classic.py", "classic"), ("test.python.resident", "resident.py", "resident"), - ("test.python.handler", "bitween_handler.py", "classic"), - ("test.python.receiver", "bitween_receiver.py", "classic"), - ("test.python.validator", "bitween_validator.py", "classic"), + ("test.python.processor", "orders_processor.py", "classic"), + ("test.python.source", "orders_source.py", "classic"), }; [ClassInitialize] @@ -235,76 +234,51 @@ public async Task A_resident_python_adapter_starts_reports_status_publishes_keep } [TestMethod] - public async Task A_bitween_handler_in_python_takes_and_returns_exchange_files_as_dotnet_ones() + public async Task An_orders_processor_in_python_takes_and_returns_json_objects() { - var answer = await InSession("test.python.handler", new Dictionary { ["Partner"] = "acme" }, s => - s.InvokeAsync("Handle", new { Data = "{\"orderId\":\"SO-1\"}", Filename = "order.json", BadData = false })); - - Assert.AreEqual("answer.json", (string)answer["Filename"]); - Assert.IsFalse((bool)answer["BadData"]); - var data = JObject.Parse((string)answer["Data"]); - Assert.AreEqual("acme", (string)data["to"]); - Assert.AreEqual("SO-1", (string)data["orderId"]); - Assert.AreEqual("order.json", (string)data["from"]); - // Hash is what .NET's ExchangeFile computes: SHA-1 of Data, lower-case hex. - using var sha1 = System.Security.Cryptography.SHA1.Create(); - Assert.AreEqual(Convert.ToHexString(sha1.ComputeHash(Encoding.UTF8.GetBytes((string)answer["Data"]))).ToLowerInvariant(), - (string)answer["Hash"]); - - var rejected = await InSession("test.python.handler", new Dictionary { ["Partner"] = "acme" }, s => - s.InvokeAsync("Handle", new { Data = "{\"reject\":true}" })); - Assert.IsTrue((bool)rejected["BadData"], "a rejected delivery is returned, not raised"); + var receipt = await InSession("test.python.processor", new Dictionary { ["Partner"] = "acme" }, s => + s.InvokeAsync("Process", new { OrderId = "SO-1", Lines = 2 })); + Assert.IsTrue((bool)receipt["Accepted"]); + Assert.AreEqual("acme:SO-1", (string)receipt["Reference"]); + + var refused = await InSession("test.python.processor", new Dictionary { ["Partner"] = "acme" }, s => + s.InvokeAsync("Process", new { OrderId = "SO-2" })); + Assert.IsFalse((bool)refused["Accepted"]); } [TestMethod] - public async Task A_bitween_validator_in_python_reports_each_failure() + public async Task An_orders_source_in_python_runs_a_session_in_the_contract_s_order() { - var result = await InSession("test.python.validator", null, s => - s.InvokeAsync("Validate", new { Data = "{\"lines\":[]}" })); - - Assert.IsFalse((bool)result["Success"]); - CollectionAssert.AreEqual(new[] { "orderId", "lines" }, - result["Validations"]!.Select(v => (string)v["Key"]).ToArray()); - - var valid = await InSession("test.python.validator", null, s => - s.InvokeAsync("Validate", new { Data = "{\"orderId\":\"SO-1\",\"lines\":[1]}" })); - Assert.IsTrue((bool)valid["Success"]); - } - - [TestMethod] - public async Task A_bitween_receiver_in_python_runs_a_session_in_the_contract_s_order() - { - var root = Path.Combine(workDirectory, "receiver"); + var root = Path.Combine(workDirectory, "source"); var folder = Path.Combine(root, "inbox"); Directory.CreateDirectory(folder); - File.WriteAllText(Path.Combine(folder, "a.json"), "{\"n\":1}"); - File.WriteAllText(Path.Combine(folder, "b.json"), "{\"n\":2}"); + File.WriteAllText(Path.Combine(folder, "a"), "1"); + File.WriteAllText(Path.Combine(folder, "b"), "22"); - var taken = await InSession("test.python.receiver", new Dictionary { ["Folder"] = folder }, async s => + var taken = await InSession("test.python.source", new Dictionary { ["Folder"] = folder }, async s => { var got = new List(); - await s.InvokeAsync("Initialize", null); - foreach (var id in await s.InvokeAsync("ListFiles", null)) + await s.InvokeAsync("Open", null); + foreach (var id in await s.InvokeAsync("List", null)) { - var file = await s.InvokeAsync("GetFile", id); - got.Add((string)file["Filename"] + "=" + (string)file["Data"]); - await s.InvokeAsync("DeleteFile", id); + var order = await s.InvokeAsync("Fetch", id); + got.Add((string)order["OrderId"] + "=" + (int)order["Lines"]); + await s.InvokeAsync("Remove", id); } - await s.InvokeAsync("Finalize", null); + await s.InvokeAsync("Close", null); return got; }); - CollectionAssert.AreEqual(new[] { "a.json={\"n\":1}", "b.json={\"n\":2}" }, taken); + CollectionAssert.AreEqual(new[] { "a=1", "b=2" }, taken); Assert.AreEqual(0, Directory.GetFiles(folder).Length); - Assert.AreEqual("Initialize,ListFiles,GetFile,DeleteFile,GetFile,DeleteFile,Finalize", - File.ReadAllText(Path.Combine(root, "calls.txt"))); + Assert.AreEqual("Open,List,Fetch,Remove,Fetch,Remove,Close", File.ReadAllText(Path.Combine(root, "calls.txt"))); } [TestMethod] public async Task A_python_adapter_describes_itself_for_the_manifest() { var dir = Path.Combine(workDirectory, "describe"); - using (var package = PythonPackage.Build("test.python.handler", "bitween_handler.py", "classic")) + using (var package = PythonPackage.Build("test.python.processor", "orders_processor.py", "classic")) using (var archive = new ZipArchive(package)) archive.ExtractToDirectory(dir); @@ -315,19 +289,20 @@ public async Task A_python_adapter_describes_itself_for_the_manifest() Assert.AreEqual("python", description.SdkLanguage); Assert.AreEqual("classic", description.Lifecycle); Assert.AreEqual(2, description.Protocol.Min); - CollectionAssert.AreEqual(new[] { "handler" }, description.Kinds); - Assert.AreEqual(1, description.Contracts["bitween"]); + CollectionAssert.AreEqual(new[] { "processor" }, description.Kinds); + Assert.AreEqual(1, description.Contracts["orders"]); var partner = description.Settings.Single(); Assert.AreEqual("Partner", partner.Name); Assert.IsTrue(partner.Required); - var handle = description.Commands.Single(); - Assert.AreEqual("Handle", handle.Name); - Assert.AreEqual("ExchangeFile", handle.InputSchema!.Value.GetProperty("title").GetString()); + var process = description.Commands.Single(); + Assert.AreEqual("Process", process.Name); + Assert.AreEqual("object", process.InputSchema!.Value.GetProperty("type").GetString()); + Assert.IsTrue(process.ReturnsValue); } } /// - /// A Python adapter packaged as serverless build packages one: the script as main.py, the SDKs + /// A Python adapter packaged as sw-serverless build packages one: the script as main.py, the SDK /// under _vendor/, and an entry that puts _vendor on the path before running main.py. /// static class PythonPackage @@ -362,8 +337,7 @@ void AddFolder(string folder, string under) archive.CreateEntryFromFile(Path.Combine(root, "SW.Serverless.UnitTests", "PythonAdapters", script), "main.py"); AddText(Entry, Bootstrap); - AddFolder(Path.Combine(root, "sdk", "python", "src", "simplyworks_serverless"), "_vendor/simplyworks_serverless"); - AddFolder(Path.Combine(root, "..", "Bitween-api", "sdk", "python", "src", "simplyworks_bitween"), "_vendor/simplyworks_bitween"); + AddFolder(Path.Combine(root, "sdk", "python", "src", "sw_serverless"), "_vendor/sw_serverless"); AddText("adapter.json", new JObject { ["id"] = id, diff --git a/SW.Serverless.UnitTests/PythonAdapters/bitween_handler.py b/SW.Serverless.UnitTests/PythonAdapters/bitween_handler.py deleted file mode 100644 index 5187d45..0000000 --- a/SW.Serverless.UnitTests/PythonAdapters/bitween_handler.py +++ /dev/null @@ -1,21 +0,0 @@ -"""A Bitween handler, mapper-free: the contract's Handle, written with simplyworks_bitween.""" -import json - -import simplyworks_serverless as sw -from simplyworks_bitween import ExchangeFile, Handler - - -class Orders(Handler): - def __init__(self): - sw.expect("Partner", description="Who receives the orders") - - def handle(self, file: ExchangeFile) -> ExchangeFile: - order = json.loads(file.data) - if order.get("reject"): - return ExchangeFile(data='{"error":"rejected"}', bad_data=True, content_type="application/json") - answer = {"to": sw.value_of("Partner"), "orderId": order["orderId"], "from": file.filename} - return ExchangeFile(data=json.dumps(answer), filename="answer.json", content_type="application/json") - - -if __name__ == "__main__": - sw.run(Orders) diff --git a/SW.Serverless.UnitTests/PythonAdapters/bitween_receiver.py b/SW.Serverless.UnitTests/PythonAdapters/bitween_receiver.py deleted file mode 100644 index d9c042a..0000000 --- a/SW.Serverless.UnitTests/PythonAdapters/bitween_receiver.py +++ /dev/null @@ -1,40 +0,0 @@ -"""A Bitween receiver over a folder, written with simplyworks_bitween.""" -import os - -import simplyworks_serverless as sw -from simplyworks_bitween import ExchangeFile, Receiver - - -class Folder(Receiver): - def __init__(self): - sw.expect("Folder") - self.calls = [] - - @property - def folder(self): - return sw.value_of("Folder") - - def initialize(self): - self.calls.append("Initialize") - - def list_files(self): - self.calls.append("ListFiles") - return sorted(os.listdir(self.folder)) - - def get_file(self, file_id): - self.calls.append("GetFile") - with open(os.path.join(self.folder, file_id), encoding="utf-8") as f: - return ExchangeFile(data=f.read(), filename=file_id) - - def delete_file(self, file_id): - self.calls.append("DeleteFile") - os.remove(os.path.join(self.folder, file_id)) - - def finalize(self): - self.calls.append("Finalize") - with open(os.path.join(self.folder, "..", "calls.txt"), "w", encoding="utf-8") as f: - f.write(",".join(self.calls)) - - -if __name__ == "__main__": - sw.run(Folder) diff --git a/SW.Serverless.UnitTests/PythonAdapters/bitween_validator.py b/SW.Serverless.UnitTests/PythonAdapters/bitween_validator.py deleted file mode 100644 index acbc60d..0000000 --- a/SW.Serverless.UnitTests/PythonAdapters/bitween_validator.py +++ /dev/null @@ -1,20 +0,0 @@ -"""A Bitween validator: an order needs an id and at least one line.""" -import json - -import simplyworks_serverless as sw -from simplyworks_bitween import ExchangeFile, ValidationResult, Validator - - -class Orders(Validator): - def validate(self, file: ExchangeFile) -> ValidationResult: - order = json.loads(file.data) - result = ValidationResult() - if not order.get("orderId"): - result.add("orderId", "An order needs an id.") - if not order.get("lines"): - result.add("lines", "An order needs at least one line.") - return result - - -if __name__ == "__main__": - sw.run(Orders) diff --git a/SW.Serverless.UnitTests/PythonAdapters/classic.py b/SW.Serverless.UnitTests/PythonAdapters/classic.py index 5f7deea..1c76707 100644 --- a/SW.Serverless.UnitTests/PythonAdapters/classic.py +++ b/SW.Serverless.UnitTests/PythonAdapters/classic.py @@ -1,9 +1,9 @@ -"""A classic adapter in Python, called as Bitween calls handlers: one session, one call at a time.""" +"""A classic adapter in Python, called one session, one call at a time.""" import asyncio import logging from dataclasses import dataclass -import simplyworks_serverless as sw +import sw_serverless as sw @dataclass diff --git a/SW.Serverless.UnitTests/PythonAdapters/orders_processor.py b/SW.Serverless.UnitTests/PythonAdapters/orders_processor.py new file mode 100644 index 0000000..dbecd66 --- /dev/null +++ b/SW.Serverless.UnitTests/PythonAdapters/orders_processor.py @@ -0,0 +1,18 @@ +"""A processor for the tests' sample "orders" contract: takes an order, answers with a receipt.""" +import sw_serverless as sw + + +@sw.implements("orders", 1, "processor") +class Processor: + def __init__(self): + sw.expect("Partner", description="Who receives the orders") + + @sw.command("Process", description="Takes an order and says whether it was accepted.") + def process(self, order: dict) -> dict: + if not order.get("Lines"): + return {"Accepted": False, "Reference": None} + return {"Accepted": True, "Reference": f"{sw.value_of('Partner')}:{order['OrderId']}"} + + +if __name__ == "__main__": + sw.run(Processor) diff --git a/SW.Serverless.UnitTests/PythonAdapters/orders_source.py b/SW.Serverless.UnitTests/PythonAdapters/orders_source.py new file mode 100644 index 0000000..a2237ad --- /dev/null +++ b/SW.Serverless.UnitTests/PythonAdapters/orders_source.py @@ -0,0 +1,45 @@ +"""A source for the tests' sample "orders" contract: the order files in a folder, one session per run.""" +import os + +import sw_serverless as sw + + +@sw.implements("orders", 1, "source") +class Source: + def __init__(self): + sw.expect("Folder") + self.calls = [] + + @property + def folder(self): + return sw.value_of("Folder") + + @sw.command("Open") + def open(self) -> None: + self.calls.append("Open") + + @sw.command("List") + def list(self) -> list[str]: + self.calls.append("List") + return sorted(os.listdir(self.folder)) + + @sw.command("Fetch") + def fetch(self, order_id: str) -> dict: + self.calls.append("Fetch") + with open(os.path.join(self.folder, order_id), encoding="utf-8") as f: + return {"OrderId": order_id, "Lines": len(f.read())} + + @sw.command("Remove") + def remove(self, order_id: str) -> None: + self.calls.append("Remove") + os.remove(os.path.join(self.folder, order_id)) + + @sw.command("Close") + def close(self) -> None: + self.calls.append("Close") + with open(os.path.join(self.folder, "..", "calls.txt"), "w", encoding="utf-8") as f: + f.write(",".join(self.calls)) + + +if __name__ == "__main__": + sw.run(Source) diff --git a/SW.Serverless.UnitTests/PythonAdapters/resident.py b/SW.Serverless.UnitTests/PythonAdapters/resident.py index 29731e2..560a1c0 100644 --- a/SW.Serverless.UnitTests/PythonAdapters/resident.py +++ b/SW.Serverless.UnitTests/PythonAdapters/resident.py @@ -1,5 +1,5 @@ """A resident adapter in Python: started, kept running, asked for its status, reset and stopped.""" -import simplyworks_serverless as sw +import sw_serverless as sw class Resident: diff --git a/SW.Serverless.UnitTests/ResourceLimitTests.cs b/SW.Serverless.UnitTests/ResourceLimitTests.cs index 78ba73b..6e40e48 100644 --- a/SW.Serverless.UnitTests/ResourceLimitTests.cs +++ b/SW.Serverless.UnitTests/ResourceLimitTests.cs @@ -300,7 +300,7 @@ public async Task Changing_the_hard_ceiling_reports_that_a_restart_is_required() /// /// Restarting keeps the instance in place under the same key. That matters beyond - /// tidiness: in Bitween the key is the data source, and dropping the registry entry would + /// tidiness: a host may key an instance by the connection it holds, and dropping the registry entry would /// release the lease that makes the broker connection exclusive — so a restart would /// briefly become a handover. /// diff --git a/SW.Serverless.sln b/SW.Serverless.sln index 8f2b7f4..e0ad1cf 100644 --- a/SW.Serverless.sln +++ b/SW.Serverless.sln @@ -61,9 +61,9 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.UnitTests.Grp EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.Tooling", "SW.Serverless.Tooling\SW.Serverless.Tooling.csproj", "{C7FCE2D1-D9D6-4E34-BBB2-A94B4B679D77}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.UnitTests.BitweenHandler", "SW.Serverless.UnitTests.BitweenHandler\SW.Serverless.UnitTests.BitweenHandler.csproj", "{DCF81A88-37CA-4746-A4D8-EADEF00422FD}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.UnitTests.OrdersProcessor", "SW.Serverless.UnitTests.OrdersProcessor\SW.Serverless.UnitTests.OrdersProcessor.csproj", "{DCF81A88-37CA-4746-A4D8-EADEF00422FD}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.UnitTests.BitweenReceiver", "SW.Serverless.UnitTests.BitweenReceiver\SW.Serverless.UnitTests.BitweenReceiver.csproj", "{4D824901-1889-4BB2-9F72-2D33E5CFA7B3}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SW.Serverless.UnitTests.OrdersSource", "SW.Serverless.UnitTests.OrdersSource\SW.Serverless.UnitTests.OrdersSource.csproj", "{4D824901-1889-4BB2-9F72-2D33E5CFA7B3}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/SW.Serverless/Resident/IAdapterEventSink.cs b/SW.Serverless/Resident/IAdapterEventSink.cs index 9490c03..65dc949 100644 --- a/SW.Serverless/Resident/IAdapterEventSink.cs +++ b/SW.Serverless/Resident/IAdapterEventSink.cs @@ -29,7 +29,7 @@ public static EventOutcome Rejected(string error) => } /// - /// Implemented by the HOST APPLICATION. In Bitween this persists an Xchange and returns its id; + /// Implemented by the HOST APPLICATION. An integration platform, say, persists the message and returns its id; /// the adapter does not acknowledge its broker until this has returned Accepted. /// public interface IAdapterEventSink diff --git a/SW.Serverless/Resident/IAdapterStateStore.cs b/SW.Serverless/Resident/IAdapterStateStore.cs index e187639..d98d9aa 100644 --- a/SW.Serverless/Resident/IAdapterStateStore.cs +++ b/SW.Serverless/Resident/IAdapterStateStore.cs @@ -29,7 +29,7 @@ public class AdapterStateKey /// An adapter cannot hold its own progress. The supervisor restarts it, the next instance may /// come up on a different node, and a pooled one is not the same process twice — so a cursor /// kept in a field is a cursor that resets to the beginning at the least convenient moment. - /// Bitween backs this with a table; a sample host can use . + /// A host application backs this with a table of its own; a sample host can use . /// /// Values are small — a bookmark, an offset, a timestamp. A host is entitled to refuse a large /// one, and should say so in the returned error rather than storing it. diff --git a/SW.Serverless/Resident/IResidentAdapterHost.cs b/SW.Serverless/Resident/IResidentAdapterHost.cs index 1063dd7..4516240 100644 --- a/SW.Serverless/Resident/IResidentAdapterHost.cs +++ b/SW.Serverless/Resident/IResidentAdapterHost.cs @@ -49,7 +49,7 @@ Task UpdateLimitsAsync(string adapterId, string instanceKey, /// /// Relaunches an instance in place, keeping its registry entry — so a lease, a data source, /// or anything else holding the key still points at it afterwards. Stopping and starting - /// instead drops the entry, which in Bitween's case would release the broker lease that + /// instead drops the entry, which for a host that leases connections would release the lease that /// makes the connection exclusive. /// Task RestartAsync(string adapterId, string instanceKey, diff --git a/SW.Serverless/Resident/ResidentAdapterHost.cs b/SW.Serverless/Resident/ResidentAdapterHost.cs index 04740fb..e446faf 100644 --- a/SW.Serverless/Resident/ResidentAdapterHost.cs +++ b/SW.Serverless/Resident/ResidentAdapterHost.cs @@ -414,7 +414,7 @@ public async Task RestartAsync(string adapterId, string // Relaunch in place: the entry stays in the registry under the same key, so whatever // owns this instance — a lease, a data source, a caller holding the key — still points // at it afterwards. Stopping and starting instead would drop the entry and, in - // Bitween's case, release the broker lease that makes the connection exclusive. + // the case of a host that leases connections, release the lease that makes the connection exclusive. // // Stopping is set for the teardown so the exit does not look like a crash and trigger // the backoff restart; this method does the relaunch itself. diff --git a/SW.Serverless/Resident/ResidentAdapterInstance.cs b/SW.Serverless/Resident/ResidentAdapterInstance.cs index 01ada85..74867b1 100644 --- a/SW.Serverless/Resident/ResidentAdapterInstance.cs +++ b/SW.Serverless/Resident/ResidentAdapterInstance.cs @@ -119,10 +119,10 @@ internal ResidentAdapterInstance(string adapterId, string instanceKey, string to /// The settings the adapter says it reads; empty from an SDK that predates them. public IReadOnlyCollection Settings { get; internal set; } = Array.Empty(); - /// The kinds it implements for a contract, e.g. handler or receiver for Bitween. + /// The kinds it implements for a contract, as that contract names them. public IReadOnlyCollection Kinds { get; internal set; } = Array.Empty(); - /// The contracts it implements and their versions, e.g. bitween → 1. + /// The contracts it implements and their versions, e.g. orders → 1. public IReadOnlyDictionary Contracts { get; internal set; } = new Dictionary(); public int ProtocolVersion { get; internal set; } public IReadOnlyDictionary AdapterValues { get; internal set; } diff --git a/sdk/node/README.md b/sdk/node/README.md index 633aef9..9cb4e2f 100644 --- a/sdk/node/README.md +++ b/sdk/node/README.md @@ -1,11 +1,11 @@ -# @simplyworks/serverless +# @simplyworks/sw-serverless Write SW-Serverless adapters in JavaScript or TypeScript, on Node 22 or later. No dependencies: Node's own `http2` speaks the host's protocol — gRPC over a Unix socket — and the SDK carries the protobuf messages itself, so it vendors into an adapter package as plain files. ```js -const sw = require("@simplyworks/serverless"); +const sw = require("@simplyworks/sw-serverless"); class Greeter { static commands = { diff --git a/sdk/node/package.json b/sdk/node/package.json index 228f904..614a562 100644 --- a/sdk/node/package.json +++ b/sdk/node/package.json @@ -1,6 +1,6 @@ { - "name": "@simplyworks/serverless", - "version": "10.1.0", + "name": "@simplyworks/sw-serverless", + "version": "10.2.0", "description": "Write SW-Serverless adapters in JavaScript or TypeScript: settings, commands, and the host protocol, with no dependencies.", "main": "src/index.js", "types": "src/index.d.ts", diff --git a/sdk/node/src/index.js b/sdk/node/src/index.js index 4b041d7..2f1d6c4 100644 --- a/sdk/node/src/index.js +++ b/sdk/node/src/index.js @@ -2,7 +2,7 @@ /** * Write SW-Serverless adapters in JavaScript or TypeScript, on Node 22 or later. * - * const sw = require("@simplyworks/serverless"); + * const sw = require("@simplyworks/sw-serverless"); * * class Greeter { * static commands = { @@ -26,7 +26,7 @@ const readline = require("node:readline"); const { AsyncLocalStorage } = require("node:async_hooks"); const wire = require("./wire"); -const SDK_VERSION = "10.1.0"; +const SDK_VERSION = "10.2.0"; const SDK_LANGUAGE = "node"; const PROTOCOL = 2; const ATTACH = "/sw.serverless.v1.AdapterHost/Attach"; @@ -55,7 +55,7 @@ const callStorage = new AsyncLocalStorage(); /** * Declares a setting the adapter reads. Required unless it has a default or `required: false` says - * otherwise. A secret is masked wherever Bitween shows it. Declaring a name again replaces it. + * otherwise. A secret is masked wherever a host application shows it. Declaring a name again replaces it. */ function expect(name, options = {}) { if (!name || typeof name !== "string") throw new TypeError("a setting needs a name"); @@ -189,7 +189,7 @@ class Stream { this.session.on("connect", () => { try { this.session.setLocalWindowSize(16 * 1024 * 1024); } catch {} }); this.call = this.session.request({ ":method": "POST", ":path": ATTACH, "content-type": "application/grpc", te: "trailers", - "user-agent": "simplyworks-serverless-node", + "user-agent": "sw-serverless-node", }); this.buffer = Buffer.alloc(0); this.waiting = []; diff --git a/sdk/node/test/sdk.test.js b/sdk/node/test/sdk.test.js index 2e093c9..080795e 100644 --- a/sdk/node/test/sdk.test.js +++ b/sdk/node/test/sdk.test.js @@ -9,14 +9,14 @@ test("a frame round-trips with every kind of field", () => { id: 42, hello: { token: "t", protocol_version: 2, capabilities: ["cancel", "command:Greet"], commands: [{ name: "Greet", returns_value: true, input_schema: '{"type":"string"}' }], - settings: [{ name: "Url", required: true, secret: true }], kinds: ["handler"], contracts: { bitween: 1 }, + settings: [{ name: "Url", required: true, secret: true }], kinds: ["processor"], contracts: { orders: 1 }, }, }; assert.deepStrictEqual(wire.decode("AdapterFrame", wire.encode("AdapterFrame", frame)), frame); }); test("the bytes are the Python SDK's, and .NET's protobuf", () => { - // Written by simplyworks_serverless._wire for the same frame. + // Written by sw_serverless._wire for the same frame. const frame = { id: -7, metric: { name: "m", value: 2.5, tags: { a: "b" } } }; assert.strictEqual(wire.encode("AdapterFrame", frame).toString("hex"), "08f9ffffffffffffffff013a140a016d1100000000000004401a060a0161120162"); diff --git a/sdk/python/README.md b/sdk/python/README.md index b18e9b1..8ae9681 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -1,11 +1,11 @@ -# simplyworks-serverless +# sw-serverless Write SW-Serverless adapters in Python 3.12 or later. The package has no dependencies: it speaks the host's protocol — gRPC over a Unix socket — with the standard library alone, so it vendors into an adapter package as plain files, for any platform. ```python -import simplyworks_serverless as sw +import sw_serverless as sw class Greeter: diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 0496666..e189548 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -3,8 +3,8 @@ requires = ["setuptools>=69"] build-backend = "setuptools.build_meta" [project] -name = "simplyworks-serverless" -version = "10.1.0" +name = "sw-serverless" +version = "10.2.0" description = "Write SW-Serverless adapters in Python: settings, commands, and the host protocol, with no dependencies." readme = "README.md" requires-python = ">=3.12" diff --git a/sdk/python/src/simplyworks_serverless/__init__.py b/sdk/python/src/sw_serverless/__init__.py similarity index 80% rename from sdk/python/src/simplyworks_serverless/__init__.py rename to sdk/python/src/sw_serverless/__init__.py index 104aba4..1265634 100644 --- a/sdk/python/src/simplyworks_serverless/__init__.py +++ b/sdk/python/src/sw_serverless/__init__.py @@ -1,6 +1,6 @@ """Write SW-Serverless adapters in Python. - import simplyworks_serverless as sw + import sw_serverless as sw class Greeter: def __init__(self): @@ -17,12 +17,12 @@ def greet(self, name: str) -> str: ``start`` hook is resident: it runs until stopped, with ``stop``, ``status`` and ``reset`` hooks. """ -from ._adapter import command, declared_settings, expect, startup_values, value_of +from ._adapter import command, declared_settings, expect, implements, startup_values, value_of from ._runner import SDK_VERSION, AdapterError, Context, context, describe, run __version__ = SDK_VERSION __all__ = [ "AdapterError", "Context", "SDK_VERSION", "command", "context", "declared_settings", "describe", - "expect", "run", "startup_values", "value_of", + "expect", "implements", "run", "startup_values", "value_of", ] diff --git a/sdk/python/src/simplyworks_serverless/_adapter.py b/sdk/python/src/sw_serverless/_adapter.py similarity index 87% rename from sdk/python/src/simplyworks_serverless/_adapter.py rename to sdk/python/src/sw_serverless/_adapter.py index 3b20bd9..d4c4128 100644 --- a/sdk/python/src/simplyworks_serverless/_adapter.py +++ b/sdk/python/src/sw_serverless/_adapter.py @@ -34,7 +34,7 @@ def expect(name, default=None, *, required=None, secret=False, description=None, """Declares a setting the adapter reads. Required unless it has a default or ``required=False`` says otherwise. A secret is masked - wherever Bitween shows it. Declaring the same name again replaces it. + wherever a host application shows it. Declaring the same name again replaces it. """ if not name or not isinstance(name, str): raise ValueError("a setting needs a name") @@ -121,6 +121,27 @@ def _type_name(tp): return getattr(tp, "__name__", str(tp)) +def implements(contract, version, *kinds): + """Declares, on an adapter class, a contract it implements and the kinds of it it is. + + @sw.implements("orders", 1, "processor") + class Orders: ... + + A host application's contract package does this for its own base classes.""" + + if not contract or not isinstance(version, int): + raise ValueError("a contract needs a name and an integer version") + + def mark(cls): + contracts = dict(vars(cls).get("__sw_contracts__", {})) + contracts[contract] = version + cls.__sw_contracts__ = contracts + cls.__sw_kinds__ = list(dict.fromkeys([*vars(cls).get("__sw_kinds__", []), *kinds])) + return cls + + return mark + + def commands_of(adapter): """Every command on the adapter's class, by wire name. Kinds' base classes add their own.""" found = {} diff --git a/sdk/python/src/simplyworks_serverless/_hpack.py b/sdk/python/src/sw_serverless/_hpack.py similarity index 100% rename from sdk/python/src/simplyworks_serverless/_hpack.py rename to sdk/python/src/sw_serverless/_hpack.py diff --git a/sdk/python/src/simplyworks_serverless/_http2.py b/sdk/python/src/sw_serverless/_http2.py similarity index 99% rename from sdk/python/src/simplyworks_serverless/_http2.py rename to sdk/python/src/sw_serverless/_http2.py index 1d15ac4..cc1213c 100644 --- a/sdk/python/src/simplyworks_serverless/_http2.py +++ b/sdk/python/src/sw_serverless/_http2.py @@ -82,7 +82,7 @@ async def _start(self): headers = _hpack.encode([ (":method", "POST"), (":scheme", "http"), (":path", self._path), (":authority", "localhost"), ("content-type", "application/grpc"), ("te", "trailers"), - ("user-agent", "simplyworks-serverless-python"), + ("user-agent", "sw-serverless-python"), ]) await self._write( PREFACE diff --git a/sdk/python/src/simplyworks_serverless/_huffman.py b/sdk/python/src/sw_serverless/_huffman.py similarity index 100% rename from sdk/python/src/simplyworks_serverless/_huffman.py rename to sdk/python/src/sw_serverless/_huffman.py diff --git a/sdk/python/src/simplyworks_serverless/_runner.py b/sdk/python/src/sw_serverless/_runner.py similarity index 99% rename from sdk/python/src/simplyworks_serverless/_runner.py rename to sdk/python/src/sw_serverless/_runner.py index 215e638..dc67620 100644 --- a/sdk/python/src/simplyworks_serverless/_runner.py +++ b/sdk/python/src/sw_serverless/_runner.py @@ -19,7 +19,7 @@ from . import _adapter, _types, _wire from ._http2 import GrpcStream -SDK_VERSION = "10.1.0" +SDK_VERSION = "10.2.0" SDK_LANGUAGE = "python" PROTOCOL = 2 ATTACH = "/sw.serverless.v1.AdapterHost/Attach" @@ -344,7 +344,7 @@ async def _shutdown(self, shutdown): try: await asyncio.wait_for(_call(stop), max(0.1, deadline - time.monotonic())) except Exception: - logging.getLogger("simplyworks_serverless").warning("stop() failed", exc_info=True) + logging.getLogger("sw_serverless").warning("stop() failed", exc_info=True) running = [task for task, _ in list(self._running.values())] if running: await asyncio.wait(running, timeout=max(0.1, deadline - time.monotonic())) diff --git a/sdk/python/src/simplyworks_serverless/_types.py b/sdk/python/src/sw_serverless/_types.py similarity index 98% rename from sdk/python/src/simplyworks_serverless/_types.py rename to sdk/python/src/sw_serverless/_types.py index 2305580..d9db527 100644 --- a/sdk/python/src/simplyworks_serverless/_types.py +++ b/sdk/python/src/sw_serverless/_types.py @@ -3,7 +3,7 @@ The encoding is the protocol's, the same in every language: a string is its raw UTF-8 text, bytes are passed as they are, nothing is an empty payload, and anything else is JSON. A class can take charge of its own JSON with ``to_wire``/``from_wire`` and describe it with ``__sw_schema__`` — the -Bitween types do, to keep the property names the contract fixes. +types of a contract package do, to keep the property names the contract fixes. """ import base64 diff --git a/sdk/python/src/simplyworks_serverless/_wire.py b/sdk/python/src/sw_serverless/_wire.py similarity index 100% rename from sdk/python/src/simplyworks_serverless/_wire.py rename to sdk/python/src/sw_serverless/_wire.py diff --git a/sdk/python/tests/test_adapter.py b/sdk/python/tests/test_adapter.py index 36d3e9e..9efc65f 100644 --- a/sdk/python/tests/test_adapter.py +++ b/sdk/python/tests/test_adapter.py @@ -2,8 +2,8 @@ import json import unittest -import simplyworks_serverless as sw -from simplyworks_serverless import _adapter, _types +import sw_serverless as sw +from sw_serverless import _adapter, _types @dataclasses.dataclass @@ -81,6 +81,22 @@ def __init__(self, required): described = sw.describe(NeedsArgs) self.assertEqual(1, len(described["warnings"])) + def test_a_class_says_which_contract_and_kinds_it_implements(self): + @sw.implements("orders", 1, "processor") + class Processor: + @sw.command("Process") + def process(self, order: dict) -> dict: + return order + + @sw.implements("audit", 2, "sink") + class Both(Processor): + pass + + self.assertEqual({"orders": 1}, sw.describe(Processor)["contracts"]) + self.assertEqual(["processor"], sw.describe(Processor)["kinds"]) + self.assertEqual({"orders": 1, "audit": 2}, sw.describe(Both)["contracts"]) + self.assertEqual(["sink", "processor"], sw.describe(Both)["kinds"]) + def test_a_command_takes_at_most_one_argument(self): class TwoArgs: @sw.command diff --git a/sdk/python/tests/test_wire.py b/sdk/python/tests/test_wire.py index 60bb5ee..035c1fc 100644 --- a/sdk/python/tests/test_wire.py +++ b/sdk/python/tests/test_wire.py @@ -1,6 +1,6 @@ import unittest -from simplyworks_serverless import _hpack, _wire +from sw_serverless import _hpack, _wire class WireTests(unittest.TestCase): @@ -9,7 +9,7 @@ def test_a_frame_round_trips_with_every_kind_of_field(self): "token": "t", "protocol_version": 2, "capabilities": ["cancel", "command:Greet"], "commands": [{"name": "Greet", "returns_value": True, "input_schema": '{"type":"string"}'}], "settings": [{"name": "Url", "required": True, "secret": True}], - "kinds": ["handler"], "contracts": {"bitween": 1}}} + "kinds": ["processor"], "contracts": {"orders": 1}}} decoded = _wire.decode("AdapterFrame", _wire.encode("AdapterFrame", frame)) self.assertEqual(frame, decoded) From 238e3f4757701d66f282731ba569a8ce1d7bbb71 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 18:34:18 +0300 Subject: [PATCH 10/17] Resolve storage flags in Tooling, for any CLI built on it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Where an adapter is published — flags, then the -c file, then SWSL_* variables — was worked out in the sw-serverless project, so a CLI built on Tooling had to copy it. It is StorageResolver and StorageFlags in Tooling now; the command line's resolver delegates to it. --- SW.Serverless.Installer/Options.cs | 11 +- .../UploadOptionsResolver.cs | 127 ++------------- SW.Serverless.Tooling/StorageResolver.cs | 149 ++++++++++++++++++ 3 files changed, 167 insertions(+), 120 deletions(-) create mode 100644 SW.Serverless.Tooling/StorageResolver.cs diff --git a/SW.Serverless.Installer/Options.cs b/SW.Serverless.Installer/Options.cs index d98d647..79de227 100644 --- a/SW.Serverless.Installer/Options.cs +++ b/SW.Serverless.Installer/Options.cs @@ -6,11 +6,6 @@ namespace SW.Serverless.Installer { - public class FileData - { - public ServerlessUploadOptions CloudFiles { get; set; } - } - /// /// Where the adapters live. Shared by publishing and by the promote, versions and withdraw /// commands, so every command finds the store the same way: flags, then -c, then SWSL_*. @@ -36,6 +31,12 @@ public class StorageCliOptions [Option('c', "cloudfilesconfigpath", HelpText = "Json cloud files config path. Oracle Cloud needs one; Google Cloud needs one or the SWSL_GC_* environment variables.")] public string CloudFilesConfigPath { get; set; } + + public StorageFlags ToFlags() => new() + { + Provider = Provider, AccessKeyId = AccessKeyId, SecretAccessKey = SecretAccessKey, + BucketName = BucketName, ServiceUrl = ServiceUrl, CloudFilesConfigPath = CloudFilesConfigPath, + }; } public class CliOptions : StorageCliOptions diff --git a/SW.Serverless.Installer/UploadOptionsResolver.cs b/SW.Serverless.Installer/UploadOptionsResolver.cs index 35e1e0a..2717817 100644 --- a/SW.Serverless.Installer/UploadOptionsResolver.cs +++ b/SW.Serverless.Installer/UploadOptionsResolver.cs @@ -1,132 +1,29 @@ using System; using System.Collections.Generic; -using Newtonsoft.Json; -using SW.PrimitiveTypes; namespace SW.Serverless.Installer { /// - /// Works out where an adapter is published to, from three sources in order of precedence: - /// command-line flags, then the JSON config file (-c), then SWSL_* environment variables. - /// - /// Environment variables are the last resort so a CI job can keep its keys in secrets rather - /// than on a command line, where they end up in shell history and process listings. + /// The command line's storage flags, resolved by in Tooling, which + /// other tools built on it share: flags, then the -c file, then SWSL_* environment variables. /// public static class UploadOptionsResolver { - /// Every environment variable read, with what it supplies. Used by the README. - public static readonly IReadOnlyDictionary EnvironmentVariables = new Dictionary - { - [Env.Provider] = "Storage provider (s3, as, oc, gc, local)", - [Env.AccessKey] = "Access key", - [Env.SecretKey] = "Secret access key", - [Env.Bucket] = "Bucket (container) name", - [Env.ServiceUrl] = "Service URL", - [Env.Region] = "Region", - [Env.GcProjectId] = "Google Cloud project_id", - [Env.GcPrivateKeyId] = "Google Cloud private_key_id", - [Env.GcPrivateKey] = "Google Cloud private_key (PEM; literal \\n is accepted for line breaks)", - [Env.GcClientEmail] = "Google Cloud client_email", - [Env.GcClientId] = "Google Cloud client_id", - [Env.GcClientX509CertUrl] = "Google Cloud client_x509_cert_url", - [Env.PublishedBy] = "Who is publishing, recorded in the catalog (then GITHUB_ACTOR, then the user name)", - }; + public static IReadOnlyDictionary EnvironmentVariables => StorageResolver.EnvironmentVariables; - public static class Env + public static ServerlessUploadOptions Resolve(StorageCliOptions options, string configJson, Func environment = null) { - public const string Provider = "SWSL_PROVIDER"; - public const string AccessKey = "SWSL_ACCESS_KEY"; - public const string SecretKey = "SWSL_SECRET_KEY"; - public const string Bucket = "SWSL_BUCKET"; - public const string ServiceUrl = "SWSL_SERVICE_URL"; - public const string Region = "SWSL_REGION"; - public const string GcProjectId = "SWSL_GC_PROJECT_ID"; - public const string GcPrivateKeyId = "SWSL_GC_PRIVATE_KEY_ID"; - public const string GcPrivateKey = "SWSL_GC_PRIVATE_KEY"; - public const string GcClientEmail = "SWSL_GC_CLIENT_EMAIL"; - public const string GcClientId = "SWSL_GC_CLIENT_ID"; - public const string GcClientX509CertUrl = "SWSL_GC_CLIENT_X509_CERT_URL"; - public const string PublishedBy = "SWSL_PUBLISHED_BY"; - public const string GitHubActor = "GITHUB_ACTOR"; - } - - /// The parsed command line: a publish, or one of the commands that only needs the store. - /// The contents of the -c file, or null when none was given. - /// Reads an environment variable; null for the process environment. - public static ServerlessUploadOptions Resolve( - StorageCliOptions options, string configJson, Func environment = null) - { - environment ??= Environment.GetEnvironmentVariable; - - ServerlessUploadOptions file = null; - if (configJson != null) + var resolved = StorageResolver.Resolve(options.ToFlags(), configJson, environment); + if (options is CliOptions publish) { - if (string.IsNullOrWhiteSpace(configJson)) - throw new SWException($"Invalid cloud Files config path, {options.CloudFilesConfigPath}"); - - file = JsonConvert.DeserializeObject(configJson)?.CloudFiles - ?? throw new SWException( - $"The cloud files config {options.CloudFilesConfigPath} has no \"CloudFiles\" section."); + resolved.Version = publish.Version; + resolved.AdapterId = publish.AdapterId; + resolved.Kind = publish.Kind; } - - string Pick(string flag, string fromFile, string envName) => - FirstSet(flag, fromFile, envName == null ? null : environment(envName)); - - var publish = options as CliOptions; - - return new ServerlessUploadOptions - { - Version = publish?.Version, - AdapterId = publish?.AdapterId, - Kind = publish?.Kind, - - Provider = Pick(options.Provider, file?.Provider, Env.Provider), - AccessKeyId = Pick(options.AccessKeyId, file?.AccessKeyId, Env.AccessKey), - SecretAccessKey = Pick(options.SecretAccessKey, file?.SecretAccessKey, Env.SecretKey), - BucketName = Pick(options.BucketName, file?.BucketName, Env.Bucket), - ServiceUrl = Pick(options.ServiceUrl, file?.ServiceUrl, Env.ServiceUrl), - Region = Pick(null, file?.Region, Env.Region), - - // Oracle: config file only, as before. - FingerPrint = file?.FingerPrint, - TenantId = file?.TenantId, - UserId = file?.UserId, - RSAKey = file?.RSAKey, - NamespaceName = file?.NamespaceName, - - // Google Cloud service account. These were accepted in the config file but never - // passed on, so a gc publish always failed to authenticate. - ProjectId = Pick(null, file?.ProjectId, Env.GcProjectId), - PrivateKeyId = Pick(null, file?.PrivateKeyId, Env.GcPrivateKeyId), - PrivateKey = UnescapeNewlines(Pick(null, file?.PrivateKey, Env.GcPrivateKey)), - ClientEmail = Pick(null, file?.ClientEmail, Env.GcClientEmail), - ClientId = Pick(null, file?.ClientId, Env.GcClientId), - ClientX509CertUrl = Pick(null, file?.ClientX509CertUrl, Env.GcClientX509CertUrl), - }; - } - - /// - /// Who a version is recorded as published by. Informational only — nothing is authorised - /// by it — so it falls back as far as the user name rather than failing a publish. - /// - public static string ResolvePublishedBy(string flag, Func environment = null) - { - environment ??= Environment.GetEnvironmentVariable; - return FirstSet(flag, environment(Env.PublishedBy), environment(Env.GitHubActor), Environment.UserName); - } - - private static string FirstSet(params string[] values) - { - foreach (var value in values) - if (!string.IsNullOrWhiteSpace(value)) return value; - return null; + return resolved; } - /// - /// A PEM key copied out of a service-account JSON file keeps its newlines as literal "\n", - /// which is how it usually arrives in an environment variable or CI secret. - /// - private static string UnescapeNewlines(string key) => - key != null && !key.Contains('\n') && key.Contains("\\n") ? key.Replace("\\n", "\n") : key; + public static string ResolvePublishedBy(string flag, Func environment = null) => + StorageResolver.ResolvePublishedBy(flag, environment); } } diff --git a/SW.Serverless.Tooling/StorageResolver.cs b/SW.Serverless.Tooling/StorageResolver.cs new file mode 100644 index 0000000..2fe89cf --- /dev/null +++ b/SW.Serverless.Tooling/StorageResolver.cs @@ -0,0 +1,149 @@ +using System; +using System.Collections.Generic; +using Newtonsoft.Json; +using SW.PrimitiveTypes; + +namespace SW.Serverless.Installer +{ + /// The storage flags a command line takes: -p, -a, -s, -b, -u and -c in sw-serverless. + public class StorageFlags + { + public string Provider { get; set; } + public string AccessKeyId { get; set; } + public string SecretAccessKey { get; set; } + public string BucketName { get; set; } + public string ServiceUrl { get; set; } + + /// A JSON file with a "CloudFiles" section; Oracle Cloud needs one, Google Cloud one or the SWSL_GC_* variables. + public string CloudFilesConfigPath { get; set; } + + /// The storage these flags, the config file they name and the SWSL_* variables point at. + public ServerlessUploadOptions Resolve(Func environment = null) => + StorageResolver.Resolve(this, CloudFilesConfigPath == null ? null : System.IO.File.ReadAllText(CloudFilesConfigPath), environment); + } + + /// The config file -c names: its "CloudFiles" section. + public class FileData + { + public ServerlessUploadOptions CloudFiles { get; set; } + } + + /// + /// Works out where an adapter is published to, from three sources in order of precedence: + /// command-line flags, then the JSON config file (-c), then SWSL_* environment variables. + /// + /// Environment variables are the last resort so a CI job can keep its keys in secrets rather + /// than on a command line, where they end up in shell history and process listings. + /// + public static class StorageResolver + { + /// Every environment variable read, with what it supplies. Used by the README. + public static readonly IReadOnlyDictionary EnvironmentVariables = new Dictionary + { + [Env.Provider] = "Storage provider (s3, as, oc, gc, local)", + [Env.AccessKey] = "Access key", + [Env.SecretKey] = "Secret access key", + [Env.Bucket] = "Bucket (container) name", + [Env.ServiceUrl] = "Service URL", + [Env.Region] = "Region", + [Env.GcProjectId] = "Google Cloud project_id", + [Env.GcPrivateKeyId] = "Google Cloud private_key_id", + [Env.GcPrivateKey] = "Google Cloud private_key (PEM; literal \\n is accepted for line breaks)", + [Env.GcClientEmail] = "Google Cloud client_email", + [Env.GcClientId] = "Google Cloud client_id", + [Env.GcClientX509CertUrl] = "Google Cloud client_x509_cert_url", + [Env.PublishedBy] = "Who is publishing, recorded in the catalog (then GITHUB_ACTOR, then the user name)", + }; + + public static class Env + { + public const string Provider = "SWSL_PROVIDER"; + public const string AccessKey = "SWSL_ACCESS_KEY"; + public const string SecretKey = "SWSL_SECRET_KEY"; + public const string Bucket = "SWSL_BUCKET"; + public const string ServiceUrl = "SWSL_SERVICE_URL"; + public const string Region = "SWSL_REGION"; + public const string GcProjectId = "SWSL_GC_PROJECT_ID"; + public const string GcPrivateKeyId = "SWSL_GC_PRIVATE_KEY_ID"; + public const string GcPrivateKey = "SWSL_GC_PRIVATE_KEY"; + public const string GcClientEmail = "SWSL_GC_CLIENT_EMAIL"; + public const string GcClientId = "SWSL_GC_CLIENT_ID"; + public const string GcClientX509CertUrl = "SWSL_GC_CLIENT_X509_CERT_URL"; + public const string PublishedBy = "SWSL_PUBLISHED_BY"; + public const string GitHubActor = "GITHUB_ACTOR"; + } + + /// The storage flags a command was given. + /// The contents of the -c file, or null when none was given. + /// Reads an environment variable; null for the process environment. + public static ServerlessUploadOptions Resolve( + StorageFlags options, string configJson, Func environment = null) + { + environment ??= Environment.GetEnvironmentVariable; + + ServerlessUploadOptions file = null; + if (configJson != null) + { + if (string.IsNullOrWhiteSpace(configJson)) + throw new SWException($"Invalid cloud Files config path, {options.CloudFilesConfigPath}"); + + file = JsonConvert.DeserializeObject(configJson)?.CloudFiles + ?? throw new SWException( + $"The cloud files config {options.CloudFilesConfigPath} has no \"CloudFiles\" section."); + } + + string Pick(string flag, string fromFile, string envName) => + FirstSet(flag, fromFile, envName == null ? null : environment(envName)); + + return new ServerlessUploadOptions + { + Provider = Pick(options.Provider, file?.Provider, Env.Provider), + AccessKeyId = Pick(options.AccessKeyId, file?.AccessKeyId, Env.AccessKey), + SecretAccessKey = Pick(options.SecretAccessKey, file?.SecretAccessKey, Env.SecretKey), + BucketName = Pick(options.BucketName, file?.BucketName, Env.Bucket), + ServiceUrl = Pick(options.ServiceUrl, file?.ServiceUrl, Env.ServiceUrl), + Region = Pick(null, file?.Region, Env.Region), + + // Oracle: config file only, as before. + FingerPrint = file?.FingerPrint, + TenantId = file?.TenantId, + UserId = file?.UserId, + RSAKey = file?.RSAKey, + NamespaceName = file?.NamespaceName, + + // Google Cloud service account. These were accepted in the config file but never + // passed on, so a gc publish always failed to authenticate. + ProjectId = Pick(null, file?.ProjectId, Env.GcProjectId), + PrivateKeyId = Pick(null, file?.PrivateKeyId, Env.GcPrivateKeyId), + PrivateKey = UnescapeNewlines(Pick(null, file?.PrivateKey, Env.GcPrivateKey)), + ClientEmail = Pick(null, file?.ClientEmail, Env.GcClientEmail), + ClientId = Pick(null, file?.ClientId, Env.GcClientId), + ClientX509CertUrl = Pick(null, file?.ClientX509CertUrl, Env.GcClientX509CertUrl), + }; + } + + /// + /// Who a version is recorded as published by. Informational only — nothing is authorised + /// by it — so it falls back as far as the user name rather than failing a publish. + /// + public static string ResolvePublishedBy(string flag, Func environment = null) + { + environment ??= Environment.GetEnvironmentVariable; + return FirstSet(flag, environment(Env.PublishedBy), environment(Env.GitHubActor), Environment.UserName); + } + + private static string FirstSet(params string[] values) + { + foreach (var value in values) + if (!string.IsNullOrWhiteSpace(value)) return value; + return null; + } + + /// + /// A PEM key copied out of a service-account JSON file keeps its newlines as literal "\n", + /// which is how it usually arrives in an environment variable or CI secret. + /// + private static string UnescapeNewlines(string key) => + key != null && !key.Contains('\n') && key.Contains("\\n") ? key.Replace("\\n", "\n") : key; + } +} From 0bbfcb648e9919d75f6cfe8bb8f3f4be6f6dd1b1 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 18:34:18 +0300 Subject: [PATCH 11/17] Describe .NET adapters from a single-file sw-serverless A self-contained single-file build has no runtime assemblies on disk, so the older publish form read an adapter's types against nothing and took every adapter for a classic one with no kinds. It now reads them against an installed .NET when its own runtime has no files: DOTNET_ROOT, the usual folders, or what dotnet --list-runtimes reports. A .NET adapter needs one to run anyway. --- SW.Serverless.Tooling/AdapterDescriber.cs | 61 ++++++++++++++++++++++- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/SW.Serverless.Tooling/AdapterDescriber.cs b/SW.Serverless.Tooling/AdapterDescriber.cs index 8c5394e..0da4abc 100644 --- a/SW.Serverless.Tooling/AdapterDescriber.cs +++ b/SW.Serverless.Tooling/AdapterDescriber.cs @@ -47,6 +47,62 @@ public static class AdapterDescriber /// assembly simply describes as a classic adapter with no declared kind — which is what /// every adapter was before any of this existed. /// + /// + /// A folder of the .NET runtime's own assemblies, for type and attribute references to + /// resolve against: this process's runtime when it runs from files, and otherwise — a + /// self-contained single-file build of the CLI has none on disk — an installed .NET, which a + /// .NET adapter needs to run anyway. + /// + internal static string RuntimeDirectory() + { + static bool Usable(string folder) => !string.IsNullOrEmpty(folder) && File.Exists(Path.Combine(folder, "System.Runtime.dll")); + + // Empty in a single-file build, which is the case the fallbacks below are for. +#pragma warning disable IL3000 + var own = Path.GetDirectoryName(typeof(object).Assembly.Location); +#pragma warning restore IL3000 + if (Usable(own)) return own; + + foreach (var root in new[] + { + Environment.GetEnvironmentVariable("DOTNET_ROOT"), + OperatingSystem.IsWindows() ? @"C:\Program Files\dotnet" : "/usr/local/share/dotnet", + "/usr/share/dotnet", "/usr/lib/dotnet", + }) + { + var shared = string.IsNullOrEmpty(root) ? null : Path.Combine(root, "shared", "Microsoft.NETCore.App"); + if (shared == null || !Directory.Exists(shared)) continue; + var newest = Directory.GetDirectories(shared) + .Where(Usable) + .OrderByDescending(d => Version.TryParse(Path.GetFileName(d).Split('-')[0], out var v) ? v : new Version()) + .FirstOrDefault(); + if (newest != null) return newest; + } + + // Wherever the dotnet on the PATH keeps its runtimes. + try + { + using var dotnet = System.Diagnostics.Process.Start(new System.Diagnostics.ProcessStartInfo("dotnet", "--list-runtimes") + { + RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, + }); + var output = dotnet!.StandardOutput.ReadToEnd(); + dotnet.WaitForExit(); + return output.Split('\n') + .Select(l => System.Text.RegularExpressions.Regex.Match(l, @"^Microsoft\.NETCore\.App (\S+) \[(.+)\]")) + .Where(m => m.Success) + .Select(m => (Version: Version.TryParse(m.Groups[1].Value.Split('-')[0], out var v) ? v : new Version(), Folder: Path.Combine(m.Groups[2].Value.Trim(), m.Groups[1].Value))) + .Where(r => Usable(r.Folder)) + .OrderByDescending(r => r.Version) + .Select(r => r.Folder) + .FirstOrDefault(); + } + catch + { + return null; + } + } + public static AdapterDescription Describe(string publishDirectory, string entryAssembly) { var description = new AdapterDescription(); @@ -57,9 +113,10 @@ public static AdapterDescription Describe(string publishDirectory, string entryA if (!File.Exists(assemblyPath)) return description; // Everything beside it, plus the runtime, so interface and attribute types resolve. + var runtime = RuntimeDirectory() + ?? throw new InvalidOperationException("No .NET runtime found to read the adapter's types against."); var assemblies = Directory.GetFiles(publishDirectory, "*.dll", SearchOption.AllDirectories) - .Concat(Directory.GetFiles( - Path.GetDirectoryName(typeof(object).Assembly.Location)!, "*.dll")) + .Concat(Directory.GetFiles(runtime, "*.dll")) .Distinct() .ToList(); From e95770c5901ed11f4033e5637099ce44dab15940 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 18:34:18 +0300 Subject: [PATCH 12/17] Release sw-serverless as self-contained binaries on GitHub A tag cli-v builds the command for Linux (x64, arm64, musl x64), macOS (x64, arm64) and Windows (x64), single-file and self-contained, and publishes them on a release with SHA256SUMS. scripts/install-cli.sh installs the latest, checking it against the checksums. --- .github/workflows/cli-release.yml | 71 +++++++++++++++++++++++++++++++ scripts/install-cli.sh | 38 +++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 .github/workflows/cli-release.yml create mode 100755 scripts/install-cli.sh diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml new file mode 100644 index 0000000..f3a4a37 --- /dev/null +++ b/.github/workflows/cli-release.yml @@ -0,0 +1,71 @@ +name: sw-serverless CLI release + +# A tag cli-v (cli-v10.2.0) publishes the sw-serverless command as self-contained +# binaries on a GitHub release: nothing needs .NET installed to run them. +on: + push: + tags: ["cli-v*"] + workflow_dispatch: + inputs: + version: + description: "Version to release, e.g. 10.2.0 (tagged cli-v)" + required: true + +permissions: + contents: write + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + rid: [linux-x64, linux-arm64, linux-musl-x64, osx-x64, osx-arm64, win-x64] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: "10.0.x" + - name: Version + id: version + run: | + v="${{ github.event.inputs.version }}" + [ -z "$v" ] && v="${GITHUB_REF_NAME#cli-v}" + echo "value=$v" >> "$GITHUB_OUTPUT" + - name: Publish + run: > + dotnet publish SW.Serverless.Installer -c Release -r ${{ matrix.rid }} --self-contained + -p:PublishSingleFile=true -p:IncludeNativeLibrariesForSelfExtract=true -p:DebugType=none + -p:Version=${{ steps.version.outputs.value }} -o out + - name: Package + run: | + mkdir -p dist + cp README.md LICENSE out/ 2>/dev/null || true + if [[ "${{ matrix.rid }}" == win-* ]]; then + (cd out && zip -q -r ../dist/sw-serverless-${{ matrix.rid }}.zip .) + else + tar -czf dist/sw-serverless-${{ matrix.rid }}.tar.gz -C out . + fi + - uses: actions/upload-artifact@v4 + with: + name: sw-serverless-${{ matrix.rid }} + path: dist/* + + release: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + - name: Checksums + run: cd dist && sha256sum * > SHA256SUMS && cat SHA256SUMS + - name: Release + env: + GH_TOKEN: ${{ github.token }} + run: | + v="${{ github.event.inputs.version }}" + [ -z "$v" ] && v="${GITHUB_REF_NAME#cli-v}" + gh release create "cli-v$v" dist/* --title "sw-serverless $v" --target "${{ github.sha }}" \ + --notes "The sw-serverless command, self-contained for each platform. Install: curl -fsSL https://raw.githubusercontent.com/${{ github.repository }}/main/scripts/install-cli.sh | sh — or download the archive for your platform and check it against SHA256SUMS." diff --git a/scripts/install-cli.sh b/scripts/install-cli.sh new file mode 100755 index 0000000..82e2b7a --- /dev/null +++ b/scripts/install-cli.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# Installs the sw-serverless command from the latest GitHub release (or SW_SERVERLESS_VERSION), +# checking it against the release's SHA256SUMS. Installs to ~/.local/bin unless INSTALL_DIR is set. +set -eu + +repo="simplify9/SW-Serverless" +name="sw-serverless" +dir="${INSTALL_DIR:-$HOME/.local/bin}" + +os=$(uname -s); arch=$(uname -m) +case "$os" in + Linux) os=linux; if ldd --version 2>&1 | grep -qi musl; then os=linux-musl; fi ;; + Darwin) os=osx ;; + *) echo "Unsupported system $os; download a release from https://github.com/$repo/releases" >&2; exit 1 ;; +esac +case "$arch" in + x86_64|amd64) arch=x64 ;; + arm64|aarch64) arch=arm64 ;; + *) echo "Unsupported architecture $arch" >&2; exit 1 ;; +esac +rid="$os-$arch" + +if [ -n "${SW_SERVERLESS_VERSION:-}" ]; then tag="cli-v$SW_SERVERLESS_VERSION" +else tag=$(curl -fsSL "https://api.github.com/repos/$repo/releases" | grep -o '"tag_name": *"cli-v[^"]*"' | head -1 | sed 's/.*"\(cli-v[^"]*\)"/\1/'); fi +[ -n "$tag" ] || { echo "No sw-serverless release found" >&2; exit 1; } + +work=$(mktemp -d); trap 'rm -rf "$work"' EXIT +base="https://github.com/$repo/releases/download/$tag" +curl -fsSL "$base/$name-$rid.tar.gz" -o "$work/$name-$rid.tar.gz" +curl -fsSL "$base/SHA256SUMS" -o "$work/SHA256SUMS" +(cd "$work" && grep " $name-$rid.tar.gz\$" SHA256SUMS | (sha256sum -c - 2>/dev/null || shasum -a 256 -c -)) >/dev/null \ + || { echo "Checksum mismatch for $name-$rid.tar.gz" >&2; exit 1; } + +mkdir -p "$work/x" "$dir" +tar -xzf "$work/$name-$rid.tar.gz" -C "$work/x" +install -m 755 "$work/x/$name" "$dir/$name" +echo "Installed $name ${tag#cli-v} to $dir/$name" +case ":$PATH:" in *":$dir:"*) ;; *) echo "Add $dir to your PATH." ;; esac From de3dc1ecdd8fa19320c3fcbe4134ff1494c9398c Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 18:53:56 +0300 Subject: [PATCH 13/17] Start resident adapters beside the read loop, in every SDK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Start was awaited inside the loop that reads the host's frames, so a start that asked the host for anything — its state, an event's ack — waited for an answer that loop could never read, and the adapter hung. Start now runs beside it: commands wait until it has returned, a stop waits for it within its own deadline, and a start that fails stops the adapter as before. The Python and Node test residents now read and write state from start. --- SW.Serverless.Sdk/AdapterContractAttribute.cs | 2 +- SW.Serverless.Sdk/Resident/AdapterStatus.cs | 2 +- SW.Serverless.Sdk/Resident/Handshake.cs | 3 +- SW.Serverless.Sdk/Resident/IAdapterContext.cs | 2 +- .../Resident/IResidentAdapter.cs | 2 +- SW.Serverless.Sdk/Resident/ResidentRunner.cs | 36 +- SW.Serverless.Sdk/Runner.cs | 2 +- SW.Serverless.Sdk/SW.Serverless.Sdk.csproj | 4 +- .../NodeAdapters/resident.js | 8 +- .../PythonAdapters/resident.py | 6 +- docs/resident-adapters-design.md | 1853 ----------------- sdk/node/src/index.js | 15 +- sdk/python/src/sw_serverless/_runner.py | 24 +- 13 files changed, 82 insertions(+), 1877 deletions(-) delete mode 100644 docs/resident-adapters-design.md diff --git a/SW.Serverless.Sdk/AdapterContractAttribute.cs b/SW.Serverless.Sdk/AdapterContractAttribute.cs index 7c7fd7d..1bad830 100644 --- a/SW.Serverless.Sdk/AdapterContractAttribute.cs +++ b/SW.Serverless.Sdk/AdapterContractAttribute.cs @@ -4,7 +4,7 @@ namespace SW.Serverless.Sdk { /// /// A contract this adapter implements, and its version — e.g. "orders" 1. Reported - /// in the adapter's description and handshake, and checked by serverless test. + /// in the adapter's description and handshake, and checked by sw-serverless test. /// [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] public class AdapterContractAttribute : Attribute diff --git a/SW.Serverless.Sdk/Resident/AdapterStatus.cs b/SW.Serverless.Sdk/Resident/AdapterStatus.cs index b1a7e74..d55086c 100644 --- a/SW.Serverless.Sdk/Resident/AdapterStatus.cs +++ b/SW.Serverless.Sdk/Resident/AdapterStatus.cs @@ -5,7 +5,7 @@ namespace SW.Serverless.Sdk.Resident { /// /// What a heartbeat answers with. Liveness alone cannot separate "alive but disconnected" - /// from "connected but receiving nothing" — see the design doc, section 6.4. + /// from "connected but receiving nothing". /// public class AdapterStatus { diff --git a/SW.Serverless.Sdk/Resident/Handshake.cs b/SW.Serverless.Sdk/Resident/Handshake.cs index 9c06e10..14f3e29 100644 --- a/SW.Serverless.Sdk/Resident/Handshake.cs +++ b/SW.Serverless.Sdk/Resident/Handshake.cs @@ -4,8 +4,7 @@ namespace SW.Serverless.Sdk.Resident { /// /// Written by the host as a single JSON line on the child's stdin, immediately after spawn. - /// It travels here rather than on argv so that nothing secret is visible in `ps aux` — - /// see the design doc, section 14.3. + /// It travels here rather than on argv so that nothing secret is visible in `ps aux`. /// public class Handshake { diff --git a/SW.Serverless.Sdk/Resident/IAdapterContext.cs b/SW.Serverless.Sdk/Resident/IAdapterContext.cs index 1182b54..e1612fa 100644 --- a/SW.Serverless.Sdk/Resident/IAdapterContext.cs +++ b/SW.Serverless.Sdk/Resident/IAdapterContext.cs @@ -9,7 +9,7 @@ public class PublishResult { public bool Accepted { get; set; } - /// The host's id for what it persisted, e.g. an Xchange id. + /// The host's id for what it persisted, e.g. a message id. public string Reference { get; set; } public string Error { get; set; } diff --git a/SW.Serverless.Sdk/Resident/IResidentAdapter.cs b/SW.Serverless.Sdk/Resident/IResidentAdapter.cs index 278525d..838360b 100644 --- a/SW.Serverless.Sdk/Resident/IResidentAdapter.cs +++ b/SW.Serverless.Sdk/Resident/IResidentAdapter.cs @@ -26,7 +26,7 @@ public interface IResidentAdapter /// /// Optional. Implement on a POOLED adapter to clear per-session state between checkouts — - /// without this, process-static state leaks across requests (design doc, section 14.5). + /// without this, process-static state leaks across requests. /// public interface IResettable { diff --git a/SW.Serverless.Sdk/Resident/ResidentRunner.cs b/SW.Serverless.Sdk/Resident/ResidentRunner.cs index 8e4133f..1f61f78 100644 --- a/SW.Serverless.Sdk/Resident/ResidentRunner.cs +++ b/SW.Serverless.Sdk/Resident/ResidentRunner.cs @@ -23,7 +23,7 @@ namespace SW.Serverless.Sdk.Resident /// /// The adapter side of protocol 2. Dials the host over a Unix domain socket (Linux / macOS) /// or a named pipe (Windows) and pumps one bidirectional gRPC stream. - /// See the design doc, section 15. + /// /// public sealed class ResidentRunner : IAdapterContext { @@ -138,7 +138,7 @@ async Task RunCoreAsync() { var line = await Console.In.ReadLineAsync(); - // A null read means the parent is gone. v1 spun here forever — design doc 3, item 7. + // A null read means the parent is gone. v1 spun here forever. if (line == null) throw new IOException("stdin closed before the handshake arrived; parent process is gone."); @@ -304,7 +304,7 @@ async Task ReadLoopAsync(IAsyncStreamReader stream) case HostFrame.BodyOneofCase.Invoke: // Deliberately not awaited: a slow command must not block the read loop, - // which is the whole point of multiplexing (design doc 3, item 1). + // which is the whole point of multiplexing. StartInvoke(frame.Id, frame.Invoke); break; @@ -351,7 +351,10 @@ static async Task MoveNextSafe(IAsyncStreamReader stream) catch (RpcException ex) when (ex.StatusCode == StatusCode.Cancelled) { return false; } } - async Task OnReadyAsync(Ready ready) + /// The resident adapter's start, which every command waits for. + Task started = Task.CompletedTask; + + Task OnReadyAsync(Ready ready) { startupValues = new Dictionary(ready.StartupValues, StringComparer.OrdinalIgnoreCase); adapterValues = new Dictionary(ready.AdapterValues, StringComparer.OrdinalIgnoreCase); @@ -363,8 +366,25 @@ async Task OnReadyAsync(Ready ready) resident = handler as IResidentAdapter; resettable = handler as IResettable; + // Started beside the read loop, not inside it: a start that publishes an event or reads + // its state waits for the host's answer, which only the read loop can take in. Commands + // wait for it to finish; if it fails, the adapter stops, as it did when it was awaited here. if (resident != null) - await resident.StartAsync(this, stopping.Token); + started = Task.Run(async () => + { + try + { + await resident.StartAsync(this, stopping.Token); + } + catch (Exception ex) + { + AdapterLogger.LogError(ex, "The resident adapter failed to start."); + Environment.ExitCode = 1; + stopping.Cancel(); + throw; + } + }); + return Task.CompletedTask; } /// @@ -395,6 +415,7 @@ async Task OnInvokeAsync(long id, Invoke invoke, CancellationToken cancellationT { if (handler == null) throw new InvalidOperationException("The adapter has not been made ready yet."); + await started.ConfigureAwait(false); if (!commands.TryGetValue(invoke.Command, out var method)) throw new MissingMethodException(handlerType.FullName, invoke.Command); @@ -522,6 +543,11 @@ async Task OnShutdownAsync(Shutdown shutdown) // its consume loop and PublishAsync calls observe — was already cancelled, so there // was nothing left to finish. Ask it to stop, let it drain, and only then cancel. var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(shutdown.Drain ? 30 : 5); + + // A start still running finishes first, as it did when start held up the read loop: a + // stop that overtook it would leave half of what start set up in place. + try { await Task.WhenAny(started, Task.Delay(deadline - DateTime.UtcNow)); } catch { } + if (resident != null) { using var cts = new CancellationTokenSource(deadline - DateTime.UtcNow); diff --git a/SW.Serverless.Sdk/Runner.cs b/SW.Serverless.Sdk/Runner.cs index d97d711..31031e0 100644 --- a/SW.Serverless.Sdk/Runner.cs +++ b/SW.Serverless.Sdk/Runner.cs @@ -58,7 +58,7 @@ public static void MockRun(object commandHandler, ServerlessOptions serverlessOp /// /// Entry point for an adapter that STAYS RUNNING. Same zip, same spawn, same installation — - /// only this line differs from Run(). See the design doc, section 15.2. + /// only this line differs from Run(). /// public static Task RunResident(object commandHandler) => Resident.ResidentRunner.RunAsync(commandHandler); diff --git a/SW.Serverless.Sdk/SW.Serverless.Sdk.csproj b/SW.Serverless.Sdk/SW.Serverless.Sdk.csproj index 3ed3672..c33865f 100644 --- a/SW.Serverless.Sdk/SW.Serverless.Sdk.csproj +++ b/SW.Serverless.Sdk/SW.Serverless.Sdk.csproj @@ -5,8 +5,8 @@ SimplyWorks.Serverless.Sdk Simplify9 SimplyWorks.Serverless.Sdk - https://github.com/simplify9/Serverless - https://github.com/simplify9/Serverless + https://github.com/simplify9/SW-Serverless + https://github.com/simplify9/SW-Serverless MIT icon.png Simplify9 diff --git a/SW.Serverless.UnitTests/NodeAdapters/resident.js b/SW.Serverless.UnitTests/NodeAdapters/resident.js index 597edb0..90b11fd 100644 --- a/SW.Serverless.UnitTests/NodeAdapters/resident.js +++ b/SW.Serverless.UnitTests/NodeAdapters/resident.js @@ -16,12 +16,16 @@ class Resident { sw.expect("Name", { default: "resident" }); } - async start() { this.started = true; } + async start() { + // Asking the host from start: answered only because start runs beside the read loop. + await sw.context().setState("started", "yes"); + this.started = true; + } async stop() { this.started = false; } status() { return { connected: this.started, state: "Listening", details: { resets: this.resets.length } }; } reset(sessionId) { this.resets.push(sessionId); } - isStarted() { return this.started; } + async isStarted() { return this.started && (await sw.context().getState("started")) === "yes"; } publish(text) { return sw.context().publish(text, { dedupeKey: "k-" + text, contentType: "text/plain", endpoint: "tests" }); } diff --git a/SW.Serverless.UnitTests/PythonAdapters/resident.py b/SW.Serverless.UnitTests/PythonAdapters/resident.py index 560a1c0..1ea1062 100644 --- a/SW.Serverless.UnitTests/PythonAdapters/resident.py +++ b/SW.Serverless.UnitTests/PythonAdapters/resident.py @@ -9,6 +9,8 @@ def __init__(self): sw.expect("Name", "resident") async def start(self): + # Asking the host from start: answered only because start runs beside the read loop. + await sw.context().set_state("started", "yes") self.started = True async def stop(self): @@ -21,8 +23,8 @@ def reset(self, session_id): self.resets.append(session_id) @sw.command("Started") - def is_started(self) -> bool: - return self.started + async def is_started(self) -> bool: + return self.started and await sw.context().get_state("started") == "yes" @sw.command("Publish") async def publish(self, text: str) -> str: diff --git a/docs/resident-adapters-design.md b/docs/resident-adapters-design.md deleted file mode 100644 index 0994b7b..0000000 --- a/docs/resident-adapters-design.md +++ /dev/null @@ -1,1853 +0,0 @@ -# Serverless Runtimes: Resident Adapters, Containers, and Observability - -> **Status:** design evaluation. If adopted, this **supersedes §4 (Plugin architecture) of -> `external-brokers-architecture.md`** — the provider *contract* and *capability* design there -> stays intact, but providers are hosted as serverless adapters instead of in-process -> `AssemblyLoadContext` plugins. - ---- - -## 1. Verdict - -### Document map — read this first - -This document was written in three passes. Later sections **revise** earlier ones; where they -conflict, the later section wins. - -```mermaid -flowchart LR - subgraph P1["Pass 1 — sections 1 to 12"] - A["Sec 2
3 types = 2 dimensions"] - B["Sec 3
hand-built framed stdio"] - C["Sec 8
docker run on the host"] - end - subgraph P2["Pass 2 — section 13
Kubernetes reframe"] - D["3 modes
Ephemeral / Resident / Orchestrated"] - E["one gRPC contract
adapter dials out"] - end - subgraph P3["Pass 3 — section 14
shared-library reality"] - F["Ephemeral is the
PRIMARY product
~190 binaries"] - G["Pooled resident
+ Phase 0 bug fixes"] - end - A -.superseded by.-> D - B -.superseded by.-> E - C -.superseded by.-> D - D --> F - E --> G -``` - -**Diagram key** — a node outlined in **green** is the recommended path, in **red** a trap or -rejected option, in **amber** something to watch. Everything else is neutral. - -*Sections 11 and 12 (resource limits, stdio performance) survive intact — but section 13 makes -most of section 11 unnecessary in Orchestrated mode, and section 13.3 removes the need for the -custom framing that section 12 was optimising.* - ---- - -### Where the connections live — the core of the verdict - -```mermaid -flowchart TB - subgraph OPT1["Option A — in-process plugin"] - direction TB - H1["Bitween host process
(one heap, one fate)"] - H1 --- K1["Kafka client
native buffers
64 MiB per partition"] - H1 --- R1["RabbitMQ client"] - H1 --- APP["Xchange pipeline
mappers, handlers, API"] - end - subgraph OPT2["Option B — adapter processes"] - direction TB - H2["Bitween host process
pipeline only"] - H2 <-.duplex stream.-> P1["kafka-adapter
memory-capped
restartable"] - H2 <-.duplex stream.-> P2["rabbitmq-adapter
memory-capped"] - end - OPT1 -->|"a provider crash or leak
takes the whole node down"| OPT2 -``` - -*Left: broker memory and crash risk land on the Bitween heap. Right: they land on a process you -can cap, restart, and place independently.* - -The idea is right, and it is the strongest available answer to the "install a bus provider at -runtime" requirement. But be clear about what is actually being bought and what it costs: - -**What you gain that the in-proc plugin design cannot give you:** - -| | In-proc plugin (§4 as written) | Resident serverless adapter | -|---|---|---| -| Install without redeploy | No — assembly loaded at startup | **Yes** — already the whole point of `Install()` | -| A provider whose only good client is not .NET | Impossible | Yes, via container launcher — but see 1.1: rarer than it sounds | -| Native deps (librdkafka) | Loads into the host process | **Isolated**, and cappable | -| Memory pressure from live connections | Lands on the Bitween node's heap | **Lands on a process you can `--memory` cap** | -| A provider that crashes | Can take the node down | Kills one adapter, supervisor restarts it | -| Provider versions side by side | ALC hell | Free — separate processes, separate dirs | - -### 1.1 Correction: "non-.NET" is the weakest argument for containers - -An earlier draft of this section claimed non-.NET clients made Pulsar, MQTT and others impossible -in-process. That is wrong, and it oversold the container launcher. Every broker on the roadmap has -a usable .NET client: - -| Broker | .NET client | Verdict | -|---|---|---| -| RabbitMQ | `RabbitMQ.Client` (official) | Pure managed. No container needed. | -| MQTT | **MQTTnet** | Mature and widely used. No container needed. | -| Pulsar | **DotPulsar** (official, under the Apache Pulsar project) | Covers the mainstream produce/consume path. The **Java** client is the reference implementation and leads on some features — check DotPulsar against what the Tuya pipeline actually needs before assuming either way. | -| Kafka | `Confluent.Kafka` | The standard choice — but it wraps **librdkafka**, a native library. That is a *native dependency* argument, not a language one. | -| Azure Event Hubs / Service Bus, Amazon SQS | first-party Azure and AWS SDKs | Pure managed. No container needed. | - -So the honest case for the container launcher is **not** language. It is: - -1. **Native dependencies** — librdkafka is `malloc`, invisible to `GCHeapHardLimit`, and platform-specific. This is the strongest reason and §11.1 depends on it. -2. **Runtime and target-framework independence** — the strongest reason of all, and it is entirely a .NET problem: Gateway's image already ships two runtimes (`COPY --from=aspnet:6.0`) because the adapter fleet multi-targets. See §14.2. -3. **Blast radius and resource limits for third-party provider code.** -4. **A specific .NET client falling short of what a client needs** — real, but narrow, and it should be established per case rather than assumed. - -Language is the fallback, not the premise. This also reinforces §11.3: `systemd-run` should ship -before `ContainerLauncher`, because most providers will never need a container at all. - ---- - -That last-but-one row directly answers the node-pressure worry in §10 of the architecture doc: -librdkafka's default `queued.max.messages.kbytes` is 64 MiB **per partition**. In-process, a -30-partition topic silently adds ~2 GB to the Bitween host. Out-of-process, it is a container -with a memory limit and a restart policy. - -**What it costs:** the current stdio protocol cannot carry this workload. That is the real -project — not the process model, not Docker. See §3. - -**Recommendation:** do it, and *replace* the in-proc plugin model rather than supporting both. -Two provider hosting models is the worst outcome — double the surface, double the bugs, and -every provider author has to pick. - ---- - -## 2. Reframe: two dimensions, not three types - -```mermaid -flowchart TB - subgraph AX["Two independent axes, not three types"] - direction LR - subgraph L["Lifecycle — a PROTOCOL concern"] - L1["Invocation
start, N calls, exit"] - L2["Resident
lives on, pushes events"] - end - subgraph W["Launcher — a PACKAGING concern"] - W1["LocalProcess
dotnet x.dll"] - W2["Container
docker run -i"] - end - end - L1 --- W1 - L1 --- W2 - L2 --- W1 - L2 --- W2 -``` - -*Docker is not a third type. `docker run -i` hands you the same three pipes, so it is a second -**launcher** for the same protocol. Keeping the axes separate means a resident RabbitMQ adapter -and an invocation-scoped mapper share one protocol implementation.* - -"Three types of adapter" will produce three divergent code paths. There are really two -independent axes: - -``` -AdapterRuntime - ├─ Lifecycle : Invocation | Resident - └─ Launcher : LocalProcess | Container ( | InProcess, for native.* ) -``` - -* **Lifecycle** is a *protocol* concern — who initiates, how many calls are in flight, does the - adapter push. -* **Launcher** is a *packaging* concern — how the process is spawned and constrained. - -Docker is not a third adapter type; it is a second launcher for the same protocol -(`docker run -i` gives you exactly the same stdin/stdout pipes). Keeping these orthogonal means -a resident RabbitMQ adapter and an invocation-scoped mapper share one protocol implementation, -and a container-packaged mapper works for free. - -**Where the flags live:** nowhere new. `GetAdapterMetadata` already reads an arbitrary -`IDictionary` off the cloud object and passes it into the adapter as -`AdapterValues`. Today it reads `EntryAssembly` and `Hash`. Add: - -| Metadata key | Values | Meaning | -|---|---|---| -| `Protocol` | absent \| `2` | absent ⇒ v1 line protocol, verbatim existing code path | -| `Lifecycle` | `invocation` (default) \| `resident` | | -| `Launcher` | `process` (default) \| `container` | | -| `Image` | e.g. `ghcr.io/…/kafka-adapter:1.4.0` | container launcher only | -| `MaxInFlight` | int | protocol credit window | -| `Capabilities` | csv | `subscribe,publish,topology,browse,query` — §3.8 of the arch doc | - -Backward compatibility is then trivial and *provable*: an adapter with no `Protocol` key takes -the existing code path byte for byte. Old adapters are already-published binaries against a -pinned SDK version; they never see the new code. - ---- - -## 3. Challenge 1 — the protocol is the whole project - -### The single-completion-source hazard, drawn - -This is the defect behind items 1 and 2 below. It is **live in Traxis production today** — see -section 14.3. - -```mermaid -sequenceDiagram - autonumber - participant G as Gateway host - participant T as the ONE tcs field - participant A as Adapter child process - G->>A: Track command - G->>T: store TCS-1 for Track - Note over A: adapter is slow, calls the carrier - Note over G: CommandTimeout fires at 30s - G->>T: TrySetException Timeout on TCS-1 - Note over A: child is NOT killed and keeps working - G->>A: GetLogs command from the finally block - G->>T: OVERWRITE with TCS-2 for GetLogs - A-->>G: late Track result arrives on stdout - G->>T: resolves TCS-2 using the Track payload - Note over G: GetLogs returns wrong-typed or garbage data -``` - -*There is no correlation id in the protocol, so stdout is pure FIFO trust. One timeout -permanently desynchronises the stream for the rest of that process's life.* - ---- - -### What the v1 protocol can and cannot carry - -```mermaid -flowchart LR - subgraph OK["Fits v1"] - O1["one call in flight"] - O2["UTF-8 text payloads"] - O3["host always initiates"] - O4["one correlation id
per PROCESS"] - end - subgraph NEED["A resident bus adapter needs"] - N1["many calls in flight"] - N2["raw bytes
Kafka and MQTT payloads"] - N3["adapter initiates
the push direction"] - N4["one correlation id
per MESSAGE"] - N5["heartbeat answered
DURING an invoke"] - end - O1 -.->|blocked| N1 - O2 -.->|blocked| N2 - O3 -.->|blocked| N3 - O4 -.->|blocked| N4 - O1 -.->|blocked| N5 -``` - -Verified against `SW.Serverless/Services/ServerlessService.cs` and `SW.Serverless.Sdk/Runner.cs`: - -1. **One call in flight, ever.** `ServerlessService` holds a single `taskCompletionSource` field, - overwritten on every `InvokeAsync`. There are no message ids, so responses cannot be - correlated. A resident bus adapter needs many concurrent operations. -2. **stdout is assumed to be responses only.** `OutputDataReceived` disposes - `invocationTimeoutTimer` on *any* line and resolves the pending TCS with it. An adapter that - spontaneously emits "I received a Kafka message" will cancel and corrupt an unrelated - in-flight invoke. Push is not merely unsupported — it is actively unsafe. -3. **The adapter suicides when idle.** `Runner.Run` arms `idleTimer` before every read and - throws `TimeoutException` from the timer callback (i.e. crashes the process) after - `ServerlessOptions.IdleTimeout`. A resident adapter that is quietly listening is, by - definition, idle. -4. **`CorrelationId` is a per-process startup value.** A resident adapter handles thousands of - correlations over its life. Correlation must move to the message envelope. -5. **The protocol cannot carry binary.** Frames are single UTF-8 lines with `\n` → `{{newline}}` - escaping and `#!#` delimiters. Kafka and MQTT payloads are `byte[]`. Base64 in a JSON string - would work but costs +33% and forces the whole payload through one console line. This is a - hard blocker, not a nice-to-have. -6. **Startup values are passed on `argv`, base64-encoded** — visible in `ps aux` to any local - user. That is already true today and already carries adapter secret properties; for broker - credentials it is not acceptable. Config must move to a first stdin frame. -7. **EOF spins hot.** In `Runner.Run`, `ReadLineAsync` returning `null` (parent process gone) - hits `if (input == null) continue;` — an infinite loop that re-arms and disposes a `Timer` on - every iteration. Today the idle timer eventually kills it; a resident adapter with no idle - timeout would spin a core forever as an orphan. **Null read must mean "parent died, exit".** - -### 3.1 Protocol v2 - -> **Amended by section 15.** The judgement below is about gRPC over **localhost TCP**, and it -> stands. Section 13.3 proposes gRPC over a **Unix domain socket / named pipe** — an IPC object, -> not a network socket — which keeps every property defended here. See section 15.1. - -Keep stdio. It is the design's quiet superpower: no ports, no port allocation, no bind -addresses, no auth token, no firewall rule, no container network config — and it works -identically for `dotnet x.dll` and `docker run -i`. gRPC-over-localhost buys better tooling and -costs all of that back. - -What changes is framing: - -``` -[ u32 length ][ u8 frameType ][ u32 headerLen ][ header: UTF-8 JSON ][ body: raw bytes ] -``` - -* **Length-prefixed** — no escaping, no line limits, binary-safe. -* **`header.id`** — a monotonic id per direction. Responses echo it. Enables multiplexing. -* **`body`** — untouched bytes. A Kafka payload passes through with zero transformation. -* **Symmetric.** Both sides can be caller and callee. Frame types: - `request`, `response`, `error`, `log`, `metric`, `event`, `ack`, `ping`, `pong`, `control`. -* **Credit-based flow control.** The host grants `MaxInFlight`; the adapter must not exceed it. - Without this, an adapter reading Kafka faster than the host persists Xchanges just buffers - until OOM. - -Host side becomes a `ConcurrentDictionary` plus a read loop — which -also removes the reflection-over-`TaskCompletionSource` hack in `ServerlessService`. - -**Do not multiplex over stdout while also using stderr for logs.** In v2, logs are `log` frames -on the same stream. stderr becomes what it should be: unstructured crash output, captured into a -ring buffer for forensics (§6.7). - ---- - -## 4. Challenge 2 — lifetime and DI - -```mermaid -flowchart TB - subgraph SCOPED["Invocation adapters — unchanged"] - S1["HTTP request / Quartz job
opens a DI scope"] - S2["IServerlessService
TRANSIENT + IDisposable"] - S3["child process"] - S1 --> S2 --> S3 - S1 -.scope disposed.-> S4["process killed
correct behaviour"] - end - subgraph RESIDENT["Resident adapters — new"] - R1["ResidentAdapterSupervisor
IHostedService"] - R2["IResidentAdapterHost
SINGLETON"] - R3["long-lived processes
keyed by DataSourceId"] - R1 --> R2 --> R3 - R4["callers get a HANDLE
never ownership"] -.-> R2 - end - style S4 stroke:#cf9a2e,stroke-width:3px -``` - -*The existing transient registration is right for invocation adapters and fatal for resident -ones — a DI scope ending must never kill a broker connection.* - ---- - -### The supervisor is the reconciliation loop you already designed - -```mermaid -flowchart LR - A["DESIRED
DataSource rows whose
placement = this node"] --> R{"reconcile
every N seconds
+ on broadcast"} - B["ACTUAL
running adapter instances"] --> R - R -->|missing| C["start adapter"] - R -->|extra| D["stop adapter"] - R -->|config changed| E["restart adapter"] - R -->|term is stale| F["stop EVERYTHING
we lost leadership"] - style F stroke:#d24b3c,stroke-width:3px -``` - -`IServerlessService` is registered **transient** and is `IDisposable`; every consumer in Bitween -does `GetRequiredService()` inside a scope -(`XchangeService`, `ReceivingJob`, `AdapterInvoker`, `GetProperties`, …). Disposing the scope -kills the process. That is exactly right for invocation adapters and exactly wrong for resident -ones. - -So: - -* `IServerlessService` — **unchanged**, transient, invocation lifecycle. No consumer changes. -* `IResidentAdapterHost` — **singleton**, owns a `ConcurrentDictionary` - keyed by `(adapterId, instanceKey)` where `instanceKey` is the `DataSourceId`. Callers get a - *handle*, never ownership. -* `ResidentAdapterSupervisor` — `IHostedService`, the reconciliation loop. - -**The supervisor is the loop already designed in §5 of the architecture doc.** Desired state = -`DataSource` rows whose placement resolves to this node; actual state = running instances; -reconcile every N seconds and on `RefreshConsumers`-style broadcast. Nothing about the cluster, -placement, or leader-election design changes — only *what a provider is* changes, from a loaded -type to a supervised process. - ---- - -## 5. Challenge 3 — the push direction (the crux for bus) - -```mermaid -sequenceDiagram - autonumber - participant B as External broker - participant A as Resident adapter - participant H as Bitween host - participant S as Cloud storage - participant D as Database - B->>A: deliver message - A->>H: Event frame with payload, dedupe key, traceparent - H->>S: write payload blob - H->>D: insert Xchange row - D-->>H: committed - H->>D: publish domain event to internal bus - H-->>A: Ack for that event id - A->>B: commit offset / basic.ack - Note over H,A: if the host crashes between commit and Ack
the broker redelivers, so the DEDUPE KEY is mandatory -``` - -*The adapter must not acknowledge the broker until Bitween has durably persisted. That makes the -transport bidirectional — the adapter is a caller too — and makes at-least-once delivery, hence -deduplication, a hard requirement rather than an open question.* - ---- - -### Why the rejected alternative is worse - -```mermaid -flowchart LR - subgraph BAD["Rejected — adapter writes directly"] - A1["adapter"] --> DB1["Bitween database"] - A1 --> S1["cloud storage"] - A1 --> BUS1["internal RabbitMQ"] - X["adapter needs DB creds,
storage creds, and a COPY
of XchangeService logic"] - end - subgraph GOOD["Adopted — adapter calls the host"] - A2["adapter
knows only the broker"] -->|Event frame| H2["host owns persistence,
credentials and pipeline"] - end - style X stroke:#d24b3c,stroke-width:3px -``` - -A resident bus adapter receives a message and must get it into `XchangeService`. Two options: - -**Rejected: adapter writes straight to the DB / internal bus.** It would need Bitween's database -credentials, cloud-storage credentials, and a copy of `XchangeService`'s persist logic. Every -provider author would reimplement it, badly. It also destroys the "non-opinionated plugin" -premise. - -**Adopted: adapter → host RPC over the same protocol.** The adapter sends an `event` frame; the -host runs the existing ingest path (persist Xchange → write payload to cloud storage → commit → -domain event) and replies with `ack` or `nack`. Only then does the adapter commit the Kafka -offset / `basic.ack` the RabbitMQ delivery. - -This is why the protocol must be **bidirectional and symmetric from day one**. Retrofitting the -reverse direction later is the kind of change that forces a v3. - -Consequences that are now mandatory rather than optional: - -* **At-least-once, therefore dedupe.** The host can persist and crash before the ack lands; the - broker redelivers. The dedupe key listed as an open question in §12 of the architecture doc - becomes a requirement. Provider supplies it (Kafka: `topic:partition:offset`; RabbitMQ: - message-id or a content hash), host enforces uniqueness. -* **Ordering is per-adapter-instance at best.** With `MaxInFlight > 1` the host may commit - Xchanges out of order. If a provider needs ordering, it must serialize per key itself and the - descriptor must say so. -* **Backpressure is the adapter's job.** The credit window is the mechanism; the adapter must - stop fetching, not buffer. - ---- - -## 6. Observability - -```mermaid -flowchart TB - subgraph AD["Adapter process"] - A1["structured log frames
level, template, args, traceId"] - A2["metric frames
lag, in-flight, reconnects"] - A3["pong with STATUS
connected, partitions, last message"] - A4["stderr crash output"] - end - subgraph HOST["Bitween host"] - H1["ILogger scope
adapterId, dataSourceId, nodeId"] - H2["System.Diagnostics.Metrics"] - H3["supervisor decisions
restart / mark unhealthy"] - H4["per-instance ring buffer
last 200 lines"] - H5["host-observed metrics
RSS, CPU, threads, restarts"] - end - A1 --> H1 - A2 --> H2 - A3 --> H3 - A4 --> H4 - H5 --> H3 - H5 --> H2 - style H5 stroke:#1f9d63,stroke-width:3px -``` - -*The green box matters most: host-observed metrics need no adapter cooperation, so they still -work **when the adapter is wedged** — which is exactly when you need them.* - ---- - -### The three states a heartbeat must distinguish - -```mermaid -stateDiagram-v2 - [*] --> Starting - Starting --> Connected: dialled broker OK - Starting --> Failed: cannot connect - Connected --> Idle: no messages for N minutes - Idle --> Connected: message arrives - Connected --> Disconnected: broker dropped us - Disconnected --> Connected: reconnect succeeded - Failed --> [*]: crash-loop, supervisor stops it - note right of Idle - A plain liveness probe - cannot tell Idle from Disconnected. - Only a status payload can. - end note -``` - -This is the part that decides whether the design is operable. Today it is: stderr lines prefixed -`{{log.information}}` / `{{log.warning}}` / `{{log.error}}`, string-replaced into an -`ILogger` named `serverless.adapters.{adapterId}`. No structure, no trace, no metrics, no health. -For an invocation adapter that is survivable, because the Xchange record *is* the trace. For a -resident adapter that runs for weeks and owns an ingress, it is not. - -Seven things, in priority order: - -### 6.1 Structured log frames -Replace prefix-parsing with a `log` frame: -`{ level, message, exception, timestamp, template, properties, correlationId, traceId }`. -Host calls `ILogger.Log` inside a scope carrying `adapterId`, `instanceKey`, `dataSourceId`, -`nodeId`. No new sink infrastructure — it lands wherever Bitween's logs already land, but now -queryable by data source instead of grep-able by string. - -Add a `control` command `setLogLevel` so verbosity is changeable at runtime, per adapter -instance. This matters more than it sounds: N resident adapters per node multiplies log volume, -and you want debug on for *one* misbehaving data source, not all of them. - -### 6.2 Trace propagation, both directions -`System.Diagnostics.Activity` / W3C `traceparent` in every frame header. - -* Host → adapter: current activity id, so an adapter's work nests under the caller. -* Adapter → host push: the adapter starts the activity, and **links it to the broker message's - own `traceparent` header** when present — Kafka, RabbitMQ, MQTT5 and Pulsar all carry headers, - and upstream producers instrumented with OpenTelemetry set it. - -The payoff is one trace spanning *producer → broker → adapter → Xchange → mapper → handler*. -This is the single feature that makes cross-process debugging bearable, and it is the thing -in-proc plugins would have given you for free — so it has to be built deliberately. - -### 6.3 Metrics -Two independent sources, deliberately: - -* **Adapter-reported**, via `metric` frames on a timer: messages received / acked / nacked, - consumer lag, in-flight count, reconnect count, last error, broker-specific gauges. Host - republishes on `System.Diagnostics.Metrics` so they export through whatever Bitween already - uses. -* **Host-observed**, requiring no adapter cooperation: `Process.WorkingSet64`, - `TotalProcessorTime` delta, thread count, handle count, restart count, uptime — sampled by the - supervisor. For containers, `docker stats` / cgroup files. - -The second source is the important one, because **it still works when the adapter is wedged**. -It also feeds the node-health screen in §10 of the architecture doc directly, with real numbers -instead of estimates. - -### 6.4 Heartbeat that means something -A `ping` frame with a deadline; the adapter answers `pong` with a **status object**, not just -liveness: connected yes/no, subscribed partitions/queues, lag, timestamp of last received -message, last exception. This distinguishes the three states operators actually care about — -*process dead*, *process alive but disconnected*, *connected but receiving nothing* — which a -plain liveness check conflates. - -Note this must be answerable **while messages are in flight**, which is another reason the -protocol has to be multiplexed. A heartbeat that queues behind a 30-second invoke is a false -alarm generator. - -Missed N pings ⇒ restart. This is also exactly what the UI's per-gateway status badge renders. - -### 6.5 Per-message correlation -Adapter generates a correlation id per pushed message, puts it in the `event` frame, host stores -it on the Xchange and echoes it in the `ack`. Now an adapter log line joins to an Xchange row. -The current per-process `CorrelationId` cannot express this at all. - -### 6.6 Reuse the existing test-connection path -The "test connection" and "discover cluster" commands from the RabbitMQ provider plan become -ordinary `request` frames against a **transient** instance of the same adapter — same code, -same protocol, no second implementation. That is a real simplification the in-proc design did -not offer. - -### 6.7 Crash forensics -A resident adapter that dies at 03:00 must leave evidence. On abnormal exit, persist a -`data_source_incident` row: exit code, signal, last N stderr lines (ring buffer, per instance), -last frames exchanged, restart count in window. Surface the ring buffer in the UI as -"last 200 lines" so the diagnose loop does not require access to the central log store — -disproportionately useful for the configure-and-test UX. - ---- - -## 7. Challenge 4 — failure, restart, split-brain - -```mermaid -flowchart TB - START["adapter process exits"] --> Q{"why"} - Q -->|clean stop requested| OK["done"] - Q -->|crash| BACKOFF["exponential backoff restart"] - BACKOFF --> COUNT{"N restarts
within M minutes"} - COUNT -->|no| BACKOFF2["restart, keep counting"] - COUNT -->|yes| STOP["STOP restarting
mark DataSource unhealthy
surface in UI, fire notifier"] - style STOP stroke:#d24b3c,stroke-width:3px -``` - -*A silent restart loop is worse than a hard stop — it burns broker connections and hides the -fault.* - ---- - -### The four failure directions - -```mermaid -flowchart LR - F1["Invocation adapter dies"] --> R1["one Xchange fails
auto-retry policy already covers it"] - F2["Resident adapter dies"] --> R2["an entire ingress goes silent
needs supervisor + crash-loop detection"] - F3["Host dies, adapter survives"] --> R3["orphan burning CPU
fix the EOF spin, or Job Object kill-on-close"] - F4["Node loses leadership
while adapter runs"] --> R4["DUPLICATE CONSUMPTION
check the DB fencing term every reconcile"] - style R4 stroke:#d24b3c,stroke-width:3px - style R3 stroke:#cf9a2e,stroke-width:3px -``` - -* **Invocation adapter dies** — one Xchange fails, the auto-retry policy already covers it. -* **Resident adapter dies** — an entire ingress goes silent. Needs: supervisor detects exit, - exponential backoff restart, **crash-loop detection** (N restarts in M minutes ⇒ stop, mark the - `DataSource` unhealthy, surface in UI, fire a notifier). A silent restart loop is worse than a - hard stop. -* **Host dies, adapter survives** — must not happen. Fix the EOF spin (§3, item 7) so a null read - means exit, and additionally have the adapter poll its parent PID. -* **Node loses leadership while the adapter runs** — the process must be killed *before* another - node starts its own, or you get duplicate consumption. This is where the DB fencing token from - §6.1 of the architecture doc earns its keep: the supervisor checks the term before every - reconcile, and a stale term means "stop everything immediately". - ---- - -## 8. Challenge 5 — the container launcher - -**Why it is worth having** — in the order the reasons actually carry weight (see §1.1): -runtime and target-framework independence, so an adapter's TFM stops being the host image's -problem; native dependencies like librdkafka, which escape `GCHeapHardLimit` and are -platform-specific; hard resource limits (`--memory`, `--cpus`, `--pids-limit`); and a real -security boundary for third-party provider code. A provider needing a non-.NET client is a -genuine but uncommon fourth reason — every broker on the current roadmap has a usable .NET -client. - -**What it costs, honestly:** - -* The Bitween process needs access to a Docker socket. Mounting `/var/run/docker.sock` into a - containerized Bitween is a privilege escalation to root-on-host; DinD is worse. This is a - genuine security conversation, not a config flag. -* Image pull credentials, per-node image cache, cold-start on first pull (seconds to minutes). -* Not available everywhere: App Service, hardened k8s without socket access, some managed hosts. -* In Kubernetes the idiomatic answer is not `docker run` — it is a sidecar or a Job. So the - launcher must stay pluggable and you should not over-invest in the Docker one. - -**Therefore:** define `IAdapterLauncher { StartAsync, StopAsync, GetResourceUsageAsync }`, ship -`ProcessLauncher` first, and gate `ContainerLauncher` behind a node capability probe. A -`DataSource` whose adapter requires `Launcher=container` simply will not be placed on a node -that reports no container runtime — which is exactly what the §6.2 placement layer is for. - -Protocol-wise the container launcher is free: `docker run -i --rm` gives the same three pipes. - ---- - -## 9. What this costs you (the counter-argument, stated fairly) - -* **Debuggability of provider code.** In-proc, you attach a debugger and step through. Out-of-proc, - you debug IPC. Mitigation: `Runner.MockRun` already exists for in-test adapter hosting; extend - it to the resident lifecycle so provider logic is unit-testable without a process. -* **Integration test complexity.** A test now needs a *published* adapter. Precedent exists — - `AdapterInstaller` in `BitweenFixture` already publishes and uploads sample adapters — but - every provider test inherits that cost, on top of the Testcontainers broker. -* **Latency on enrichment.** `IQueryCapable` (a mapper querying a data source, §3.8) becomes an - IPC round trip: roughly 0.1–1 ms instead of ~0. Against the measured Traxis baseline — 32.67 s - p50 handler time versus 0.33 s of total plumbing — this is noise. It would matter only if - a mapper did thousands of lookups per message. -* **Two moving parts to version.** Protocol version negotiation must be explicit and checked at - startup, with a clear error, or you get mysterious hangs. - -None of these outweigh runtime installability and connection isolation for the bus use case. -All of them would outweigh it for, say, a JSON mapper — which is precisely why the invocation -lifecycle stays exactly as it is. - ---- - -## 10. Sequencing - -| Phase | Work | Why here | -|---|---|---| -| **0** | Fix `argv` secret exposure; fix EOF hot-spin; keep v1 behaviour otherwise | Both are live bugs today, independent of this design | -| **1** | Protocol v2: framed, multiplexed, binary body, symmetric, credit-based. v1 fallback keyed on absent `Protocol` metadata | Everything else depends on it | -| **2** | `IResidentAdapterHost` + `ResidentAdapterSupervisor` + log/metric/ping/trace frames; prove with a trivial "tick" resident adapter | Runtime and observability before any real provider | -| **3** | RabbitMQ resident bus adapter | The provider design in `provider-plan-rabbitmq-kafka.md` is unchanged — only where it runs changes | -| **4** | `ContainerLauncher` + node capability probe | Needed before Kafka if librdkafka is containerized | -| **5** | Kafka resident bus adapter | Highest client demand, hardest resource profile | - -Phases 0–2 are the real investment and buy nothing visible to a customer. Worth saying out loud -before committing, because it is the part that gets cut under pressure and it is the part that -cannot be retrofitted. - ---- - -## 11. Resource limits without Docker - -```mermaid -flowchart TB - subgraph L1["Layer 1 — portable, cooperative"] - A1["child env vars
DOTNET_gcServer=0
GCHeapHardLimit
GCConserveMemory"] - A2["parent samples
WorkingSet64, TotalProcessorTime"] - A3["supervisor watchdog
soft threshold = ask to DRAIN
hard threshold = kill"] - end - subgraph L2["Layer 2 — Linux, nearly free"] - B1["oom_score_adj = 500
kernel kills the ADAPTER,
not Bitween"] - B2["RLIMIT_NOFILE, RLIMIT_NPROC"] - B3["nice, ProcessorAffinity"] - B4["NEVER RLIMIT_AS
the GC reserves huge virtual space"] - end - subgraph L3["Layer 3 — real enforcement"] - C1["Linux cgroup v2
memory.high < memory.max
or systemd-run --scope"] - C2["Windows Job Object
+ KILL_ON_JOB_CLOSE"] - end - L1 --> L2 --> L3 - C1 -.->|".NET reads cgroup limits
and self-sizes the heap"| A1 - style B4 stroke:#d24b3c,stroke-width:3px - style B1 stroke:#1f9d63,stroke-width:3px -``` - -*Layer 3 makes Layer 1 self-tuning: .NET detects a cgroup memory limit and sizes its heap to -about 75% of it automatically.* - ---- - -### Why the watchdog is needed even when Layer 3 exists - -```mermaid -flowchart LR - M["RSS climbing"] --> S{"which threshold"} - S -->|"soft — watchdog"| D1["ask adapter to DRAIN
stop fetching, nack in-flight,
then stop cleanly"] - S -->|"hard — cgroup memory.max"| D2["SIGKILL
no chance to nack
in-flight messages redelivered"] - style D1 stroke:#1f9d63,stroke-width:3px - style D2 stroke:#d24b3c,stroke-width:3px -``` - -*And note native allocations — librdkafka's fetch buffers are `malloc`, invisible to -`GCHeapHardLimit`. RSS, not GC heap, is the number to watch.* - -In §1 and §8 I attributed memory capping to containers. That is imprecise and worth correcting: -**Docker's limits *are* cgroups.** You can have the same enforcement natively. What Docker -uniquely buys is *packaging* — a self-contained runtime per adapter, native deps, and image -distribution — not limits. -That materially lowers the priority of the container launcher. - -Three layers, from portable to enforcing. - -### 11.1 Layer 1 — portable: tune the child runtime, watch from the parent - -Injected as environment variables at spawn (the host already builds `ProcessStartInfo`): - -| Variable | Why it matters for a resident adapter | -|---|---| -| `DOTNET_gcServer=0` | **Biggest single win.** Server GC allocates a heap *and a GC thread per core*. On a 32-core host, 15 resident adapters on Server GC is ~480 GC threads and a very large committed footprint, for processes that are IO-bound and barely allocate. Workstation GC is the correct default for adapters. | -| `DOTNET_GCHeapHardLimit=` | Hard cap on the managed heap. Crossing it raises `OutOfMemoryException` in the adapter instead of growing into the host's memory. | -| `DOTNET_GCHeapHardLimitPercent` | Same, relative to the cgroup limit / physical memory. Use when Layer 3 is in play. | -| `DOTNET_GCConserveMemory=5..9` | Trades CPU for a smaller footprint. Right trade for an adapter that mostly waits on a socket. | -| `DOTNET_ThreadPool_MaxThreads` | Stops a misbehaving adapter from thread-storming the box. | -| `DOTNET_TieredPGO=0`, `DOTNET_ReadyToRun=1` | Lower startup cost and code-heap size; matters when the supervisor restarts adapters. | - -**The critical caveat: `GCHeapHardLimit` does not bound native allocations.** librdkafka's fetch -buffers are `malloc`, entirely outside GC accounting. For Kafka the only real control is -librdkafka's own `queued.max.messages.kbytes` / `queued.min.messages` / `fetch.max.bytes`. So -the provider descriptor must expose those, and the supervisor must treat RSS — not GC heap — as -the number it watches. - -Parent-side monitoring is what you already need for §6.3: `Process.WorkingSet64` and -`TotalProcessorTime` are accurate on both Linux and Windows. Note `Process.MaxWorkingSet` is not -a useful lever — Windows-only, and even there a soft trimming hint rather than a limit. - -Pair that with a **supervisor watchdog**: sample RSS every few seconds, and on crossing a soft -threshold ask the adapter to drain (stop fetching, finish in-flight, `nack` the rest), then stop -it; on crossing a hard threshold, kill. Operationally this is close to as good as a kernel limit, -because the failure you actually face is a slow leak degrading a node over hours, not an -instantaneous spike — **and it is strictly better than a cgroup OOM kill, which is `SIGKILL` -with no chance to nack in-flight messages.** Build the watchdog even when Layer 3 exists. - -Also: `Launcher=process` should read an `Executable` metadata key rather than hardcoding -`dotnet`. That one change lets a provider ship as ReadyToRun or NativeAOT — much lower baseline -RSS and startup — or as a Go/Rust binary, without needing containers at all. - -### 11.2 Layer 2 — Linux, nearly free - -Two file writes and a spawn flag, high value: - -* **`/proc//oom_score_adj`** — write a positive value (say `500`) for every adapter. Under - memory pressure the kernel then kills an *adapter*, not the Bitween host. This is a handful of - lines and it converts your worst outage mode into a supervised restart. -* **`RLIMIT_NOFILE` / `RLIMIT_NPROC`** — safe, bounds fd and thread storms. -* **`nice` / `ProcessorAffinity`** — `Process.PriorityClass` maps to `nice` on Unix, and - `ProcessorAffinity` works on Linux. Crude CPU containment that costs nothing. - -**Do not use `RLIMIT_AS`.** The .NET GC reserves very large virtual address ranges up front; -an address-space limit kills healthy processes. `ulimit -v` and .NET do not mix. - -### 11.3 Layer 3 — real enforcement, per platform - -This is what `IAdapterLauncher` exists for. - -**Linux — cgroup v2.** Create `/sys/fs/cgroup//adapter-/`, write `memory.high`, -`memory.max`, `cpu.max`, `pids.max`, then write the child PID into `cgroup.procs`. Plain file -writes; no P/Invoke, no Docker socket. Set **`memory.high` below `memory.max`** — `high` throttles -and reclaims (backpressure), `max` OOM-kills. That gives the watchdog a window to drain -gracefully before the kernel intervenes. - -Bonus synergy: **.NET detects cgroup memory limits automatically** and sizes the GC heap against -them (default heap hard limit ≈75% of the cgroup limit). So Layer 3 makes Layer 1 self-tuning. - -The catch is delegation — you need write access to a cgroup subtree, which you do not always -have inside a container or under a restrictive systemd config. Where systemd is present, the -easiest correct launcher is not raw cgroup writes at all: - -``` -systemd-run --scope --collect -p MemoryMax=512M -p MemoryHigh=384M -p CPUQuota=50% -p TasksMax=64 \ - dotnet /adapters//Adapter.dll -``` - -Declarative limits, no Docker socket, no privilege escalation, stdin/stdout pass through -unchanged. For most on-prem Linux deployments this is the answer, and it should probably ship -*before* `ContainerLauncher`. - -**Windows — Job Objects.** The genuine equivalent, and there is no BCL wrapper, so ~100 lines of -P/Invoke: `CreateJobObject` + `SetInformationJobObject` with -`JOBOBJECT_EXTENDED_LIMIT_INFORMATION` (`ProcessMemoryLimit`, `JobMemoryLimit`) and -`JOBOBJECT_CPU_RATE_CONTROL_INFORMATION` (CPU rate cap). Worth it for one flag alone: -**`JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE`** — when the Bitween process dies, the OS kills every -adapter it spawned. That is the orphan problem solved by the kernel rather than by the -parent-PID-polling hack in §7. - -**macOS** — effectively nothing beyond `setrlimit`. Dev-only; do not design for it. - -### 11.4 The per-process baseline is itself a budget item - -Every resident adapter carries a fixed cost before it does any work — a .NET console process -with workstation GC is on the order of tens of MB RSS, plus threads and fds. Fifteen data -sources is a real, permanent slice of the node. Consequences: - -* **One adapter process per `DataSource`, not per endpoint.** One RabbitMQ connection can serve - many queues; do not spawn per gateway. -* Consider **one process per provider *type* per node**, multiplexing several data sources of the - same kind, once the count grows. The protocol already needs an instance key, so leave room for - it in the addressing even if you start one-per-`DataSource`. -* Measure the baseline for real before committing to placement density — do not take the number - above as given. - ---- - -## 12. Is stdio performant enough for logging and observability? - -### Where the cost actually is - -```mermaid -flowchart LR - subgraph SLOW["Today — per log line"] - S1["Console.Error.WriteLine
SYNCHRONIZED writer, a lock"] - S2["2-3 string allocs
newline escaping"] - S3["autoflush = one syscall"] - S4["host BeginErrorReadLine
event + fresh string per line"] - S1 --> S2 --> S3 --> S4 - end - subgraph FAST["Fixed — per batch"] - F1["bounded Channel of frames
DropOldest + dropped counter"] - F2["single writer task"] - F3["PipeWriter over the RAW stream
many frames per flush"] - F4["host PipeReader loop
no intermediate strings"] - F1 --> F2 --> F3 --> F4 - end - SLOW -->|"the pipe was never the bottleneck —
the syscall and alloc pattern was"| FAST -``` - ---- - -### Use the two pipes you already have - -```mermaid -flowchart TB - subgraph BADC["One shared stream"] - X1["log storm"] --> X2["pipe buffer fills"] - X2 --> X3["pong sits behind the logs"] - X3 --> X4["supervisor sees a missed heartbeat"] - X4 --> X5["restarts a HEALTHY adapter"] - end - subgraph GOODC["Split by purpose"] - Y1["stdout = DATA plane
RPC, events, acks
high volume"] - Y2["stderr = CONTROL plane
heartbeat, status, metrics
low, bounded rate"] - end - BADC --> GOODC - style X5 stroke:#d24b3c,stroke-width:3px -``` - -Short answer: **the pipe is not the problem; the current framing and syscall pattern are.** But -stdio is still the wrong place for *high-volume* telemetry, and the fix is a split rather than a -faster pipe. - -### 12.1 What is actually slow today - -Anonymous pipes move data at GB/s. What costs you is per-message overhead, and the current -implementation maximizes it: - -* **`Console.Error.WriteLine` per log line.** `Console.Out`/`Console.Error` are *synchronized* - writers — a global lock per write — and autoflush, so every log line is a lock plus a syscall. -* **`.Replace("\n", "{{newline}}").Replace("\r", "")` on every line, on both sides.** Two to - three string allocations per log, doubled because the host re-parses with `StartsWith` + - `Replace`. -* **`BeginOutputReadLine` / `OutputDataReceived`** gives you an event and a fresh `string` per - line, over a line-oriented reader, for every stream of every adapter. Fine for a handful of - logs per invocation; not the model you want for fifteen long-lived processes. - -All three are fixable without changing transport: - -* Adapter side: grab `Console.OpenStandardOutput()` / `OpenStandardError()` **once** and write to - the raw `Stream` via `PipeWriter`; never touch `Console.Out`/`Console.Error` again. Feed it - from a bounded `Channel` drained by a single writer task that **coalesces many frames - per flush**. This turns one-syscall-per-log into one-syscall-per-batch. -* Host side: drop `BeginOutputReadLine` entirely; run one async `PipeReader` loop per stream over - `process.StandardOutput.BaseStream`, parsing length-prefixed frames with zero intermediate - strings. -* Logs go over the wire as **template + arguments**, not a formatted string, so formatting - happens once, in the sink, and the log stays structured. - -### 12.2 Two rules that matter more than throughput - -**Logging must never apply backpressure to the data path.** The adapter's log channel must be -*bounded* with `DropOldest` and a `logs_dropped` counter. An adapter in an exception storm that -blocks on a full pipe — because the host is busy — has just deadlocked its own message -processing. This is the failure mode to design against, not raw MB/s. - -**A log storm must not delay a heartbeat.** If logs and RPC share one stream, a burst of log -frames sits ahead of a `pong` in the pipe buffer, the supervisor sees a missed heartbeat, and it -restarts a perfectly healthy adapter. You already have two independent pipes with independent -kernel buffers, so use them for what they are good at: - -* **stdout — data plane:** RPC requests/responses, pushed `event` frames, acks. High volume. -* **stderr — control plane:** heartbeat, status, metric snapshots, lifecycle and error events. - Low, bounded rate, never starved by the data plane. - -That also happens to preserve the existing convention, where stderr was already the log channel. - -### 12.3 The split that actually resolves the question - -```mermaid -flowchart TB - AD["Adapter"] - AD -->|"low rate, ALWAYS on
never lossy, no external dependency"| CP["CONTROL PLANE
over stdio/stderr"] - AD -->|"high volume, when configured
batching, sampling, retry"| DP["DEBUG TELEMETRY
OTLP to a collector"] - CP --> SUP["supervisor decisions
restart, placement, UI status"] - DP --> COL["OTel collector
traces, logs, metrics"] - NOCOL["no collector configured"] -.->|"fall back to stdio log frames
at reduced verbosity"| CP - style CP stroke:#1f9d63,stroke-width:3px -``` - -*They have opposite requirements, so one channel cannot serve both. The supervisor's inputs must -never depend on external infrastructure being up; the human's inputs must never cost the host -CPU.* - -Do not try to make one channel serve both the supervisor and the human. They have opposite -requirements. - -| | Control-plane telemetry | Debug telemetry | -|---|---|---| -| Carries | heartbeat/status, metric snapshots (one frame every few seconds), errors, lifecycle, stderr ring buffer | per-message spans, debug logs, detailed metrics | -| Volume | bounded by construction | unbounded, bursty | -| Transport | **stdio (stderr), always on** | **OTLP to a collector, when configured** | -| Why | drives restart and placement decisions — must never be lossy, must never depend on external infrastructure being up | must never cost the host CPU, and needs batching/sampling/retry that is already solved | - -The host already injects configuration at adapter startup, so it can inject -`OTEL_EXPORTER_OTLP_ENDPOINT` and the resource attributes (`service.name`, `adapter.id`, -`datasource.id`, `node.id`) with zero work from the adapter author. When no endpoint is -configured — a customer with no collector — the adapter falls back to stdio `log` frames at -reduced verbosity, and the `setLogLevel` control command turns detail on for one data source -when someone is actually debugging. - -It also keeps the door open cheaply: if some provider ever does need a non-.NET client, that -adapter uses its own OTel SDK and lands in the same traces — which would be very awkward if all -telemetry had to be tunnelled through a bespoke stdio frame format. That is a cheap option to -hold, not a reason to build the container launcher. - -### 12.4 The data path, and the copy worth eliminating - -For message payloads, stdio is not a bottleneck at Bitween's measured throughput — a pipe copy -is trivial next to the cloud-storage write that follows it. But note the payload is copied -adapter → host → cloud storage. For large payloads it is worth letting the adapter write -directly to cloud storage and send only a reference in the `event` frame, above a size -threshold. That removes the largest copy and the host CPU with it, at the cost of giving the -adapter storage credentials — or, better, having the host hand out pre-issued upload URLs. Treat -this as an optimization to enable later, not a day-one requirement. - -### 12.5 Measure before committing - -Before building on any of this, benchmark the v2 framing standalone: frames/second and -µs/frame for small control frames, allocations per frame on both sides, and host CPU at -*N* adapters × *M* messages/sec. It is a day of work and it decides whether the batching design -above is sufficient or whether the data plane needs a different transport. - ---- - -## 13. Revision: orchestrated adapters, and why this changes the protocol choice - -> **This section revises §2, §3, §8 and parts of §11.** The third mode is not -> "`docker run` on the Bitween host" — it is Bitween driving the **Kubernetes API** to deploy a -> separate application that Bitween talks to over the network. That is a materially better idea, -> and it removes the reason for the bespoke stdio framing proposed in §3. - -### 13.1 The three modes, restated - -```mermaid -flowchart TB - subgraph M1["EPHEMERAL — today, unchanged"] - E1["Bitween / Gateway"] -->|"spawn per invocation"| E2["child process"] - E3["v1 line stdio"] - E4["mappers, validators, handlers
~190 binaries"] - end - subgraph M2["RESIDENT — new"] - R1["Bitween supervisor"] -->|"owns lifetime"| R2["long-lived child"] - R3["gRPC over UDS / named pipe
child dials the host"] - R4["broker ingress where there
is no orchestrator"] - end - subgraph M3["ORCHESTRATED — new"] - O1["Bitween"] -->|"Kubernetes API"| O2["Deployment"] - O2 --> O3["adapter pod"] - O3 -.->|"dials OUT, gRPC over TLS"| O1 - O4["broker ingress + independent scale"] - end - M2 -.->|"SAME contract,
different binding"| M3 -``` - -*Resident and Orchestrated are one adapter with two bindings. Ephemeral keeps its own path -because a per-invocation spawn cannot afford gRPC startup — and because ~190 binaries depend on -it being frozen.* - -| Mode | Lifetime owned by | Transport | Fits | -|---|---|---|---| -| **Ephemeral** (today) | Bitween, per invocation | v1 line stdio, **unchanged** | mappers, validators, handlers — short, CPU-bound, thousands of spawns | -| **Resident** | Bitween supervisor | gRPC over UDS / named pipe | broker connections where there is no orchestrator | -| **Orchestrated** | Kubernetes | gRPC over TCP+TLS | broker connections and anything needing independent scale | - -Resident and Orchestrated are **the same adapter with a different binding**. Ephemeral stays on -its own path because per-Xchange process spawn cannot afford a gRPC stack, and because the -existing adapter fleet must keep working untouched. - -### 13.2 Two planes, not one - -```mermaid -flowchart TB - subgraph K8S["Kubernetes cluster"] - API["Kubernetes API server"] - POD1["adapter pod
kafka, replicas N"] - POD2["adapter pod
rabbitmq, replicas 1"] - API -.creates, scales, deletes.-> POD1 - API -.creates, scales, deletes.-> POD2 - end - BW["Bitween
leader replica only"] - BW ==>|"ORCHESTRATION PLANE
create / scale / delete Deployment
read pod status, events, metrics"| API - POD1 ==>|"DATA PLANE
adapter DIALS OUT
one bidi gRPC stream"| BW - POD2 ==>|"DATA PLANE"| BW - BROKER["external Kafka / RabbitMQ"] --> POD1 - BROKER --> POD2 - style BW stroke:#1f9d63,stroke-width:3px -``` - -*Dial-**out** is the decision that makes everything else fall into place — no Service, no -Ingress, no TLS cert or DNS name for the adapter, no service discovery, works through NAT and -egress-only NetworkPolicy, one auth direction instead of two. And the stream is symmetric, -exactly like the pipe was.* - ---- - -### Why dial-in would have been worse - -```mermaid -flowchart LR - subgraph IN["Bitween dials the adapter"] - I1["needs a Service + DNS name"] - I2["needs TLS cert for the adapter"] - I3["needs service discovery"] - I4["adapter must authenticate Bitween"] - I5["and for PUSH the adapter
must dial back anyway"] - I5 --> I6["two connection directions,
two auth setups"] - end - subgraph OUT["Adapter dials Bitween"] - U1["Deployment only, no Service"] - U2["one stream, symmetric"] - U3["one token, mounted by the pod"] - end - IN --> OUT - style I6 stroke:#d24b3c,stroke-width:3px -``` - -The orchestrated mode splits into two independent directions, and conflating them is the main -way this design goes wrong: - -* **Orchestration plane — Bitween → Kubernetes API.** Create / update / scale / delete the - adapter's `Deployment`, read pod status, events, and metrics. This replaces "spawn a process." -* **Data plane — adapter pod → Bitween.** The adapter **dials out** to Bitween and opens one - long-lived bidirectional gRPC stream. This replaces "the pipes." - -Making the adapter dial *out* rather than Bitween dial *in* is the decision that makes everything -else fall into place: - -* No `Service`, no `Ingress`, no TLS certificate, no DNS name for the adapter. -* No service discovery — Bitween never needs to find the adapter. -* Works through NAT and restrictive `NetworkPolicy` egress-only setups. -* One authentication direction instead of two (the adapter presents a token; the pod got it from - a mounted secret Bitween wrote when it created the Deployment). -* **The stream is symmetric, exactly like the pipe was.** Host→adapter commands and - adapter→host events multiplex over it, and the push/ack path from §5 works identically. - -The cost to be explicit about: Bitween must expose a gRPC endpoint the pods can reach. In-cluster -that is trivial; if Bitween runs outside the cluster it must be publicly reachable, which is a -real deployment constraint and belongs in the prerequisites. - -### 13.3 This removes the need for protocol v2 framed stdio - -```mermaid -flowchart LR - subgraph HAND["Hand-built framing — sec 3"] - H1["custom length prefix"] - H2["custom message ids"] - H3["custom credit protocol"] - H4["custom frame types"] - H5["ad-hoc versioning"] - H6["a codec per language"] - H7["custom log/metric frames"] - end - subgraph GRPC["gRPC — free"] - G1["bytes"] - G2["HTTP/2 streams"] - G3["HTTP/2 flow control"] - G4["bidi streaming"] - G5["proto field numbers"] - G6["protoc"] - G7["interceptors + OTel"] - end - H1 --> G1 - H2 --> G2 - H3 --> G3 - H4 --> G4 - H5 --> G5 - H6 --> G6 - H7 --> G7 - style GRPC stroke:#1f9d63,stroke-width:3px -``` - -*One transport abstraction, two bindings:* - -```mermaid -flowchart TB - PROTO["ONE .proto envelope contract"] - PROTO --> B1["Resident binding
UDS on Linux/macOS
named pipe on Windows"] - PROTO --> B2["Orchestrated binding
TCP + TLS"] - B1 --> C1["child dials the host"] - B2 --> C2["pod dials the host"] - C1 --> SAME["IDENTICAL code path
above the transport"] - C2 --> SAME - style SAME stroke:#1f9d63,stroke-width:3px -``` - -§3 argued for a hand-built length-prefixed framed protocol. Once the orchestrated mode is in -scope, that is over-engineering. Define the contract **once as a `.proto` service** and gRPC -supplies, for free, every property §3 was hand-rolling: - -| §3 requirement | Hand-built framing | gRPC | -|---|---|---| -| Binary payloads | custom body segment | `bytes` | -| Multiplexing | custom framing and stream ids | HTTP/2 streams | -| Transport backpressure | custom windowing | HTTP/2 flow control | -| Bidirectional, symmetric | custom frame types | bidi streaming | -| Contract versioning | ad-hoc | proto field numbers | -| Polyglot adapters | write a codec per language | `protoc` | -| Tracing, metrics, logging | custom `log`/`metric` frames | interceptors + first-class OTel instrumentation | - -**Two things gRPC does not give you, and the contract still has to.** HTTP/2 flow control governs -bytes on the wire, not application semantics: it will happily let an adapter push a thousand -events the host has not persisted. So `AdapterFrame.id` correlation, the `Ack`/`Nack` reply on an -event, and the `MaxInFlight` credit window remain **application-level requirements** — defined in -the proto and enforced on both sides — rather than anything the transport supplies. - -That is a large amount of design, test, benchmark and version work removed — and §12.5's -"benchmark the framing first" becomes unnecessary. - -For **Resident** (no orchestrator), use the same gRPC contract over a **Unix domain socket** on -Linux/macOS and a **named pipe** on Windows — both are supported transports for Kestrel and -`Grpc.Net.Client` on .NET 8. The host creates the endpoint, passes its path plus a one-time -token to the child, and the child dials in. **Identical dial-out shape to the orchestrated mode, -so resident and orchestrated share the entire code path above the transport.** - -The one thing to keep from §12: **stdio still carries early-life diagnostics** — everything the -adapter logs before it manages to dial, and its crash output. Keep the stderr ring buffer. - -Ephemeral adapters keep the v1 line protocol verbatim. The §3 bugs still need fixing (secrets on -`argv`, the EOF hot-spin), because they are live today. - -### 13.4 What orchestrated mode genuinely buys, beyond packaging - -* **Independent scale-out.** A Kafka adapter for a 60-partition topic can run six replicas in one - consumer group. Neither the resident nor the in-proc plugin model can do that at all — they are - capped at one connection owner per data source. This is the largest single win and it did not - exist in the earlier design. -* **The node-pressure problem disappears.** Broker connections and librdkafka's native buffers - never live on a Bitween node. All of §11 collapses to `resources.limits.memory` — a declarative - field, kernel-enforced, zero code. -* **Better health data than a child process gives you.** Pod status, `OOMKilled` as an explicit - reason rather than an inference, `CrashLoopBackOff`, restart counts, pod events, and - CPU/memory from `metrics.k8s.io`. -* **Independent rollout.** Upgrade one provider without touching the Bitween deployment. -* **Each adapter carries its own runtime**, so a target framework stops being the host image's - problem — the concrete pain behind Gateway's `COPY --from=aspnet:6.0` (§14.2). A non-.NET - client, in the uncommon case one is needed, is then just another image. - -### 13.5 What it costs — stated plainly - -* **A second operational model, permanently.** Non-k8s customers get Resident; k8s customers get - Orchestrated. Two health models, two log paths, two failure taxonomies, two sets of UI states. - This is the biggest cost and it does not go away. It is only tolerable because both modes share - one contract and one data plane. -* **RBAC.** Bitween needs a ServiceAccount with create/update/delete on Deployments. Many - enterprises will push back. Mitigate with a namespace-scoped `Role` in a dedicated namespace — - never cluster-wide — and make the whole thing opt-in. -* **Runtime install gets weaker, not stronger.** This is the important counterpoint. Resident - adapters install from a zip in cloud storage — Bitween fully controls it. Orchestrated adapters - are **images in a registry**, so runtime install now depends on the *customer's* registry, - pull secrets, air-gap policy, image scanning, and admission controllers that may reject - unsigned images. The "install a provider without a redeploy" property survives, but it stops - being purely Bitween's to guarantee. -* **You are writing a Kubernetes controller.** Imperatively creating and deleting Deployments - from a web application drifts. The idiomatic answer is a CRD plus an operator, which needs - cluster-admin to install CRDs. Pragmatic middle ground: manage plain Deployments labelled - `bitween.io/datasource-id`, and reconcile by listing labelled Deployments rather than by - remembering what you created. No CRD, namespace-scoped RBAC, drift still self-corrects. -* **Cold start.** Deploy → pull image → schedule → start → dial back is seconds to minutes. "Test - connection" against a not-yet-deployed adapter is therefore awkward; either run it as a - short-lived `Job`, or keep one warm generic adapter pod per provider type for - test/describe/discover calls. -* **Local dev and integration testing get harder.** Testcontainers cannot hand you a cluster; you - need k3s or kind in CI. Given that the integration suite does not run in CI at all today, this - is a real concern rather than a theoretical one. Mitigation: make the Kubernetes orchestrator a - thin, mockable `IAdapterOrchestrator`, and test adapter *behaviour* in Resident mode — which is - the same code — so only the orchestrator itself needs a cluster. - -### 13.6 Exclusivity is *not* free, and this is a trap - -```mermaid -sequenceDiagram - autonumber - participant D as Deployment replicas=1 - participant P1 as Pod v1 old config - participant P2 as Pod v2 new config - participant B as Broker - Note over D: someone edits the DataSource config - D->>P2: create new pod - Note over P1,P2: RollingUpdate keeps the old pod
until the new one is Ready - P1->>B: still consuming - P2->>B: also consuming - Note over B: DUPLICATE CONSUMPTION WINDOW
on every single config change - D->>P1: terminate old pod - P1->>B: stops -``` - -*Fix: `strategy: Recreate` for exclusive consumers, or a StatefulSet where at-most-one really -matters. Better still, where the broker coordinates (Kafka consumer groups, RabbitMQ competing -consumers), run `replicas: N` and drop the exclusivity requirement — that is the whole point of -the mode.* - -```mermaid -flowchart LR - Q{"does the provider
coordinate consumers?"} - Q -->|"yes — Kafka groups,
RabbitMQ competing consumers"| A["replicas: N
no exclusivity needed
SCALE OUT"] - Q -->|"no — exclusive consumer"| B{"how strict?"} - B -->|"tolerable overlap"| C["Deployment
strategy: Recreate"] - B -->|"must be at-most-one"| D["StatefulSet"] - style A stroke:#1f9d63,stroke-width:3px -``` - -The resident model relied on Bitween's leader election to guarantee exactly one consumer per data -source. It is tempting to assume `replicas: 1` replaces that. It does not: - -* A `Deployment` with the default `RollingUpdate` strategy runs **two pods simultaneously** - during every rollout. For an exclusive consumer that is a duplicate-consumption window on every - config change. -* During a node partition, a Deployment can transiently exceed its replica count. - -So: use `strategy: Recreate` for exclusive consumers, and prefer a **StatefulSet** where -at-most-one really matters — its at-most-one-per-ordinal guarantee is far stronger than a -Deployment's. Where the provider supports it (Kafka consumer groups, RabbitMQ competing -consumers), prefer `replicas: N` with broker-side coordination and drop the exclusivity -requirement entirely — that is the whole point of §13.4. - -And Bitween's own leader election still matters, for a different reason: **exactly one Bitween -replica may drive the Kubernetes API**, or concurrent replicas will fight over the same -Deployments. - -### 13.7 Revised sequencing - -| Phase | Work | Note | -|---|---|---| -| **0** | Fix `argv` secret exposure and the EOF hot-spin in the v1 path | Live bugs, independent of everything else | -| **1** | Define the adapter contract as `.proto` — one service, bidi stream, host and adapter commands | The single most important artifact; everything binds to it | -| **2** | Resident mode: gRPC over UDS / named pipe, adapter dials host, `IResidentAdapterHost` + supervisor + §11 limits | Ships value without any orchestrator | -| **3** | RabbitMQ resident adapter | First real provider; §11 limits and §12 telemetry get exercised | -| **4** | `IAdapterOrchestrator` + `SW.Serverless.Kubernetes` — optional package, namespace-scoped RBAC, labelled Deployments, warm pod for describe/test | Same contract, second binding. Non-k8s customers never load it | -| **5** | Kafka adapter, running orchestrated with `replicas: N` in a consumer group | The case that justifies the whole orchestrated mode | - -Phase 1 is the hinge. If the contract is defined properly once, phases 2 and 4 are two bindings -of the same thing rather than two subsystems. - ---- - -## 14. Revision: SW-Serverless is a shared library, not a Bitween subsystem - -> **This section revises §2, §3, §4 and §13.7.** Evidence: `Traxis/Adapters/Agent` (~107 adapter -> projects), `Traxis/Adapters/Bitween` (~80 more), and `Traxis/Microservices/Gateway`, cross-read -> against `Adapters/Agent/SERVERLESS_ADAPTERS.md`. - -### 14.1 The constraint that dominates everything else - -```mermaid -flowchart TB - subgraph FLEET["The installed base"] - F1["~107 Traxis Agent adapters
one exe per carrier x command"] - F2["~80 Traxis Bitween adapters
mappers, handlers, receivers"] - F3["SDK versions in live use
2.0.16 / 2.0.22 / 5.0.5
6.0.0 / 6.0.9 / 6.0.28
8.0.1 / 8.1.1 / 8.1.2"] - F4["Host versions in live use
6.0.9 / 6.0.28
8.1.1 / 8.1.2 / 8.1.5"] - end - FLEET --> RULE["RULE
every change is additive and opt-in,
selected PER ADAPTER via metadata,
never per host"] - RULE --> R1["absent Protocol key
= v1 code path, byte for byte"] - style RULE stroke:#1f9d63,stroke-width:3px -``` - ---- - -### The Dockerfile line that proves the model is straining - -```mermaid -flowchart LR - IMG["Gateway image
FROM aspnet:8.0"] - COPY["COPY --from=aspnet:6.0
/usr/share/dotnet/shared"] - IMG --> COPY - COPY --> WHY["because the adapter fleet
multi-targets net6 AND net8"] - WHY --> PROB["every adapter's target framework
is the HOST IMAGE's problem,
forever and cumulatively"] - PROB --> FIX["Orchestrated mode dissolves this —
each adapter image carries
its own runtime"] - style PROB stroke:#d24b3c,stroke-width:3px - style FIX stroke:#1f9d63,stroke-width:3px -``` - -| Fact | Consequence | -|---|---| -| ~107 Traxis **Agent** adapter executables (one per carrier × command) plus ~80 Traxis **Bitween** adapters | Roughly 190 published binaries sitting in S3 as the installed base | -| `SimplyWorks.Serverless.Sdk` versions in live use: **2.0.16, 2.0.22, 5.0.5, 6.0.0, 6.0.9, 6.0.28, 8.0.1, 8.1.1, 8.1.2** | The fleet spans four major versions. Some of these adapters likely cannot be rebuilt cheaply | -| Host `SimplyWorks.Serverless` versions in use: **6.0.9, 6.0.28, 8.1.1, 8.1.2, 8.1.5** | Two independent products upgrade on independent cadences | -| Gateway's `Dockerfile` does `COPY --from=mcr.microsoft.com/dotnet/aspnet:6.0 /usr/share/dotnet/shared` into an `aspnet:8.0` image | **The host image already ships two .NET runtimes because the adapter fleet multi-targets.** This is a live workaround for a real problem | - -**Rule that follows: every change to SW-Serverless must be additive and opt-in, selected -per-adapter, never per-host.** The "absent `Protocol` metadata ⇒ v1 verbatim" rule from §2 is not -a courtesy; it is the only thing that keeps ~190 binaries alive. And "ephemeral is the legacy -path" was wrong — **ephemeral is the primary product**, with the bulk of the installed base and -the highest invocation rate. It must be treated as first-class and frozen, not tolerated. - -### 14.2 The multi-runtime problem is a genuine, existing win for orchestrated mode - -The `COPY --from=aspnet:6.0` line is the clearest evidence in the repository that the current -model is straining. A single host process can only offer the runtimes baked into its image, so -every adapter's target framework becomes the host image's problem, forever, cumulatively. - -**Orchestrated mode dissolves this**: each adapter image carries its own runtime. So does the -container launcher. This is a concrete, present-day pain — not a speculative future benefit — -and it raises the value of §13 beyond what I credited it with. - -It is also an argument for the `Executable` metadata key from §11.1 even in the local process -launcher: an adapter published self-contained or ReadyToRun stops depending on the host image's -runtime set entirely. - -### 14.3 Fixes that pay for themselves before any of this design lands - -Three defects are live in Traxis production today, all fixable **host-side only, with zero -adapter changes**, benefiting ~190 existing binaries: - -1. **The un-correlated `TaskCompletionSource`.** §3 listed this as a blocker for resident - adapters. It is not theoretical — it is a documented production hazard: `CommandTimeout` - fires `TrySetException` but **does not kill the child**, so the timed-out command's late - result line resolves the *next* invocation's TCS. Because Gateway calls `GetLogs` immediately - after every command, that stray result normally lands on and corrupts the log fetch. - Fix: a FIFO queue of pending completions instead of one field, **and kill the child on - `CommandTimeout`** — after a timeout the process state is unknown and reusing it is unsafe. -2. **`IdleTimeout` kills the whole process, not the pending command** (300s default, adapter-side, - thrown from a timer callback). Fine for ephemeral, fatal for resident — it must become opt-out - via metadata. -3. **`Install` never deletes superseded `{ETag}` directories** — roughly 7 MB per version per - pod, accumulating forever. Harmless-ish with short-lived pods; a real leak once adapters are - resident or pods are long-lived. - -Plus the two from §3 that this evidence confirms are not hypothetical: **`agent.Settings` is -carrier credentials, and it rides to every adapter as base64 on `argv`**, readable via `ps` — and -the same settings are written unredacted into the S3 audit JSON, masked only at read time by a -substring heuristic. - -This reorders the business case. I framed protocol work as a cost to be paid for the bus feature. -Item 1 alone is a Traxis production bug fix that happens to also unblock Bitween. - -### 14.4 Process reuse already exists — resident is a smaller step than stated - -§4 described the jump from "one process per invocation" to "long-lived process" as the big -change. It is smaller than that, because the current model is already **one process per -*session*, not per *command***: - -* Gateway issues **two** `InvokeAsync` calls per logical operation — the command, then `GetLogs` - in a `finally`. -* Multipiece shipments call `CreateShipment` **once per piece, sequentially, against the same - already-started child**. - -So adapters already tolerate multiple commands over one process lifetime, and the host already -manages a session. Resident mode extends the session's lifetime and adds the push direction; it -does not introduce process reuse. - -### 14.5 A third resident shape: pooled workers - -```mermaid -flowchart TB - subgraph EX["EXCLUSIVE resident — brokers"] - E1["exactly 1 instance"] - E2["owns a connection, holds state"] - E3["PUSHES events to the host"] - E4["keyed by DataSourceId"] - E5["guarded by leader election"] - end - subgraph PO["POOLED resident — stateless workers"] - P1["N warm instances"] - P2["no connection state"] - P3["host always initiates"] - P4["keyed by adapterId"] - P5["checkout / checkin per invocation"] - end -``` - -*Pooled resident has nothing to do with brokers — it is a latency win for Traxis and Bitween -alike, removing process spawn + JIT + an S3 metadata check from every request.* - ---- - -### Why pooling cannot simply be switched on - -```mermaid -sequenceDiagram - autonumber - participant R1 as Request A - participant P as Pooled adapter process - participant R2 as Request B - R1->>P: CreateShipment - Note over P: HttpClientWithLog appends to
the PROCESS-STATIC LogStore - R1->>P: GetLogs - P-->>R1: logs for A - Note over P: LogStore is NEVER cleared —
multipiece relies on accumulation - R2->>P: CreateShipment (checked out again) - R2->>P: GetLogs - P-->>R2: logs for A **and** B - Note over R2: request A's carrier audit trail
leaks into request B's S3 log -``` - -*So pooling needs `Poolable=true` opt-in, an explicit session boundary (a `Reset` command, or -scoping `LogStore` to a session id), and the FIFO plus kill-on-timeout fix first — otherwise one -timeout poisons every subsequent checkout rather than one call.* - -§4 assumed one resident instance per `DataSource`. Traxis reveals a second, equally valuable -shape that has nothing to do with brokers: - -| Shape | Instances | State | Push? | Keyed by | Use | -|---|---|---|---|---|---| -| **Exclusive resident** | exactly 1 | owns a connection | yes | `DataSourceId` | broker ingress | -| **Pooled resident** | N warm | stateless | no | `adapterId` | replaces per-invocation spawn | - -Pooled resident is a drop-in latency win for both products: Gateway currently pays -`dotnet` process spawn + JIT + an S3 metadata check on every tracking request and every shipment -creation, and Bitween pays it per Xchange stage. A warm pool with checkout/checkin removes that -from the request path with no change to adapter *logic*. - -**But it cannot be turned on blindly, and the reason is instructive.** `AdapterBase`'s `LogStore` -is **process-static**, deliberately: `GetLogs` returns everything accumulated since the process -started, and multipiece relies on that accumulation across calls. Pool the process and you leak -one request's HTTP audit trail into the next one's S3 log. So pooling requires: - -* opt-in via metadata (`Poolable=true`), never a default; -* an explicit session boundary — a `Reset` command, or scoping `LogStore` to a session id — so - "per process" stops being the unit of accumulation; -* the FIFO/kill-on-timeout fix from §14.3 first, because a pooled process that survives a timeout - in unknown state would poison every subsequent checkout rather than one. - -This changes §4: `IResidentAdapterHost` needs **pool semantics** (min/max instances, checkout, -idle eviction, max-invocations-before-recycle), not just a dictionary of singletons. - -### 14.6 The `.proto` must be a generic envelope, not a domain contract - -```mermaid -flowchart TB - subgraph HOSTS["Host SDKs own the DOMAIN"] - T["SimplyWorks.TraxisGateway.Sdk
Track, CreateShipment, DeleteShipment,
GetPudoCode, GetPod, GetLogs"] - B["SW.Bitween.Sdk
mapper, handler,
receiver, validator"] - end - subgraph SL["SW-Serverless owns the ENVELOPE only"] - E["Invoke(command, bytes) -> Result(bytes)
Event(bytes, correlationId, traceparent) -> Ack
Log / Metric / Ping / Pong
SetLogLevel / Describe / Reset"] - end - T -->|"serialises its own types
into opaque bytes"| E - B -->|"serialises its own types
into opaque bytes"| E - E --> W["transport, lifecycle, installation
NEVER learns a domain type"] - style W stroke:#1f9d63,stroke-width:3px -``` - -*If the proto grew `rpc Track(...)`, Traxis's five commands and Bitween's four adapter roles -would land in one file, and every host would be coupled to every other host's domain.* - -§13.3 said "define the contract once as a `.proto`." With two hosts in view, be precise about -*which* contract. Traxis Gateway's command names (`Track`, `CreateShipment`, `DeleteShipment`, -`GetPudoCode`, `GetPod`, `GetLogs`) and its payload types (`TrackableShipment`, -`StandardShipmentTrace`, `ShipmentCreationResult`) live in **Gateway's own SDK NuGet** -(`SimplyWorks.TraxisGateway.Sdk`) — not in SW-Serverless. Bitween's mapper/handler/receiver/ -validator contracts live in Bitween's SDK. That separation is correct and must survive. - -So the proto defines the **envelope only**: - -``` -Invoke(command: string, payload: bytes) -> Result(payload: bytes | error) -Event(payload: bytes, correlationId, traceparent) -> Ack | Nack -Log / Metric / Ping / Pong / SetLogLevel / Describe / Reset -``` - -Opaque `bytes`, with serialization owned by the host SDK. SW-Serverless stays what it already -is — **transport, lifecycle and installation** — and never learns a domain type. If the proto -grew `rpc Track(...)`, Traxis's five commands and Bitween's four adapter roles would end up in -one file, and every host would be coupled to every other host's domain. - -This also resolves something §13 left implicit: the reason `{{expected}}` / `Runner.Expect` -exists and is plumbed but unused on Gateway's hot path is that it is *administrative* metadata, -not part of the invocation contract. `Describe` is its successor and belongs in the envelope. - -### 14.7 Revised sequencing - -```mermaid -flowchart TB - P0["PHASE 0 — bug fixes, host-side only
FIFO completions + kill-on-timeout
IdleTimeout opt-out
ETag directory cleanup
secrets off argv, EOF spin"] - P1["PHASE 1 — .proto envelope
per-adapter opt-in via metadata
v1 path untouched"] - P2["PHASE 2 — resident host
BOTH shapes: pooled + exclusive
gRPC over UDS / named pipe"] - P3["PHASE 3 — RabbitMQ
exclusive-resident adapter"] - P4["PHASE 4 — IAdapterOrchestrator
SW.Serverless.Kubernetes
per-adapter runtime images"] - P5["PHASE 5 — Kafka adapter
orchestrated, replicas N
in a consumer group"] - P0 --> P1 --> P2 --> P3 - P2 --> P4 --> P5 - P0 -.->|"ships value ALONE
to ~190 adapters
with no adapter changes"| V0["TRAXIS TODAY"] - P2 -.-> V2["Traxis latency
+ Bitween brokers"] - P4 -.-> V4["Bitween brokers
+ retires Gateway's
dual-runtime image hack"] - style P0 stroke:#1f9d63,stroke-width:3px - style V0 stroke:#1f9d63,stroke-width:3px - style P1 stroke:#cf9a2e,stroke-width:3px -``` - -*Phase 0 ships value on its own, to the product with the larger installed base, before any -architectural commitment. Phase 1 is the hinge — get the envelope right once and phases 2 and 4 -are two bindings of one thing rather than two subsystems.* - -| Phase | Work | Who benefits | -|---|---|---| -| **0** | FIFO pending-completions + kill-on-`CommandTimeout`; `IdleTimeout` opt-out; `{ETag}` directory cleanup; secrets off `argv`; EOF hot-spin | **Traxis today**, ~190 adapters, no adapter changes | -| **1** | `.proto` envelope + per-adapter opt-in via metadata; v1 path untouched | Foundation | -| **2** | Resident host with **both** shapes — pooled and exclusive — over gRPC on UDS / named pipe | Traxis latency **and** Bitween brokers | -| **3** | RabbitMQ exclusive-resident adapter | Bitween | -| **4** | `IAdapterOrchestrator` + `SW.Serverless.Kubernetes`; per-adapter runtime images | Bitween brokers **and** retires Gateway's dual-runtime image hack | -| **5** | Kafka adapter, orchestrated, `replicas: N` in a consumer group | Bitween | - -Phase 0 now ships value on its own, to the product with the larger installed base, before any -architectural commitment is made. That is a much better position to start from than "build a -protocol, then a supervisor, then finally something a customer notices." - -### 14.8 Governance - -Per the existing note that `SimplyWorks.*` are public Simplify9 repositories with CI/CD -auto-publish: changes land as PRs and publish automatically. With two products pinned to -different versions, the operating rule is **version the behaviour, not the library** — new -capability behind metadata flags and new APIs, so Gateway can take a NuGet bump with zero -behavioural change and adopt features on its own schedule. A breaking protocol change would -require a coordinated upgrade across ~190 binaries and is effectively off the table. - -### 14.9 Host-held adapter state - -An adapter must not be the system of record for its own progress, and until now nothing in the -contract let it avoid being one. The supervisor restarts it, the next instance may come up on a -different node, and a pooled one is not the same process twice — so a polling receiver that keeps -its cursor in a field replays from the beginning at the least convenient moment. - -So `IAdapterContext` gains two calls, sitting beside `PublishAsync` and answered the same way — an -adapter-initiated frame, correlated by id, awaited before the adapter carries on: - -```csharp -Task GetStateAsync(string name, CancellationToken ct = default); -Task SetStateAsync(string name, string value, CancellationToken ct = default); // null deletes -``` - -On the wire that is `StateRequest` (get / set / delete) answered by `StateResult`. On the host side -it is `IAdapterStateStore`, the counterpart of `IAdapterEventSink`: where the sink is how an -adapter hands work in, this is how it remembers where it got to. `InMemoryAdapterStateStore` is the -default so samples and tests work untouched; a real deployment registers its own through -`AddResidentAdapters()` and backs it with a table. - -Three properties are deliberate: - -* **Keyed by instance, not by adapter.** Two instances of one adapter are two connections. One - reading the other's cursor would skip rows that were never processed. -* **Not bounded by the in-flight window.** That window exists to stop an adapter flooding the host - with events it must persist. A receiver saving its cursor *after* a batch would otherwise queue - behind the very events whose progress it is recording. -* **Failures are raised, not swallowed.** A cursor that silently failed to save is a batch that - will be replayed, and the adapter is the only thing positioned to stop rather than carry on. - -It is a bookmark, not a data store, and a host is entitled to refuse a large value. - -This is the same role Airbyte's `state` argument plays for its connectors, and it is what makes a -polling database receiver possible at all — see Bitween's `docs/provider-plan-databases.md`. - -### 14.10 A pool keyed by adapter id is a configuration leak - -`RentAsync` keyed its pools on `spec.AdapterId`, and `GetOrAdd` captures the spec of whichever -caller created the pool first — **including its startup values, which is where the connection -string and the credentials live**. One adapter serving two data sources therefore handed the second -one a process connected as the first, with no error anywhere: every later renter silently ran -against the wrong system. - -Masked until now because bus providers run as *exclusive* instances keyed by data source and never -go through the pool. Anything that rents — Bitween's Xchange pipeline does, through -`ResidentAdapterRuntime` — was exposed, and a pooled database adapter would be exposed by design. - -Fixed by keying on `AdapterSpec.PoolKey` when set, and otherwise on the adapter id plus a hash of -the startup values, so identical configuration shares warm processes and differing configuration -cannot. Hashed rather than concatenated because the key reaches logs and diagnostics. - -### 14.11 A shared instance needs per-call configuration - -§14.5 gave the exclusive resident one instance per key, and §4 assumed that instance had one -caller. It does not. In Bitween a relational data source is one process holding one connection -pool, and *every* subscription bound to that data source runs through it — each with its own -settings: which statement to run, which operation it is, which tenant this is. - -Startup values cannot carry any of that. They are handed over once, in `Ready`, and they belong to -the process — which belongs to all of those callers at once. So the invocation grows a -`map properties`, alongside the `session_id` that exists for the same underlying -reason: a shared instance has to be told whose call this is. - -On the adapter side that surfaces as two members on `IAdapterContext`: - -```csharp -IReadOnlyDictionary InvocationValues { get; } // this call's, empty outside one -string ValueOf(string name); // invocation first, then startup -``` - -`ValueOf` is the one to reach for. A per-call setting overrides the process default, and an adapter -whose callers send no properties behaves exactly as it did before any of this existed — which is -what keeps the existing fleet working. - -**It is an `AsyncLocal`, and that is the whole subtlety.** Several commands run on one instance at -the same time; multiplexing is the point of the stream. A field would have the last caller in -overwrite everyone else's configuration mid-flight, and the failure would be intermittent, -load-dependent and near-impossible to reproduce — one subscription silently running another's -statement. It is set on the invoking flow before the handler is called, so it survives the -handler's own awaits and cannot leak sideways. - -Without this the shared-instance shape is only usable by callers that all want identical behaviour, -which for a database connection is nobody. - ---- - -## 15. What "gRPC over UDS / named pipe" actually means - -> **Clarifies §3.1 and §13.3.** Those two sections look contradictory — §3.1 rejects gRPC to keep -> stdio's simplicity, §13.3 adopts gRPC. They are talking about different transports. This -> section closes the gap and walks through what changes inside an S3-zipped adapter. - -### 15.1 The apparent contradiction, resolved - -§3.1 said gRPC "buys better tooling and costs all of that back." That judgement was about -**gRPC over localhost TCP** — and it still stands. A UDS is not a network socket. - -A **Unix domain socket (UDS)** uses the socket *API* but is a **filesystem path**, not an address: -`/run/bitween/adapters/a1b2c3.sock`. There is no IP, no port, no network stack, no routing. The -kernel moves the bytes between two processes on the same machine, and access is controlled by -ordinary filesystem permissions. It is the same *category* of object as a pipe — it just happens -to be full-duplex and to carry many concurrent streams. - -On Windows the equivalent is a **named pipe**: `\\.\pipe\bitween-adapter-a1b2c3`, a kernel IPC -object secured by an ACL. .NET 8 gave Kestrel first-class named-pipe support, so this is a -supported transport, not a hack. - -| | stdio pipes | **UDS / named pipe** | localhost TCP | -|---|---|---|---| -| Port allocation / collisions | none | **none** | yes | -| Bind address, firewall rule | none | **none** | yes | -| Reachable from another machine | no | **no** | possible — one `0.0.0.0` mistake away | -| Access control | inherited handles | **filesystem perms / ACL** | needs an auth token | -| Many concurrent streams | **no** — 2 half-duplex pipes | **yes** | yes | -| Binary-safe | needs custom framing | **yes** | yes | -| Crosses a container boundary | yes, `docker run -i` | **only via a mounted volume** | yes | - -So "gRPC over UDS" means: keep protobuf messages, bidirectional streams, HTTP/2 flow control, -interceptors and codegen — and run the whole thing over an IPC object with **no port, no bind -address, no firewall rule and no network exposure**. Every property §3.1 was protecting survives. - -The one genuine loss is the last row: stdio crosses a container boundary for free, a UDS needs a -shared mount (`-v /run/bitween:/run/bitween`, or an `emptyDir` in a pod). That is a real cost and -the reason it is called out here rather than glossed over. - -```mermaid -flowchart TB - subgraph A["stdio pipes — today"] - A1["2 half-duplex byte streams"] - A2["no ports, no firewall"] - A3["ONE call in flight
framing is yours to build"] - end - subgraph B["UDS / named pipe — proposed"] - B1["full-duplex kernel IPC
a filesystem path, not an address"] - B2["no ports, no firewall"] - B3["HTTP/2 multiplexing, flow control,
binary, codegen — all free"] - end - subgraph C["localhost TCP — rejected in sec 3.1"] - C1["real network socket"] - C2["port allocation, bind address,
firewall, auth token"] - C3["same gRPC benefits"] - end - A -->|"keeps the simplicity,
gains the plumbing"| B - C -->|"gains the plumbing,
LOSES the simplicity"| B - style B stroke:#1f9d63,stroke-width:3px - style C stroke:#d24b3c,stroke-width:3px -``` - -### 15.2 What changes inside an S3-zipped adapter — nothing about the zip - -This is the practical question, and the answer is reassuring: **installation, packaging and -distribution are untouched.** - -The zip is still the published output of a .NET console app. `GetAdapterMetadata` still reads -`EntryAssembly` and `Hash` from S3 object metadata; `Install` still extracts to -`{AdapterLocalPath}/{Hash}/`; the host still spawns `dotnet `. The **only** -difference is what the SDK does once `Main` runs: - -```csharp -// Ephemeral adapter — today, unchanged, still the v1 stdin line loop -static Task Main() => Runner.Run(new Handler()); - -// Resident adapter — same zip, same spawn, different SDK entry point -static Task Main() => Runner.RunResident(new Handler()); -``` - -Packaging cost: `Grpc.Net.Client` + `Google.Protobuf` become transitive dependencies of the SDK, -adding roughly 2–3 MB to that adapter's published output. It is **opt-in per adapter** — an -adapter that is never rebuilt against the new SDK never gains the dependency and never leaves the -v1 path. That is what protects the ~190 existing binaries (§14.1). - -### 15.3 The handshake, step by step - -```mermaid -sequenceDiagram - autonumber - participant H as Bitween host - participant FS as socket path or pipe name - participant P as Adapter child process - H->>FS: Kestrel listens on /run/bitween/adapters.sock - Note over FS: directory is chmod 0700, owned by the service user
Windows equivalent is an ACL restricted to the current user - H->>P: spawn dotnet Adapter.dll (NO secrets on argv) - H->>P: write ONE handshake line to stdin
socket path, one-time token, protocol version - P->>FS: open a Unix socket and connect - P->>H: Attach RPC carrying the token - H->>H: match token to the pending spawn, bind the instance - Note over H,P: one bidirectional stream is now open - H->>P: Invoke, Ping, SetLogLevel, Reset - P->>H: Event, Log, Metric, Pong - Note over H,P: both directions multiplex over the SAME stream -``` - -Three details worth noticing: - -* **The child dials the host, not the reverse.** The host is already running and already - listening; the child is transient. It also means the child never picks a path, never handles a - bind failure, and — critically — this is the **identical shape as the Kubernetes case**, where a - pod dials out (§13.2). One code path above the transport. -* **The handshake goes over stdin, not `argv`.** The socket path is not secret but the token is, - and this is the same mechanism that fixes the `argv` credential leak in §14.3. No new - machinery: the child already has a stdin pipe. -* **stdio does not disappear.** It keeps carrying what it is genuinely good at — anything the - adapter logs *before* it manages to attach, and its crash output. That is the ring buffer in - §6.7. - -### 15.4 The wiring, concretely - -Nothing exotic; both sides are supported .NET APIs. - -**Host — listen on the socket** (`SW.Serverless`, once at startup, one listener for all adapters): - -```csharp -// Linux / macOS -webBuilder.ConfigureKestrel(o => o.ListenUnixSocket("/run/bitween/adapters.sock", - l => l.Protocols = HttpProtocols.Http2)); - -// Windows (.NET 8+) -webBuilder.ConfigureKestrel(o => o.ListenNamedPipe("bitween-adapters", - l => l.Protocols = HttpProtocols.Http2)); -``` - -**Adapter — dial it** (inside `Runner.RunResident`, so no adapter author ever writes this): - -```csharp -var handler = new SocketsHttpHandler -{ - ConnectCallback = async (_, ct) => - { - var sock = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.Unspecified); - await sock.ConnectAsync(new UnixDomainSocketEndPoint(socketPathFromHandshake), ct); - return new NetworkStream(sock, ownsSocket: true); - } -}; - -// the address is a placeholder — ConnectCallback decides where the bytes actually go -var channel = GrpcChannel.ForAddress("http://localhost", - new GrpcChannelOptions { HttpHandler = handler }); - -var call = new AdapterHost.AdapterHostClient(channel).Attach(); -await call.RequestStream.WriteAsync(new Frame { Hello = new Hello { Token = token } }); -``` - -On Windows the `ConnectCallback` returns a `NamedPipeClientStream` instead; everything above it is -the same. The `http://localhost` address is never resolved — `ConnectCallback` intercepts before -any DNS or TCP happens. **That is the whole trick: HTTP/2 does not care what byte stream it runs -on.** - -### 15.5 So why not just build framed stdio after all? - -Stated fairly, because it is a close call for the *local* case: - -```mermaid -flowchart TB - Q{"is Orchestrated mode
in scope?"} - Q -->|"no"| S["framed stdio wins
no extra dependency,
nothing to mount,
crosses container boundaries free"] - Q -->|"YES"| G["gRPC over UDS wins
Orchestrated needs gRPC over TCP anyway,
so UDS means ONE contract, one codegen,
one interceptor and observability path,
one test suite"] - style G stroke:#1f9d63,stroke-width:3px -``` - -If Kubernetes-orchestrated adapters were off the table, framed stdio would be the right call for -local resident adapters and §3.1 would stand unamended. The decisive argument is not about the -local transport at all — it is **"do not build and maintain two protocols."** Orchestrated mode -forces gRPC into the design; once it is there, using it locally too is free, and hand-rolling a -second framing format alongside it is not. - -### 15.6 The option not taken: gRPC directly over the stdio pipes - -Worth naming because it looks like the obvious best of both worlds. HTTP/2 runs over any duplex -byte stream, so in principle you can run gRPC over the child's existing stdin/stdout — no socket, -no mount, no path, and container boundaries stay free. - -The client half is easy: `ConnectCallback` returns a `Stream` wrapping the two pipes. The server -half is not — Kestrel has no "listen on this `Stream`" transport, so one side needs a custom -`IConnectionListenerFactory`. It is genuinely possible, and it is more work than it looks, with -worse diagnostics when it misbehaves. - -Recommendation: start with UDS / named pipe. Keep this in the back pocket for the container -launcher, where the shared-mount requirement is the one real wrinkle UDS introduces. - -### 15.7 Both resident shapes use the identical transport - -Finally, to close the phrase in §14.7 — pooled and exclusive residents differ only in lifecycle -and keying, never in wire protocol: - -| | Exclusive resident | Pooled resident | -|---|---|---| -| Transport | gRPC over UDS / named pipe | **the same** | -| Instances | exactly 1 | N warm | -| Keyed by | `DataSourceId` | `adapterId` | -| Who initiates | **both** — host invokes, adapter pushes `Event` | host only | -| Placement | guarded by leader election | any node | - -The pooled shape simply never uses the `Event` direction of the stream. diff --git a/sdk/node/src/index.js b/sdk/node/src/index.js index 2f1d6c4..c3d179d 100644 --- a/sdk/node/src/index.js +++ b/sdk/node/src/index.js @@ -405,8 +405,16 @@ class Runner { if (frame.ready) { startupValues.clear(); for (const [k, v] of Object.entries(frame.ready.startup_values ?? {})) startupValues.set(k, v); - if (typeof this.adapter.start === "function") await this.adapter.start(); - this.readyResolve(); + // Started beside the read loop, not inside it: a start that publishes an event or reads its + // state waits for the host's answer, which only the read loop can take in. Commands wait for + // ready; if start fails, the adapter stops. + this.starting = Promise.resolve() + .then(() => (typeof this.adapter.start === "function" ? this.adapter.start() : undefined)) + .then(() => this.readyResolve(), (e) => { + console.error(e?.stack ?? e); + process.exitCode = 1; + this.stopping.abort(); + }); } else if (frame.invoke) { this.startInvoke(id, frame.invoke); } else if (frame.cancel) { @@ -503,6 +511,9 @@ class Runner { // promised those answers. const deadline = Date.now() + (shutdown.drain ? 30000 : 5000); const within = (promise) => Promise.race([promise, new Promise((r) => setTimeout(r, Math.max(100, deadline - Date.now())))]); + // A start still running finishes first: a stop that overtook it would leave half of what start + // set up in place. + if (this.starting) await within(this.starting.catch(() => {})); if (typeof this.adapter.stop === "function") { try { await within(Promise.resolve(this.adapter.stop())); } catch (e) { this.log(3, `stop() failed: ${e?.message ?? e}`); } } diff --git a/sdk/python/src/sw_serverless/_runner.py b/sdk/python/src/sw_serverless/_runner.py index dc67620..fad14b7 100644 --- a/sdk/python/src/sw_serverless/_runner.py +++ b/sdk/python/src/sw_serverless/_runner.py @@ -141,6 +141,7 @@ def __init__(self, adapter): self._running = {} self._telemetry = None self._ready = None + self._starting = None # ------------------------------------------------------------------ description @@ -249,10 +250,10 @@ async def _on_frame(self, frame): ready = frame["ready"] _adapter._startup_values.clear() _adapter._startup_values.update(ready.get("startup_values", {})) - start = getattr(self.adapter, "start", None) - if callable(start): - await _call(start) - self._ready.set() + # Started beside the read loop, not inside it: a start that publishes an event or reads its + # state waits for the host's answer, which only the read loop can take in. Commands wait + # for _ready; if start fails, the adapter stops. + self._starting = self.loop.create_task(self._start()) elif "invoke" in frame: self._start_invoke(frame_id, frame["invoke"]) elif "cancel" in frame: @@ -277,6 +278,17 @@ async def _on_frame(self, frame): waiter.set_result(frame[answer]) return False + async def _start(self): + start = getattr(self.adapter, "start", None) + try: + if callable(start): + await _call(start) + self._ready.set() + except Exception: + logging.getLogger("sw_serverless").exception("the adapter failed to start") + traceback.print_exc() + self.stopping.set() + # ------------------------------------------------------------------ commands def _start_invoke(self, frame_id, invoke): @@ -339,6 +351,10 @@ async def _shutdown(self, shutdown): # Ask it to stop first, let commands already running answer, and only then go: a drain # promised those answers. deadline = time.monotonic() + (30 if shutdown.get("drain") else 5) + # A start still running finishes first: a stop that overtook it would leave half of what + # start set up in place. + if self._starting is not None and not self._starting.done(): + await asyncio.wait({self._starting}, timeout=max(0.1, deadline - time.monotonic())) stop = getattr(self.adapter, "stop", None) if callable(stop): try: From 726adae3d107f1e0623c2b3bc3ea8880bfe491d9 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 18:53:56 +0300 Subject: [PATCH 14/17] Let a host with resident adapters only start without AddServerless The default locator took the installer when it was built, which needs ServerlessOptions and storage, so a host giving every adapter by path could not start. It takes the installer only when it installs from storage, and says what is missing then. --- SW.Serverless.UnitTests/RegistrationTests.cs | 25 +++++++++++++++++++ .../Resident/IResidentAdapterLocator.cs | 12 ++++++--- 2 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 SW.Serverless.UnitTests/RegistrationTests.cs diff --git a/SW.Serverless.UnitTests/RegistrationTests.cs b/SW.Serverless.UnitTests/RegistrationTests.cs new file mode 100644 index 0000000..df59e00 --- /dev/null +++ b/SW.Serverless.UnitTests/RegistrationTests.cs @@ -0,0 +1,25 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Serverless.Resident; +using SW.Serverless.UnitTests.Fixtures; + +namespace SW.Serverless.UnitTests +{ + /// What a host has to register for the parts it uses, and no more. + [TestClass] + public class RegistrationTests + { + [TestMethod] + public async System.Threading.Tasks.Task Resident_adapters_given_by_path_need_neither_AddServerless_nor_storage() + { + var services = new ServiceCollection(); + services.AddLogging(l => l.ClearProviders()); + services.AddResidentAdapters(); + + await using var provider = services.BuildServiceProvider(new ServiceProviderOptions { ValidateOnBuild = false }); + + Assert.IsNotNull(provider.GetRequiredService()); + } + } +} diff --git a/SW.Serverless/Resident/IResidentAdapterLocator.cs b/SW.Serverless/Resident/IResidentAdapterLocator.cs index 4d9bb0a..5c24f49 100644 --- a/SW.Serverless/Resident/IResidentAdapterLocator.cs +++ b/SW.Serverless/Resident/IResidentAdapterLocator.cs @@ -6,7 +6,7 @@ namespace SW.Serverless.Resident /// /// Turns an AdapterSpec into something launchable. The default resolves an explicit /// EntryAssemblyPath first, and otherwise installs from cloud storage — the same - /// download-and-extract step the classic path already uses (design doc 15.2). + /// download-and-extract step the classic path already uses. /// public interface IResidentAdapterLocator { @@ -29,9 +29,12 @@ public class ResolvedAdapter internal class DefaultResidentAdapterLocator : IResidentAdapterLocator { - readonly AdapterInstaller installer; + readonly System.IServiceProvider services; - public DefaultResidentAdapterLocator(AdapterInstaller installer) => this.installer = installer; + // The installer, and the storage and options it needs, only when an adapter is installed + // from storage: a host that gives every adapter by path registers neither AddServerless nor + // storage, and must still start. + public DefaultResidentAdapterLocator(System.IServiceProvider services) => this.services = services; public async Task ResolveAsync(AdapterSpec spec, CancellationToken cancellationToken = default) { @@ -44,6 +47,9 @@ public async Task ResolveAsync(AdapterSpec spec, CancellationTo AdapterValues = spec.AdapterValues }; + var installer = services.GetService(typeof(AdapterInstaller)) as AdapterInstaller + ?? throw new System.InvalidOperationException( + $"Adapter '{spec.AdapterId}' has no EntryAssemblyPath, and installing it from storage needs AddServerless and a cloud files service."); var installed = await installer.InstallAsync(spec.AdapterId); // Cloud metadata is where Protocol / Lifecycle / Launcher / MaxInFlight live, so an From 91da7042f2004dd1d26b99e426d44e8ef4f2620b Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 18:53:56 +0300 Subject: [PATCH 15/17] Fix what the docs review found, and drop names that aren't ours - Manifests no longer carry isResident, which is worked out from lifecycle. - A package's Lang metadata is its runtime, not always dotnet. - test says when a contract file can't be read, instead of crashing. - --allow and --contract say they take several values separated by spaces. - musl platforms are no longer offered for Python builds, which manifests and hosts don't recognise. - Comments no longer cite a design document this repository doesn't have, or name other products; InternalsVisibleTo names sw-serverless and the packages point at this repository. --- .../Catalog/AdapterManifest.cs | 2 ++ SW.Serverless.Contract/Protos/adapter.proto | 14 +++++++------- .../SW.Serverless.Contract.csproj | 4 ++-- .../CliCommandTests.cs | 4 ++++ SW.Serverless.Installer/AdapterCommands.cs | 17 ++++++++++++++--- SW.Serverless.Samples.Carrier/CallLog.cs | 4 ++-- SW.Serverless.Samples.Carrier/CarrierHandler.cs | 11 +++++------ SW.Serverless.Samples.Carrier/CarrierOptions.cs | 2 +- SW.Serverless.Samples.Carrier/Contracts.cs | 2 +- .../Protos/carrier.proto | 2 +- SW.Serverless.Samples.Classic/Handler.cs | 4 ++-- SW.Serverless.Samples.FolderSource/Handler.cs | 4 ++-- SW.Serverless.Samples.Host/Program.cs | 2 +- SW.Serverless.Tooling/AdapterRepository.cs | 11 ++++++----- SW.Serverless.Tooling/Building/PythonBuild.cs | 2 -- .../SW.Serverless.Tooling.csproj | 2 +- .../Extensions/IServiceCollectionExtensions.cs | 2 +- SW.Serverless/Resident/AdapterHostService.cs | 2 +- SW.Serverless/Resident/AdapterMetrics.cs | 2 +- SW.Serverless/Resident/AdapterPool.cs | 4 ++-- .../Resident/AdapterProcessLauncher.cs | 4 ++-- SW.Serverless/Resident/AdapterSpec.cs | 2 +- SW.Serverless/Resident/IResidentAdapterHost.cs | 4 ++-- SW.Serverless/Resident/InstanceHealth.cs | 1 - SW.Serverless/Resident/ResidentAdapterHost.cs | 4 ++-- .../Resident/ResidentAdapterInstance.cs | 6 +++--- SW.Serverless/Resident/ResidentOptions.cs | 4 ++-- SW.Serverless/SW.Serverless.csproj | 4 ++-- SW.Serverless/Services/ServerlessService.cs | 2 +- 29 files changed, 71 insertions(+), 57 deletions(-) diff --git a/SW.Serverless.Contract/Catalog/AdapterManifest.cs b/SW.Serverless.Contract/Catalog/AdapterManifest.cs index 7093bfe..63c2b91 100644 --- a/SW.Serverless.Contract/Catalog/AdapterManifest.cs +++ b/SW.Serverless.Contract/Catalog/AdapterManifest.cs @@ -149,6 +149,8 @@ public static AdapterManifest Parse(string json) => public string EntryFor(string platform) => platform != null && Entries != null && Entries.TryGetValue(platform, out var entry) ? entry : Entry; + /// Whether is resident. Worked out, so never written. + [JsonIgnore] public bool IsResident => string.Equals(Lifecycle, ResidentLifecycle, StringComparison.OrdinalIgnoreCase); diff --git a/SW.Serverless.Contract/Protos/adapter.proto b/SW.Serverless.Contract/Protos/adapter.proto index 0da4de5..3d2b7d9 100644 --- a/SW.Serverless.Contract/Protos/adapter.proto +++ b/SW.Serverless.Contract/Protos/adapter.proto @@ -4,8 +4,8 @@ option csharp_namespace = "SW.Serverless.Contract"; package sw.serverless.v1; // The ENVELOPE only. Command names and payload shapes belong to the host application (its -// own SDK or contract package), never to this file — see the -// design doc, section 14.6. Payloads are opaque bytes here on purpose. +// own SDK or contract package), never to this file. Payloads are opaque bytes here on purpose; +// docs/protocol.md describes the protocol. service AdapterHost { // The adapter DIALS the host and opens one bidirectional stream. @@ -45,8 +45,8 @@ message Invoke { int32 timeout_seconds = 3; // Groups several invocations into one logical session — a pooled lease, or one inbound - // request. Traxis calls a command and then GetLogs; both must see the SAME session or the - // audit trail comes back empty. Absent means "this call alone". + // request. A host that calls a command and then asks for its logs needs both in the SAME + // session, or the logs come back empty. Absent means "this call alone". string session_id = 4; // Configuration for THIS call, on top of the startup values the process was given. @@ -85,7 +85,7 @@ message StateResult { message EventAck { bool accepted = 1; - string reference = 2; // the host's id for what it persisted, e.g. an Xchange id + string reference = 2; // the host's id for what it persisted, e.g. a message id Error error = 3; } @@ -189,7 +189,7 @@ message Error { } // The push direction. The adapter must NOT acknowledge its broker until the -// matching EventAck arrives — see the design doc, section 5. +// matching EventAck arrives. message Event { bytes payload = 1; string dedupe_key = 2; // topic:partition:offset, message-id, content hash, ... @@ -232,7 +232,7 @@ message Metric { } // Liveness is not enough: this separates "process dead" from "alive but disconnected" -// from "connected but receiving nothing" — see the design doc, section 6.4. +// from "connected but receiving nothing". message Pong { bool connected = 1; string state = 2; diff --git a/SW.Serverless.Contract/SW.Serverless.Contract.csproj b/SW.Serverless.Contract/SW.Serverless.Contract.csproj index 89b8c34..0028823 100644 --- a/SW.Serverless.Contract/SW.Serverless.Contract.csproj +++ b/SW.Serverless.Contract/SW.Serverless.Contract.csproj @@ -7,8 +7,8 @@ SimplyWorks.Serverless.Contract Simplify9 SimplyWorks.Serverless.Contract - https://github.com/simplify9/Serverless - https://github.com/simplify9/Serverless + https://github.com/simplify9/SW-Serverless + https://github.com/simplify9/SW-Serverless MIT Simplify9 Simplify9 diff --git a/SW.Serverless.Installer.UnitTests/CliCommandTests.cs b/SW.Serverless.Installer.UnitTests/CliCommandTests.cs index 2690349..7f868d2 100644 --- a/SW.Serverless.Installer.UnitTests/CliCommandTests.cs +++ b/SW.Serverless.Installer.UnitTests/CliCommandTests.cs @@ -129,6 +129,10 @@ public async Task Build_test_run_and_publish_take_a_project_into_storage() StringAssert.Contains(test.Output, "PASS orders processor: Process answers example 1"); StringAssert.Contains(test.Output, "Conforms."); + var missing = await Cli("test", zip, "--settings", settings, "--contract", Path.Combine(Path.GetDirectoryName(project)!, "nowhere.json")); + Assert.AreEqual(Program.Failure, missing.Exit); + StringAssert.Contains(missing.Output, "The contract couldn't be read"); + var run = await Cli("run", zip, "--settings", settings, "--call", "Process", "--input", """{"OrderId":"SO-1"}"""); Assert.AreEqual(Program.Success, run.Exit, run.Output); StringAssert.Contains(run.Output, "\"Accepted\":true"); diff --git a/SW.Serverless.Installer/AdapterCommands.cs b/SW.Serverless.Installer/AdapterCommands.cs index 4836777..789b8a9 100644 --- a/SW.Serverless.Installer/AdapterCommands.cs +++ b/SW.Serverless.Installer/AdapterCommands.cs @@ -43,7 +43,7 @@ public class BuildCliOptions [Option("no-source", HelpText = "Leave the source out of the package.")] public bool NoSource { get; set; } - [Option("allow", HelpText = "A source file whose secret-scan finding is a false positive. Repeat for more.")] + [Option("allow", HelpText = "Source files whose secret-scan findings are false positives, separated by spaces.")] public IEnumerable Allow { get; set; } [Option("dry-run", HelpText = "Show the source that would be carried, and build nothing.")] @@ -63,7 +63,7 @@ public class TestCliOptions [Option("allow-delete", HelpText = "Let a receiver's DeleteFile run: it removes or moves a real file at the source.")] public bool AllowDelete { get; set; } - [Option("contract", HelpText = "A contract file to check against, beyond those the CLI carries. Repeat for more.")] + [Option("contract", HelpText = "Contract files to check against, separated by spaces.")] public IEnumerable Contracts { get; set; } [Option("timeout", Default = 60, HelpText = "Seconds one call may take.")] @@ -173,6 +173,17 @@ public static async Task Build(BuildCliOptions opts) public static async Task Test(TestCliOptions opts) { + List contracts; + try + { + contracts = (opts.Contracts ?? Enumerable.Empty()).Select(ContractDocument.FromFile).ToList(); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or Newtonsoft.Json.JsonException) + { + Console.WriteLine($"The contract couldn't be read: {ex.Message}"); + return Failure; + } + var (package, cleanup) = await PackageFolderAsync(opts.Package); if (package == null) return Failure; try @@ -182,7 +193,7 @@ public static async Task Test(TestCliOptions opts) PackageDirectory = package, Settings = ReadSettings(opts.Settings), AllowDelete = opts.AllowDelete, - Contracts = (opts.Contracts ?? Enumerable.Empty()).Select(ContractDocument.FromFile).ToList(), + Contracts = contracts, CommandTimeoutSeconds = opts.Timeout, Log = Console.WriteLine, }); diff --git a/SW.Serverless.Samples.Carrier/CallLog.cs b/SW.Serverless.Samples.Carrier/CallLog.cs index 8935948..98973de 100644 --- a/SW.Serverless.Samples.Carrier/CallLog.cs +++ b/SW.Serverless.Samples.Carrier/CallLog.cs @@ -7,10 +7,10 @@ namespace SW.Serverless.Samples.Carrier { /// - /// The audit trail of every upstream call, which is what Traxis's GetLogs returns and what + /// The audit trail of every upstream call, which is what GetLogs returns and what /// ends up in the S3 record for a shipment. /// - /// In Traxis this is a process-STATIC list, and that is safe there only because a classic + /// Kept as a process-STATIC list, and that is safe there only because a classic /// adapter serves exactly one caller before exiting. Pool the process and one request's /// carrier calls leak into the next request's audit log. /// diff --git a/SW.Serverless.Samples.Carrier/CarrierHandler.cs b/SW.Serverless.Samples.Carrier/CarrierHandler.cs index e010677..2f03840 100644 --- a/SW.Serverless.Samples.Carrier/CarrierHandler.cs +++ b/SW.Serverless.Samples.Carrier/CarrierHandler.cs @@ -13,7 +13,7 @@ namespace SW.Serverless.Samples.Carrier { /// - /// A typical carrier adapter — the shape the ~107 Traxis agent adapters already have — except + /// A typical carrier adapter — the shape most carrier adapters already have — except /// that it STAYS RUNNING. /// /// The point is what does NOT change. The host calls it exactly as it calls a classic adapter: @@ -168,9 +168,8 @@ public async Task CreateShipment(ShipmentRequest request) if (!reply.Accepted) { // A carrier rejection is a RESULT, not an exception. Throwing here would turn a - // business outcome into an adapter outage — the pattern Traxis's - // CreateShipmentAdapterBase exists to enforce, and which a third of its adapters - // skip by deriving from raw AdapterBase. + // business outcome into an adapter outage — the pattern a shipment + // adapter base class exists to enforce. logger.LogInformation("Carrier rejected {Reference}: {Code} {Message}", request.Reference, reply.ErrorCode, reply.ErrorMessage); @@ -235,7 +234,7 @@ public async Task CancelShipment(TrackRequest request) } /// - /// The Traxis pattern, made safe. Gateway calls GetLogs after every command and merges the + /// A common pattern, made safe. The host calls GetLogs after every command and merges the /// result into the shipment's audit record; here the entries belong to this session only. /// public Task GetLogs() => Task.FromResult(callLog.Current()); @@ -264,7 +263,7 @@ public Task TestConnection() => CallAsync(nameof(TestConnection), /// /// One place for deadlines, retries, timing and the audit entry, so no command has to - /// remember them — which is exactly what a base class does for the Traxis adapters. + /// remember them — which is exactly what a base class does for a family of adapters. /// async Task CallAsync(string operation, Func> call, bool idempotent = true) diff --git a/SW.Serverless.Samples.Carrier/CarrierOptions.cs b/SW.Serverless.Samples.Carrier/CarrierOptions.cs index 2e3287c..70f4700 100644 --- a/SW.Serverless.Samples.Carrier/CarrierOptions.cs +++ b/SW.Serverless.Samples.Carrier/CarrierOptions.cs @@ -1,6 +1,6 @@ namespace SW.Serverless.Samples.Carrier { - /// Bound straight from startup values — the same keys a Traxis agent's Settings hold. + /// Bound straight from startup values — the same keys a host's adapter settings hold. public class CarrierOptions { /// The carrier's gRPC endpoint. diff --git a/SW.Serverless.Samples.Carrier/Contracts.cs b/SW.Serverless.Samples.Carrier/Contracts.cs index a424673..4c3767d 100644 --- a/SW.Serverless.Samples.Carrier/Contracts.cs +++ b/SW.Serverless.Samples.Carrier/Contracts.cs @@ -4,7 +4,7 @@ namespace SW.Serverless.Samples.Carrier { // The HOST's contract, not the carrier's. In a real deployment these types live in the host's - // own SDK — SimplyWorks.TraxisGateway.Sdk — and the adapter's whole job is translating between + // own SDK, and the adapter's whole job is translating between // them and whatever the upstream happens to speak. SW.Serverless never sees either. public class ShipmentRequest diff --git a/SW.Serverless.Samples.CarrierContract/Protos/carrier.proto b/SW.Serverless.Samples.CarrierContract/Protos/carrier.proto index 45fd7b3..0ce934c 100644 --- a/SW.Serverless.Samples.CarrierContract/Protos/carrier.proto +++ b/SW.Serverless.Samples.CarrierContract/Protos/carrier.proto @@ -3,7 +3,7 @@ syntax = "proto3"; option csharp_namespace = "SW.Serverless.Samples.CarrierContract"; package sw.samples.carrier.v1; -// A stand-in for a real carrier's API — the kind of upstream a Traxis agent adapter talks to. +// A stand-in for a real carrier's API — the kind of upstream a carrier adapter talks to. // It belongs to the CARRIER, not to SW.Serverless: the adapter's job is to translate between // this and whatever the host's own SDK defines. service Carrier { diff --git a/SW.Serverless.Samples.Classic/Handler.cs b/SW.Serverless.Samples.Classic/Handler.cs index 653744b..f1ae723 100644 --- a/SW.Serverless.Samples.Classic/Handler.cs +++ b/SW.Serverless.Samples.Classic/Handler.cs @@ -24,7 +24,7 @@ public class Handler /// /// It is also precisely why a POOLED resident adapter cannot simply reuse this process: /// one caller's entries would leak into the next. Pooling needs IResettable and an - /// explicit session boundary. See the design doc, section 14.5. + /// explicit session boundary. /// static readonly List CallLog = new(); @@ -137,7 +137,7 @@ public Task Fail() /// /// Blocks, so a caller can hit CommandTimeout. Under the classic protocol a timeout does /// NOT kill this process, and the late reply is what the correlation fix on the host now - /// discards — see the design doc, section 14.3. + /// discards. /// public async Task Slow(int seconds) { diff --git a/SW.Serverless.Samples.FolderSource/Handler.cs b/SW.Serverless.Samples.FolderSource/Handler.cs index ef1ad58..a2f9588 100644 --- a/SW.Serverless.Samples.FolderSource/Handler.cs +++ b/SW.Serverless.Samples.FolderSource/Handler.cs @@ -18,7 +18,7 @@ namespace SW.Serverless.Samples.FolderSource /// /// The ordering below is the part worth copying: the file is only archived AFTER the host /// acknowledges. Crash in between and the file is still there, so it is redelivered — which - /// is exactly why the dedupe key is mandatory (design doc, section 5). + /// is exactly why the dedupe key is mandatory. /// public class Handler : IResidentAdapter { @@ -92,7 +92,7 @@ public Task GetStatusAsync() { Connected = Directory.Exists(root), // Idle and Disconnected are genuinely different things, and a plain liveness - // probe cannot tell them apart (design doc, section 6.4). + // probe cannot tell them apart. State = !Directory.Exists(root) ? "Disconnected" : pending == 0 && received == 0 ? "Idle" : state, diff --git a/SW.Serverless.Samples.Host/Program.cs b/SW.Serverless.Samples.Host/Program.cs index 640374c..1eb6746 100644 --- a/SW.Serverless.Samples.Host/Program.cs +++ b/SW.Serverless.Samples.Host/Program.cs @@ -152,7 +152,7 @@ static async Task StatusLoopAsync(IResidentAdapterHost adapters, ILogger log, Ca /// /// A sample shortcut. In the real host this is the existing S3 download-and-extract step, - /// which is unchanged by protocol 2 — see the design doc, section 15.2. + /// which is unchanged by protocol 2. /// static string Locate(string project) { diff --git a/SW.Serverless.Tooling/AdapterRepository.cs b/SW.Serverless.Tooling/AdapterRepository.cs index 6865272..70a72e7 100644 --- a/SW.Serverless.Tooling/AdapterRepository.cs +++ b/SW.Serverless.Tooling/AdapterRepository.cs @@ -105,10 +105,11 @@ public AdapterRepository(ICloudFilesService files, Action log = null, st /// Version is empty for an unversioned upload. /// public static Dictionary LegacyMetadata( - string entryAssembly, string lifecycle, string kind, string sha256, string version) => new() + string entryAssembly, string lifecycle, string kind, string sha256, string version, string runtime = null) => new() { { "EntryAssembly", entryAssembly }, - { "Lang", "dotnet" }, + // The runtime it starts on; dotnet for every package before runtimes. + { "Lang", string.IsNullOrWhiteSpace(runtime) ? AdapterManifest.DotnetRuntime : runtime }, { "Timestamp", DateTime.UtcNow.ToString("yyyy-MM-ddTHH:mm:ssZ") }, { "Lifecycle", string.IsNullOrWhiteSpace(lifecycle) ? AdapterDescription.ClassicLifecycle : lifecycle }, { "Kind", kind ?? "" }, @@ -235,7 +236,7 @@ public async Task PublishVersionAsync(string adapterId, string version, string z // Loaded before uploading, so the new version is not mistaken for one the catalog missed. var entry = await LoadEntryAsync(adapterId); var sha256 = InstallerLogic.Sha256Of(zipPath); - var metadata = LegacyMetadata(package.EntryAssembly, package.Lifecycle, package.Kind, sha256, version); + var metadata = LegacyMetadata(package.EntryAssembly, package.Lifecycle, package.Kind, sha256, version, package.Manifest?.Runtime); await UploadAsync(versionKey, zipPath, metadata); // adapters/{id} is what hosts before manifests list and run, always with dotnet. An @@ -278,7 +279,7 @@ public async Task PublishUnversionedAsync(string adapterId, string zipPath, Pack var sha256 = InstallerLogic.Sha256Of(zipPath); await UploadAsync(CurrentKeyOf(adapterId), zipPath, - LegacyMetadata(package.EntryAssembly, package.Lifecycle, package.Kind, sha256, null)); + LegacyMetadata(package.EntryAssembly, package.Lifecycle, package.Kind, sha256, null, package.Manifest?.Runtime)); MakeCurrent(entry, null, package.Manifest, sha256, package.IconDataUri); await catalog.SaveAsync(entry); @@ -366,7 +367,7 @@ public async Task PromoteAsync(string adapterId, string version, string workDire var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var item in versionMetadata.Where(m => !ProviderMetadata.Contains(m.Key))) metadata[item.Key] = item.Value; - foreach (var item in LegacyMetadata(entryAssembly, lifecycle, kind, sha256, version)) + foreach (var item in LegacyMetadata(entryAssembly, lifecycle, kind, sha256, version, manifest?.Runtime)) metadata[item.Key] = item.Value; await UploadAsync(CurrentKeyOf(adapterId), zipPath, metadata); diff --git a/SW.Serverless.Tooling/Building/PythonBuild.cs b/SW.Serverless.Tooling/Building/PythonBuild.cs index a23b398..6013dd0 100644 --- a/SW.Serverless.Tooling/Building/PythonBuild.cs +++ b/SW.Serverless.Tooling/Building/PythonBuild.cs @@ -46,8 +46,6 @@ public static class PythonBuild { ["linux-x64"] = new[] { "manylinux2014_x86_64", "manylinux_2_28_x86_64", "manylinux_2_17_x86_64" }, ["linux-arm64"] = new[] { "manylinux2014_aarch64", "manylinux_2_28_aarch64", "manylinux_2_17_aarch64" }, - ["linux-musl-x64"] = new[] { "musllinux_1_2_x86_64" }, - ["linux-musl-arm64"] = new[] { "musllinux_1_2_aarch64" }, ["osx-arm64"] = new[] { "macosx_11_0_arm64" }, ["osx-x64"] = new[] { "macosx_10_9_x86_64" }, ["win-x64"] = new[] { "win_amd64" }, diff --git a/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj b/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj index 1665497..ebf10fc 100644 --- a/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj +++ b/SW.Serverless.Tooling/SW.Serverless.Tooling.csproj @@ -41,7 +41,7 @@ - + diff --git a/SW.Serverless/Extensions/IServiceCollectionExtensions.cs b/SW.Serverless/Extensions/IServiceCollectionExtensions.cs index 9524c56..c22ef66 100644 --- a/SW.Serverless/Extensions/IServiceCollectionExtensions.cs +++ b/SW.Serverless/Extensions/IServiceCollectionExtensions.cs @@ -40,7 +40,7 @@ public static IServiceCollection AddAdapterRuntimes(this IServiceCollection serv /// /// Adds the resident adapter runtime: a Kestrel endpoint on a Unix domain socket or named /// pipe that adapters dial, plus the supervisor that owns their processes. - /// Classic per-invocation adapters are untouched by this — see the design doc, section 15. + /// Classic per-invocation adapters are untouched by this. /// public static IServiceCollection AddResidentAdapters(this IServiceCollection services, Action configure = null) diff --git a/SW.Serverless/Resident/AdapterHostService.cs b/SW.Serverless/Resident/AdapterHostService.cs index 3f91d51..26b77e9 100644 --- a/SW.Serverless/Resident/AdapterHostService.cs +++ b/SW.Serverless/Resident/AdapterHostService.cs @@ -8,7 +8,7 @@ namespace SW.Serverless.Resident { /// /// The gRPC endpoint the adapter dials. It listens only on a Unix domain socket or a named - /// pipe — no port, no bind address, no firewall rule (design doc 15.1). + /// pipe — no port, no bind address, no firewall rule. /// internal class AdapterHostService : Contract.AdapterHost.AdapterHostBase { diff --git a/SW.Serverless/Resident/AdapterMetrics.cs b/SW.Serverless/Resident/AdapterMetrics.cs index 71b4162..37c65fb 100644 --- a/SW.Serverless/Resident/AdapterMetrics.cs +++ b/SW.Serverless/Resident/AdapterMetrics.cs @@ -6,7 +6,7 @@ namespace SW.Serverless.Resident { /// /// Adapter-reported metrics republished on System.Diagnostics.Metrics, so they export - /// wherever the host already exports (design doc 6.3). + /// wherever the host already exports. /// public static class AdapterMetrics { diff --git a/SW.Serverless/Resident/AdapterPool.cs b/SW.Serverless/Resident/AdapterPool.cs index dd3f73c..3a03df1 100644 --- a/SW.Serverless/Resident/AdapterPool.cs +++ b/SW.Serverless/Resident/AdapterPool.cs @@ -10,7 +10,7 @@ namespace SW.Serverless.Resident { /// /// Warm stateless workers, checked out per logical session. This is the shape that removes - /// process spawn plus JIT from every request — nothing to do with brokers (design doc 14.5). + /// process spawn plus JIT from every request — nothing to do with brokers. /// Only for adapters that declare Poolable and implement IResettable. /// internal class AdapterPool : IAsyncDisposable @@ -187,7 +187,7 @@ async Task ReturnAsync(ResidentAdapterInstance instance, string sessionId) /// /// Retires warm instances that have sat checked-in longer than the idle timeout, so a /// quiet pool shrinks back down instead of holding its peak size forever. Called - /// periodically by the host's supervisor loop (design doc 14.5's "idle eviction"). A no-op + /// periodically by the host's supervisor loop. A no-op /// when no idle timeout is configured for this adapter. /// public async Task EvictIdleAsync() diff --git a/SW.Serverless/Resident/AdapterProcessLauncher.cs b/SW.Serverless/Resident/AdapterProcessLauncher.cs index 09b1d88..d604353 100644 --- a/SW.Serverless/Resident/AdapterProcessLauncher.cs +++ b/SW.Serverless/Resident/AdapterProcessLauncher.cs @@ -10,7 +10,7 @@ namespace SW.Serverless.Resident { /// /// Spawns an adapter as a local child process and hands it the handshake on stdin. - /// Runtime tuning is applied through environment variables — design doc 11.1. + /// Runtime tuning is applied through environment variables. /// internal class AdapterProcessLauncher { @@ -111,7 +111,7 @@ public Process Launch(AdapterSpec spec, ResidentAdapterInstance instance) /// /// One file write that converts the worst outage mode into a supervised restart: under - /// memory pressure the kernel kills an adapter rather than the host (design doc 11.2). + /// memory pressure the kernel kills an adapter rather than the host. /// static void ApplyUnixHardening(Process process, AdapterSpec spec) { diff --git a/SW.Serverless/Resident/AdapterSpec.cs b/SW.Serverless/Resident/AdapterSpec.cs index 1626615..d7f47d5 100644 --- a/SW.Serverless/Resident/AdapterSpec.cs +++ b/SW.Serverless/Resident/AdapterSpec.cs @@ -15,7 +15,7 @@ public class AdapterSpec /// /// Path to the entry assembly. When null the configured locator resolves it — which is - /// where the existing S3 download-and-extract step plugs in unchanged (design doc 15.2). + /// where the existing S3 download-and-extract step plugs in unchanged. /// public string EntryAssemblyPath { get; set; } diff --git a/SW.Serverless/Resident/IResidentAdapterHost.cs b/SW.Serverless/Resident/IResidentAdapterHost.cs index 4516240..1a8a360 100644 --- a/SW.Serverless/Resident/IResidentAdapterHost.cs +++ b/SW.Serverless/Resident/IResidentAdapterHost.cs @@ -23,7 +23,7 @@ Task InvokeAsync(string command, object input = null, /// /// Owns every long-lived adapter process on this node. Registered as a SINGLETON — a request - /// scope ending must never kill a broker connection (design doc 4). + /// scope ending must never kill a broker connection. /// public interface IResidentAdapterHost { @@ -58,7 +58,7 @@ Task RestartAsync(string adapterId, string instanceKey, /// /// Check out one warm instance from a pool of stateless workers. Replaces a per-invocation /// process spawn. Only for adapters declaring Poolable — a process-static field would - /// otherwise leak across sessions (design doc 14.5). + /// otherwise leak across sessions. /// Task RentAsync(AdapterSpec spec, CancellationToken cancellationToken = default); diff --git a/SW.Serverless/Resident/InstanceHealth.cs b/SW.Serverless/Resident/InstanceHealth.cs index 7d00a19..361ab2a 100644 --- a/SW.Serverless/Resident/InstanceHealth.cs +++ b/SW.Serverless/Resident/InstanceHealth.cs @@ -7,7 +7,6 @@ namespace SW.Serverless.Resident /// One instance as the node-health view sees it. Two independent sources deliberately: /// HOST-OBSERVED figures (memory, CPU, restarts) need no adapter cooperation, so they still /// work when the adapter is wedged; ADAPTER-REPORTED ones carry provider detail. - /// Design doc, section 6.3. /// public class InstanceHealth { diff --git a/SW.Serverless/Resident/ResidentAdapterHost.cs b/SW.Serverless/Resident/ResidentAdapterHost.cs index e446faf..f38d196 100644 --- a/SW.Serverless/Resident/ResidentAdapterHost.cs +++ b/SW.Serverless/Resident/ResidentAdapterHost.cs @@ -194,7 +194,7 @@ async Task SpawnAsync(Supervised supervised, CancellationToken cancellationToken spec.AdapterValues = supervised.RequestedAdapterValues; // Resolves an explicit path, or installs from cloud storage — which is what makes - // "add a provider without redeploying" real (design doc 15.2). + // "add a provider without redeploying" real. var resolved = await locator.ResolveAsync(spec, cancellationToken); spec.EntryAssemblyPath = resolved.EntryAssemblyPath; spec.Executable = resolved.Executable ?? spec.Executable; @@ -698,7 +698,7 @@ async Task HeartbeatAsync(Supervised supervised) if (instance.State != InstanceState.Ready) return; // Host-observed metrics need no adapter cooperation, so they still work when the - // adapter is wedged (design doc 6.3). + // adapter is wedged. SampleProcess(supervised, instance); try diff --git a/SW.Serverless/Resident/ResidentAdapterInstance.cs b/SW.Serverless/Resident/ResidentAdapterInstance.cs index 74867b1..ae9849d 100644 --- a/SW.Serverless/Resident/ResidentAdapterInstance.cs +++ b/SW.Serverless/Resident/ResidentAdapterInstance.cs @@ -19,7 +19,7 @@ public enum InstanceState { Spawning, Attached, Ready, Draining, Stopped, Quaran /// /// One live adapter process, as the host sees it. Callers hold a HANDLE, never ownership — - /// a DI scope ending must not kill a broker connection (design doc 4). + /// a DI scope ending must not kill a broker connection. /// public sealed class ResidentAdapterInstance : IAsyncDisposable { @@ -30,7 +30,7 @@ public sealed class ResidentAdapterInstance : IAsyncDisposable readonly IAdapterStateStore stateStore; // The correlation fix: every outstanding call is keyed, so a late reply can never - // resolve an unrelated one the way the single v1 field did (design doc 14.3). + // resolve an unrelated one the way the single v1 field did. readonly ConcurrentDictionary pending = new(); readonly Channel outbound = Channel.CreateUnbounded( @@ -420,7 +420,7 @@ public async Task InvokeAsync(string command, byte[] payload = null, : options.InvokeTimeout; // On timeout the call is REMOVED, so a late reply is discarded rather than - // resolving the next caller's completion — the v1 defect (design doc 14.3). + // resolving the next caller's completion — the v1 defect. call.Timer = new Timer(_ => { if (pending.TryRemove(id, out var c)) diff --git a/SW.Serverless/Resident/ResidentOptions.cs b/SW.Serverless/Resident/ResidentOptions.cs index 0fb2d92..45a8d94 100644 --- a/SW.Serverless/Resident/ResidentOptions.cs +++ b/SW.Serverless/Resident/ResidentOptions.cs @@ -20,14 +20,14 @@ public class ResidentOptions /// Missed heartbeats before the supervisor restarts the instance. public int MissedHeartbeatsBeforeRestart { get; set; } = 3; - /// Stderr lines kept per instance for crash forensics (design doc 6.7). + /// Stderr lines kept per instance for crash forensics. public int DiagnosticBufferLines { get; set; } = 200; /// Restarts allowed inside CrashLoopWindow before the instance is quarantined. public int CrashLoopThreshold { get; set; } = 5; public TimeSpan CrashLoopWindow { get; set; } = TimeSpan.FromMinutes(5); - /// Soft RSS ceiling; the watchdog asks the adapter to drain (design doc 11.1). + /// Soft RSS ceiling; the watchdog asks the adapter to drain. public long SoftMemoryLimitBytes { get; set; } = 0; public long HardMemoryLimitBytes { get; set; } = 0; diff --git a/SW.Serverless/SW.Serverless.csproj b/SW.Serverless/SW.Serverless.csproj index fc759cb..1b66877 100644 --- a/SW.Serverless/SW.Serverless.csproj +++ b/SW.Serverless/SW.Serverless.csproj @@ -7,8 +7,8 @@ SimplyWorks.Serverless latest MIT - https://github.com/simplify9/Serverless - https://github.com/simplify9/Serverless + https://github.com/simplify9/SW-Serverless + https://github.com/simplify9/SW-Serverless Simplify9 Simplify9 Open-source .NET framework for building and running serverless adapters and services. Provides process isolation, cloud storage integration, and ASP.NET Core dependency injection. See https://github.com/simplify9/SW-Serverless for full documentation. diff --git a/SW.Serverless/Services/ServerlessService.cs b/SW.Serverless/Services/ServerlessService.cs index c0b2625..4426050 100644 --- a/SW.Serverless/Services/ServerlessService.cs +++ b/SW.Serverless/Services/ServerlessService.cs @@ -118,7 +118,7 @@ Task StartAsync(string adapterId, AdapterMetadata adapterMetadata, string correl throw new Exception("Already started."); // Copy rather than mutate. The caller's dictionary is often a long-lived entity's own - // settings — Traxis passes agent.Settings straight in — so adding CorrelationId to it + // settings, passed straight in — so adding CorrelationId to it // leaked into that entity, and a second call with the same dictionary threw // "An item with the same key has already been added". var values = startupValues == null From 402f25504f105f730e7b28c7d33dd45c9a0113a9 Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 18:53:56 +0300 Subject: [PATCH 16/17] Document SW-Serverless on its own A README and docs written for anyone building a host or an adapter, with no particular application in mind: concepts, hosting, writing adapters in .NET, Python and Node, the sw-serverless CLI, the manifest, packaging and storage, contracts and the conformance kit, the protocol, extending the tooling with an application's own CLI, and compatibility. Installing the CLI from its GitHub releases. The resident adapters design, written for one application, moves to that application's repository. --- README.md | 480 +++++++++----------------- docs/README.md | 243 +++---------- docs/cli.md | 413 +++++++++++++++++++++++ docs/compatibility.md | 141 ++++++++ docs/concepts.md | 200 +++++++++++ docs/contracts.md | 330 ++++++++++++++++++ docs/extending.md | 369 ++++++++++++++++++++ docs/hosting.md | 467 +++++++++++++++++++++++++ docs/manifest.md | 227 +++++++++++++ docs/packaging-and-storage.md | 312 +++++++++++++++++ docs/protocol.md | 350 +++++++++++++++++++ docs/writing-adapters.md | 619 ++++++++++++++++++++++++++++++++++ sdk/node/README.md | 6 +- sdk/python/README.md | 7 +- 14 files changed, 3643 insertions(+), 521 deletions(-) create mode 100644 docs/cli.md create mode 100644 docs/compatibility.md create mode 100644 docs/concepts.md create mode 100644 docs/contracts.md create mode 100644 docs/extending.md create mode 100644 docs/hosting.md create mode 100644 docs/manifest.md create mode 100644 docs/packaging-and-storage.md create mode 100644 docs/protocol.md create mode 100644 docs/writing-adapters.md diff --git a/README.md b/README.md index 5be8699..d275ad1 100644 --- a/README.md +++ b/README.md @@ -1,387 +1,231 @@ +# SW-Serverless -# SW.Serverless - -[![GitHub Actions](https://github.com/simplify9/SW-Serverless/actions/workflows/nuget-publish.yml/badge.svg)](https://github.com/simplify9/SW-Serverless/actions/workflows/nuget-publish.yml) [![NuGet - SimplyWorks.Serverless](https://img.shields.io/nuget/v/SimplyWorks.Serverless.svg)](https://www.nuget.org/packages/SimplyWorks.Serverless) [![NuGet - SimplyWorks.Serverless.Sdk](https://img.shields.io/nuget/v/SimplyWorks.Serverless.Sdk.svg)](https://www.nuget.org/packages/SimplyWorks.Serverless.Sdk) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) -**SW.Serverless** is an open-source .NET framework for building and running serverless adapters and services. It provides a runtime environment for executing .NET applications as serverless functions with process isolation and communication through stdin/stdout. +SW-Serverless runs small programs, called **adapters**, as separate child processes of a host +application. The host installs an adapter from cloud storage by its id, starts it, calls its +commands by name, and stops it. Adapters can be written in .NET, Python, JavaScript or TypeScript, +or be any self-contained binary that speaks the protocol. The repository also has the tools to +create, build, test, version and publish adapters: the `sw-serverless` command-line tool and the +library behind it. -## NuGet Packages +## When to use it -| Package | Version | Downloads | -| ------- | ------- | --------- | -| `SimplyWorks.Serverless` | [![NuGet](https://img.shields.io/nuget/v/SimplyWorks.Serverless.svg)](https://www.nuget.org/packages/SimplyWorks.Serverless) | [![Downloads](https://img.shields.io/nuget/dt/SimplyWorks.Serverless.svg)](https://www.nuget.org/packages/SimplyWorks.Serverless) | -| `SimplyWorks.Serverless.Sdk` | [![NuGet](https://img.shields.io/nuget/v/SimplyWorks.Serverless.Sdk.svg)](https://www.nuget.org/packages/SimplyWorks.Serverless.Sdk) | [![Downloads](https://img.shields.io/nuget/dt/SimplyWorks.Serverless.Sdk.svg)](https://www.nuget.org/packages/SimplyWorks.Serverless.Sdk) | +Use SW-Serverless when your application needs pieces of code that: -## What's Included +- are added or updated without redeploying the application: publish a new adapter version to + storage, and the host installs it the next time it is asked for; +- must not take the application down when they fail, leak memory or hang: each adapter is its own + process, with timeouts, and resident adapters are supervised and restarted; +- are written by other teams or partners, possibly in other languages, against a contract your + application defines. -- **SW.Serverless**: Core serverless service library with dependency injection extensions for ASP.NET Core -- **SW.Serverless.Sdk**: SDK for developing serverless adapters with the `Runner` class and logging utilities -- **SW.Serverless.SampleWeb**: Example ASP.NET Core web application showing integration -- **SW.Serverless.Installer**: Command-line tool for packaging and deploying adapters to cloud storage +Typical examples are integrations with outside systems: one adapter per partner API, file format +or message broker. -## Quick Start +It is not a general function-as-a-service platform. There is no HTTP gateway, autoscaling or +multi-node scheduler: adapters run on the machine that runs the host. -### Install NuGet Packages +## The parts -```bash -dotnet add package SimplyWorks.Serverless -dotnet add package SimplyWorks.Serverless.Sdk +| Part | Package | What it is for | +|---|---|---| +| Host library | NuGet `SimplyWorks.Serverless` (`SW.Serverless`) | Add to the application that runs adapters: `AddServerless`, `IServerlessService`, `AddResidentAdapters`, `IResidentAdapterHost`. | +| .NET SDK | NuGet `SimplyWorks.Serverless.Sdk` (`SW.Serverless.Sdk`) | Write an adapter in .NET: `Runner.Run`, `Runner.RunResident`, `Runner.Expect`. | +| Python SDK | `sw-serverless` (`import sw_serverless`), in `sdk/python` | Write an adapter in Python 3.12 or later. No dependencies. | +| Node SDK | `@simplyworks/sw-serverless`, in `sdk/node` | Write an adapter in JavaScript or TypeScript on Node 22 or later. No dependencies. | +| Contract | NuGet `SimplyWorks.Serverless.Contract` (`SW.Serverless.Contract`) | The gRPC protocol (`adapter.proto`), the manifest (`adapter.json`) and catalog models. Shared by everything else. | +| Tooling | NuGet `SimplyWorks.Serverless.Tooling` (`SW.Serverless.Tooling`) | What the CLI does, as a library: build, conformance tests, scaffolding, publishing. For an application that builds its own tools. | +| CLI | `sw-serverless` (project `SW.Serverless.Installer`) | `init`, `build`, `test`, `run`, `manifest validate`, `publish`, `promote`, `versions`, `withdraw`. | + +The Python and Node SDKs are not on PyPI or npm yet. You do not need them there: +`sw-serverless build` copies the SDK into every Python and Node package it builds. They will be +published to PyPI and npm later. + +## Install the CLI + +Self-contained binaries are published on the +[GitHub releases](https://github.com/simplify9/SW-Serverless/releases) tagged `cli-v`, one +per platform: `sw-serverless-.tar.gz` for `linux-x64`, `linux-arm64`, `linux-musl-x64`, +`osx-x64` and `osx-arm64`, and `sw-serverless-win-x64.zip`, with a `SHA256SUMS` file. On Linux or +macOS, the install script picks your platform, checks the download and installs to `~/.local/bin`: + +```sh +curl -fsSL https://raw.githubusercontent.com/simplify9/SW-Serverless/main/scripts/install-cli.sh | sh ``` -### Add to ASP.NET Core +`INSTALL_DIR` changes where it goes; `SW_SERVERLESS_VERSION=10.2.0` picks a version. On Windows, +download `sw-serverless-win-x64.zip` from the release and put `sw-serverless.exe` on your `PATH`. -```csharp -// In Startup.cs or Program.cs -services.AddServerless(); +Or build it from source with the .NET 10 SDK: + +```sh +dotnet run --project SW.Serverless.Installer -- # run without installing +dotnet publish SW.Serverless.Installer -c Release -o ./out # ./out/sw-serverless ``` -### Create an Adapter +See [docs/cli.md](docs/cli.md) for every command. -```csharp -using SW.Serverless.Sdk; +## Quick start: an adapter -class Handler -{ - public async Task ProcessData(string input) - { - // Your serverless logic here - return $"Processed: {input}"; - } -} +`sw-serverless init` writes a small working adapter with two settings and two commands. The same +five commands then build, check, call and publish it, whatever its language. -class Program -{ - static async Task Main(string[] args) => await Runner.Run(new Handler()); -} +```sh +sw-serverless init Greeter # .NET; or --lang python, node, typescript +cd Greeter +sw-serverless build # -> bin/serverless/greeter-0.1.0.zip +cp settings.example.json settings.json +sw-serverless test --settings settings.json # runs it as a host would and checks it +sw-serverless run --settings settings.json --call Greet --input Ada +# Hello, Ada! +sw-serverless publish bin/serverless/greeter-0.1.0.zip -p local -b adapters-dev -u /tmp/swsl-store ``` -## Publishing Adapters (Installer) +`-p local` publishes to a folder, which is handy for trying things out. For S3, Azure, Google Cloud +or Oracle storage, see [storage providers](docs/cli.md#storage). -`SW.Serverless.Installer` builds an adapter project (`dotnet publish -c Release`), zips the output and uploads it to the store the runtime installs adapters from. The executable is named `serverless`. From a checkout: +What `init` writes, in each language: -```bash -dotnet run --project SW.Serverless.Installer -- [options] -``` +**.NET** (`Program.cs`) -```bash -# S3-compatible storage, next patch version -serverless -p s3 -a -s -b -u https://s3.example.com \ - -v patch ./MyAdapter/MyAdapter.csproj my.adapter +```csharp +using SW.Serverless.Sdk; -# Credentials from the environment (CI) -export SWSL_PROVIDER=s3 SWSL_ACCESS_KEY=... SWSL_SECRET_KEY=... SWSL_BUCKET=adapters SWSL_SERVICE_URL=https://s3.example.com -serverless -v minor ./MyAdapter/MyAdapter.csproj my.adapter +public class Adapter +{ + public Adapter() + { + // Declare settings here; read them in the commands, never in the constructor. + Runner.Expect("Greeting", "Hello", description: "What to say before the name."); + Runner.Expect("ApiKey", optional: true, isPrivate: true, description: "A key, to show how a secret is declared."); + } -# Oracle or Google Cloud: settings from a config file -serverless -c cloudfiles.json -v 2.1.0 ./MyAdapter/MyAdapter.csproj my.adapter -``` + [AdapterCommand(Description = "Greets someone by name.")] + public Task Greet(string name) => + Task.FromResult($"{Runner.StartupValueOf("Greeting")}, {name}!"); +} -| Flag | Meaning | -|---|---| -| `-p`, `--provider` | `s3`, `as` (Azure), `oc` (Oracle), `gc` (Google Cloud) or `local` (filesystem, for development) | -| `-a`, `--accesskey` | Access key | -| `-s`, `--secret` | Secret access key | -| `-b`, `--bucketname` | Bucket name | -| `-u`, `--url` | Service URL (for `local`, the storage folder) | -| `-c`, `--cloudfilesconfigpath` | JSON config file (see below) | -| `-v`, `--version` | `major`, `minor`, `patch`, or an explicit version such as `2.1.0` / `2.1.0-rc.1`. Omit to upload only `adapters/`, as before (see Versions below) | -| `-k`, `--kind` | Roles the adapter serves (e.g. `handler,mapper`), when it does not declare them with `[AdapterKind]` or in `adapter.json` | - -The adapter id may contain only lowercase letters, digits, `.`, `_` and `-`, and must start with a letter or digit (uppercase input is lowercased). - -**Where settings come from.** Each setting is taken from the first of: the command-line flag, the config file, then the environment variable. - -| Environment variable | Setting | -|---|---| -| `SWSL_PROVIDER` | Provider | -| `SWSL_ACCESS_KEY` | Access key | -| `SWSL_SECRET_KEY` | Secret access key | -| `SWSL_BUCKET` | Bucket name | -| `SWSL_SERVICE_URL` | Service URL | -| `SWSL_REGION` | Region | -| `SWSL_PUBLISHED_BY` | Who is publishing, recorded in the catalog (then `GITHUB_ACTOR`, then the user name) | -| `SWSL_GC_PROJECT_ID`, `SWSL_GC_PRIVATE_KEY_ID`, `SWSL_GC_PRIVATE_KEY`, `SWSL_GC_CLIENT_EMAIL`, `SWSL_GC_CLIENT_ID`, `SWSL_GC_CLIENT_X509_CERT_URL` | Google Cloud service account fields (`SWSL_GC_PRIVATE_KEY` may use literal `\n` for line breaks) | - -The config file holds the same settings under `CloudFiles`. Oracle settings (`Region`, `TenantId`, `UserId`, `FingerPrint`, `RSAKey`, `NamespaceName`) are read only from the file: - -```json +static class Program { - "CloudFiles": { - "Provider": "gc", - "BucketName": "adapters", - "ProjectId": "my-project", - "PrivateKeyId": "…", - "PrivateKey": "-----BEGIN PRIVATE KEY-----\n…\n-----END PRIVATE KEY-----\n", - "ClientEmail": "publisher@my-project.iam.gserviceaccount.com", - "ClientId": "…" - } + static Task Main() => Runner.Run(new Adapter()); } ``` -**What is uploaded.** The entry assembly is the project's real `AssemblyName` (checked to exist in the publish output). Every package carries an `adapter.json` manifest (below), and every upload carries metadata: `EntryAssembly`, `Lang`, `Timestamp`, `Lifecycle` and `Kind` (read from the assembly), `Sha256` and `Hash` (hex SHA-256 of the zip; `Hash` is what hosts name the extraction folder after — S3 keeps using its ETag), and `Version` (empty for an unversioned upload). A file that cannot be read fails the packaging rather than being left out. +**Python** (`main.py`) -The tool exits `0` only when the upload completed; a bad command line, an invalid adapter id, an invalid manifest, a failed build or a failed upload all exit non-zero. +```python +import sw_serverless as sw -### Versions, promote and rollback -```bash -# Publish 1.4.0 and make it the one that runs (the default) -serverless -v 1.4.0 --notes "Retries on 503" ./MyAdapter/MyAdapter.csproj my.adapter +class Greeter: + def __init__(self): + sw.expect("Greeting", "Hello", description="What to say before the name.") + sw.expect("ApiKey", secret=True, required=False, description="A key, to show how a secret is declared.") -# Publish without switching: stage it, check it, then promote -serverless -v minor --no-promote ./MyAdapter/MyAdapter.csproj my.adapter -serverless promote my.adapter 1.5.0 + @sw.command("Greet", description="Greets someone by name.") + def greet(self, name: str) -> str: + return f"{sw.value_of('Greeting')}, {name}!" -# Roll back = promote an older version (no rebuild) -serverless promote my.adapter 1.4.0 -# History, and taking a bad version out of use -serverless versions my.adapter -serverless withdraw my.adapter 1.5.0 +if __name__ == "__main__": + sw.run(Greeter) ``` -The commands take the same storage flags, `-c` file and `SWSL_*` variables as publishing. - -| Flag | Meaning | -|---|---| -| `--no-promote` | With `-v`: upload the version and record it, but leave what runs alone | -| `--notes "…"` | Release notes for this version (overrides `releaseNotes` in `adapter.json`) | -| `--published-by` | Recorded in the catalog; defaults to `SWSL_PUBLISHED_BY`, then `GITHUB_ACTOR`, then the user name | -| `--no-probe` | Do not start a classic adapter to ask which startup values it expects (see below) | - -- **Publish with `-v`** uploads the package to `adapters-versions//` (immutable: an existing version is refused) and, unless `--no-promote`, the same package to `adapters/` — the key every host runs — then records the version in the catalog. `major`, `minor` and `patch` bump the highest released version already published (or start at `1.0.0`); pre-release and other non-version keys are ignored when finding it. An explicit version must be higher than every released version. -- **Publish without `-v`** works exactly as it always has: only `adapters/` is written. The catalog's manifest follows it and its current version is cleared (an unversioned package is running); the version history is kept. -- **`promote `** downloads the version, checks its SHA-256 against the one recorded when it was published, and copies it over `adapters/` with the full metadata (including anything else the version carried, such as `Protocol`). A withdrawn or unknown version is refused. -- **`withdraw `** marks the version withdrawn in the catalog: listed for history, refused by `promote`, not offered for pinning. The current version cannot be withdrawn — promote another first. The package itself stays, since a deployment may pin it. -- **`versions `** lists versions, newest first, with publish time, publisher, a short SHA-256, `*` for the current one and `withdrawn`. For an adapter published before the catalog it lists the packages instead. +**TypeScript** (`main.ts`; JavaScript is the same without the types, using `require`) -A host runs a specific version when asked for the adapter id `/`; a plain `` runs whatever is current. +```ts +import { expect, run, valueOf } from "@simplyworks/sw-serverless"; -### Storage layout +class Greeter { + static commands = { + Greet: { method: "greet", input: "string", output: "string", description: "Greets someone by name." }, + }; -| Key | What | Written by | -|---|---|---| -| `adapters/` | The package that runs when no version is pinned, with the full metadata above | publish (unversioned, or `-v` without `--no-promote`), `promote` | -| `adapters-versions//` | One immutable package per version | publish with `-v` | -| `adapters-catalog/.json` | The catalog entry: current version, current manifest and SHA-256, the icon as a `data:` URI, and every version with its manifest, digest, time, publisher and withdrawn flag | every command but `versions` | -| `adapters//` | Where installers before the catalog put versions. Still read (`versions`, `promote`, pinned refs), never written | — | - -Versions and the catalog live **beside** `adapters/`, not under it: storage backed by a file system cannot keep `adapters/` as a file and a folder at once, and hosts and Bitween builds older than the catalog treat every key under `adapters/` as an adapter. An adapter published by an older installer has no catalog entry until its next publish or promote, which create one from the packages and their metadata. - -### The adapter manifest (`adapter.json`) - -Put an `adapter.json` beside the project file to describe the adapter for a catalog or marketplace. It is optional: without one, the installer still writes a manifest into the package from what it can find out. The installer **merges** it: the author owns presentation, the installer owns the facts and overwrites them. + constructor() { + expect("Greeting", { default: "Hello", description: "What to say before the name." }); + expect("ApiKey", { secret: true, required: false, description: "A key, to show how a secret is declared." }); + } -```json -{ - "displayName": "Acme Carrier", - "summary": "Creates shipments and labels with Acme.", - "description": "Longer **Markdown** description.", - "publisher": { "name": "Simplify9", "url": "https://simplify9.com", "email": "support@simplify9.com" }, - "license": "MIT", - "homepage": "https://example.com/acme", - "repository": "https://github.com/example/acme-adapter", - "icon": "assets/icon.png", - "tags": [ "shipping", "labels" ], - "categories": [ "Carriers" ], - "kinds": [ "handler" ], - "releaseNotes": "Retries on 503.", - "compatibility": { "minHostVersion": "10.0.0", "minBitweenVersion": "9.2.0" }, - "properties": [ - { "name": "BaseUrl", "displayName": "API URL", "type": "text", "default": "https://api.acme.test", "group": "Connection" }, - { "name": "ApiKey", "type": "text", "required": true, "secret": true, "group": "Connection" }, - { "name": "Mode", "type": "select", "options": [ "test", "live" ], "default": "test" } - ] + greet(name: string): string { + return `${valueOf("Greeting")}, ${name}!`; + } } -``` -| Field | Owner | Meaning | -|---|---|---| -| `manifestVersion` | installer | `1`; never lowered when an author file from a newer tool says more | -| `id` | installer | The adapter id published | -| `version` | installer | The version published; absent for an unversioned upload | -| `displayName`, `summary`, `description` | author | Name, one line, longer Markdown | -| `publisher` | author | `{ name, url, email }` | -| `license`, `homepage`, `repository` | author | | -| `icon` | author | PNG, JPEG or SVG **inside the package**, relative to its root — include it in the publish output (`CopyToPublishDirectory`). A missing file fails the publish; one of 64 KB or less is also inlined in the catalog as a `data:` URI | -| `tags`, `categories` | author | Lists of strings | -| `releaseNotes` | author / `--notes` | What changed in this version | -| `kinds` | author | handler, mapper, receiver, validator… `--kind` wins over it; without either, the `[AdapterKind]` attributes in the assembly are used | -| `runtime` | installer | `dotnet` | -| `language` | author | Defaults from the project file: `csharp`, `fsharp` or `vb` | -| `entry` | installer | The entry assembly | -| `lifecycle` | installer | `classic` or `resident`, read from the assembly | -| `protocol` | installer | `{ "min": 2, "max": 2 }` for resident adapters; absent for classic | -| `sdkVersion` | installer | The `SimplyWorks.Serverless.Sdk` version it was built against | -| `publishedOn` | installer | When it was published | -| `compatibility` | author | `minHostVersion`, `minBitweenVersion` | -| `properties` | author / probe | What has to be configured: `name`, `displayName`, `description`, `type` (`text`, `multiline`, `number`, `boolean`, `select`, `json`), `required`, `secret`, `default`, `options` (for `select`), `group` | - -Fields the installer does not know are kept and written back, so a manifest from a newer tool survives. The final manifest is validated (id and version format, paths inside the package, lifecycle, property names, types and duplicates) and a problem fails the publish before anything is uploaded. - -**Properties.** If `adapter.json` declares `properties`, those are used. Otherwise, for a classic adapter, the installer starts the built adapter the way a host does and asks it for its expected startup values (`Runner.Expect`): `required` is the inverse of optional, `secret` is private, and `default` and `description` are carried over. This runs the adapter's constructor on the build machine; `--no-probe` skips it. A probe that fails is a warning, not a failed publish — the manifest then lists no properties. Resident adapters are not probed. - -### Backward compatibility - -Ten production deployments run hosts on `SimplyWorks.Serverless` 10.0.x and Bitween builds that predate the catalog. What this installer writes is held to what they read: - -- `adapters/` always holds the current package with the full metadata an old host needs (`EntryAssembly`, `Hash`, and everything it wrote before) — after a versioned publish, a promote, a rollback or an unversioned publish alike. -- Nothing is ever written under `adapters/` except `adapters/`, so an old Bitween listing sees exactly the adapters that exist. -- Every existing command line works unchanged; without `-v` the upload is the same key with the same metadata, plus `adapter.json` inside the zip and an empty `Version`. -- A package without `adapter.json`, and an adapter with no catalog entry, still install, run, list, and can be promoted. - -One limit: an old host asked for a pinned ref `/` looks only at `adapters//`, so it cannot pin versions published by this installer (those are under `adapters-versions/`). Old hosts never used pinning; current hosts resolve pinned refs in both places. - -`SW.Serverless.CompatibilityTests` holds these to account: a host built on the published 10.0.0 package (`SW.Serverless.Compat.OldHost`) installs and runs what this installer publishes, through promote and rollback; the current host runs packages made the old way and adapters built on the published 10.0.0 SDK, classic and resident; the old Bitween listing rule sees nothing new; and manifests and catalog entries with unknown fields round-trip. - -## Resident Adapters - -A classic adapter is launched per invocation and exits when it returns. A **resident** adapter is -launched once and stays running, so it can hold state — an open broker connection, a pooled HTTP/2 -channel, a warm cache — and push work into the host as well as receive it. - -```mermaid -sequenceDiagram - participant H as Host - participant S as Cloud storage - participant A as Adapter process - - H->>S: fetch + extract by adapter id - H->>A: launch, with memory and CPU ceilings - A->>H: dial back over UDS / named pipe (gRPC) - A->>H: Hello — commands, capabilities, SDK version - - Note over H,A: process stays up - - loop while running - H->>A: Invoke(command, argument) - A-->>H: result - A->>H: Publish(event) - H-->>A: accepted / rejected - H->>A: Ping - A-->>H: status, counters, last error - end - - H->>A: Stop(drain) - A-->>H: finishes in flight, exits +run(Greeter); ``` -The adapter dials **out** to the host over a Unix domain socket or a named pipe — nothing listens -on a TCP port, and the adapter needs no inbound reachability. +More in [docs/writing-adapters.md](docs/writing-adapters.md). -### Two directions +## Quick start: a host -| | | -| --- | --- | -| **Host → adapter** | `InvokeAsync("Command", argument)` — any public `Task`/`Task` method on the handler, discovered by reflection. | -| **Adapter → host** | `context.PublishAsync(payload, dedupeKey, endpoint, …)` — the host persists, then answers accepted or rejected. The adapter acknowledges its source only after that. | +Add the host library and a storage provider to your application: -### Writing one +```sh +dotnet add package SimplyWorks.Serverless +dotnet add package SimplyWorks.CloudFiles.LocalTests.Extensions # or .S3.Extensions, .AS.Extensions, ... +``` ```csharp -using SW.Serverless.Sdk; -using SW.Serverless.Sdk.Resident; - -class Handler : IResidentAdapter -{ - IAdapterContext _context; - - public Task StartAsync(IAdapterContext context, CancellationToken ct) - { - _context = context; // connect, subscribe, warm up - return Task.CompletedTask; - } - - public Task StopAsync(CancellationToken ct) => Task.CompletedTask; +using SW.CloudFiles.Extensions; +using SW.PrimitiveTypes; +using SW.Serverless; +using SW.Serverless.Resident; - public Task GetStatusAsync() => - Task.FromResult(new AdapterStatus { Connected = true, State = "Ready" }); - - [AdapterCommand("What this command does.")] - public Task GetStats() => Task.FromResult(new { ok = true }); -} +var builder = WebApplication.CreateBuilder(args); -class Program +// Where adapters are published: the same store the CLI published to above. +builder.Services.AddLocalTestsCloudFiles(o => { - static Task Main() => Runner.RunResident(new Handler()); -} -``` + o.BucketName = "adapters-dev"; + o.StoragePath = "/tmp/swsl-store"; +}); -Host side: +builder.Services.AddServerless(); +// Needed for Python, Node and exec adapters, and for any resident adapter. +builder.Services.AddResidentAdapters(); -```csharp -services.AddResidentAdapters(o => -{ - o.HeartbeatInterval = TimeSpan.FromSeconds(15); - o.SoftMemoryLimitBytes = 512L * 1024 * 1024; -}); +var app = builder.Build(); -var instance = await adapters.StartExclusiveAsync(new AdapterSpec +app.MapGet("/greet/{name}", async (string name, IServiceProvider services) => { - AdapterId = "my.adapter", - InstanceKey = "1", - StartupValues = { ["Host"] = "broker.example.com" }, + using var scope = services.CreateScope(); // the session ends with the scope + var serverless = scope.ServiceProvider.GetRequiredService(); + await serverless.StartAsync("greeter", correlationId: Guid.NewGuid().ToString(), + new Dictionary { ["Greeting"] = "Hi" }); + return await serverless.InvokeAsync("Greet", name); }); -``` - -### Supervision - -The host samples each adapter process on every heartbeat and acts on what it finds. - -```mermaid -flowchart LR - Sample[Heartbeat sample] --> Check{What did it find?} - Check -- healthy --> Sample - Check -- over soft memory or CPU --> Drain[Ask to drain, relaunch] - Check -- over hard memory --> Kill[Kill process tree, relaunch] - Check -- no answer --> Miss[Restart after N misses] - Drain --> Quarantine[Repeated crashes: quarantine] - Kill --> Quarantine - Miss --> Quarantine -``` - -| Control | Effect | -| --- | --- | -| `SoftMemoryLimitBytes` | Asks the adapter to drain — in-flight work finishes, nothing is lost. | -| `HardMemoryLimitBytes` | Kills the process tree, and is applied to the runtime as `DOTNET_GCHeapHardLimit`. | -| `CpuPercentLimit` + `CpuLimitSamples` | Trips after N consecutive samples above the line, then asks to drain. The figure is a share of the whole machine, not of one core. | -| `UpdateLimitsAsync` | Changes ceilings on a running adapter. Soft and CPU apply on the next sample; a hard-memory change reports `RestartRequired`. | -| `RestartAsync` | Relaunches in place, keeping the instance key. | -### Discovery +app.Run(); -Every public `Task`/`Task` method on a handler is a command. The adapter reports them on attach, -with the shape needed to call one: - -```csharp -foreach (var c in adapters.Describe().Single(h => h.InstanceKey == "1").CommandDetails) - Console.WriteLine($"{c.Name}({c.ParameterType}) {c.ParameterSchema} — {c.Description}"); +// Receives events that resident adapters publish. This one accepts and drops them. +class IgnoreEvents : IAdapterEventSink +{ + public Task OnEventAsync(InboundEvent inboundEvent, CancellationToken cancellationToken) => + Task.FromResult(EventOutcome.Ok("ignored")); +} ``` -`ParameterSchema` lists a complex argument's properties as name → type, so a caller can build a form -for a command it has not seen before. - -## Features - -- **Process Isolation**: Each adapter runs in its own process with timeout management -- **Cloud Storage Integration**: Support for AWS S3, Azure Storage, and Oracle Cloud -- **Dependency Injection**: Built-in integration with ASP.NET Core DI container -- **Logging**: Structured logging support through `AdapterLogger` -- **Caching**: Adapter metadata caching with configurable duration -- **Version Management**: Semantic versioning support for adapter deployments - -## Architecture +The host machine needs the runtimes its adapters use on the `PATH`: `dotnet` for .NET adapters, +`python3` (3.12 or later) for Python, `node` (22 or later) for JavaScript and TypeScript. More in +[docs/hosting.md](docs/hosting.md). -The framework consists of: +## Documentation -1. **ServerlessService**: Manages adapter lifecycle, installation, and invocation -2. **Runner**: Entry point for adapter applications with command processing -3. **Installer**: CLI tool for building and deploying adapter packages -4. **Cloud Storage**: Abstraction layer for different cloud storage providers +- [Documentation index](docs/README.md) +- [Concepts](docs/concepts.md): host, adapter, package, manifest, versions, lifecycles, protocols, contracts +- [Hosting adapters](docs/hosting.md): the host library, sessions, resident adapters, limits, security +- [Writing adapters](docs/writing-adapters.md): .NET, Python and Node side by side +- [The CLI](docs/cli.md): every command, flag and environment variable +- [The manifest](docs/manifest.md): `adapter.json`, field by field +- [Packaging and storage](docs/packaging-and-storage.md): what a package holds and where it is stored +- [Contracts](docs/contracts.md): defining what adapters for your application must do, and testing it +- [The protocol](docs/protocol.md): for SDKs in other languages and `exec` adapters +- [Extending the tools](docs/extending.md): your own CLI or server on `SW.Serverless.Tooling` +- [Compatibility](docs/compatibility.md): what stays working across versions -## Support +## License -For issues, questions, or contributions, please visit the [GitHub repository](https://github.com/simplify9/SW-Serverless). +MIT. See [LICENSE](LICENSE). diff --git a/docs/README.md b/docs/README.md index 51b626e..c7b10c0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,213 +1,60 @@ -# Resident adapters — protocol 2 +# SW-Serverless documentation -The design lives in [resident-adapters-design.md](resident-adapters-design.md). This page is the -short version plus how to run the samples. +Start with [Concepts](concepts.md). It explains the words the other pages use. Then read the page +for what you are doing. -## What changed, in one line per side +## If you are building an application that runs adapters -```csharp -// adapter — the ONLY difference. Same zip, same S3 metadata, same install, same spawn. -static Task Main() => Runner.Run(new Handler()); // classic, per invocation -static Task Main() => Runner.RunResident(new Handler()); // stays running -``` - -```csharp -// host -services.AddResidentAdapters(o => o.HeartbeatInterval = TimeSpan.FromSeconds(15)); -``` +1. [Concepts](concepts.md): host, adapter, package, manifest, catalog, versions, lifecycles, + protocols, runtimes, contracts, settings. +2. [Hosting adapters](hosting.md): adding the host library, classic sessions, resident adapters, + the event sink and state store, storage, pinning versions, limits, what the host machine needs, + security. +3. [Contracts](contracts.md): describing what your application expects of its adapters, and + checking adapters against it. +4. [Extending the tools](extending.md): your own CLI or server on `SW.Serverless.Tooling`, with + your own templates, packages and contracts. -Classic adapters are **untouched**. An adapter whose cloud metadata has no `Protocol` key takes -the v1 code path byte for byte — which is what keeps the existing fleet alive. +## If you are writing adapters -## Dependency injection in an adapter - -`Runner.Run(new Handler())` still works and is unchanged. When an adapter grows past a single -class, `AdapterHost` gives it the same shape as any .NET service: - -```csharp -static Task Main() => AdapterHost.CreateBuilder() - .ConfigureServices((configuration, services) => - { - services.Configure(configuration); // bound from startup values - services.AddSingleton(); - services.AddHttpClient(); - }) - .Build() - .RunResidentAsync(); // or .RunAsync() for the classic path -``` +1. [Concepts](concepts.md). +2. [Writing adapters](writing-adapters.md): settings, commands, encoding, errors, logging, + resident hooks, the adapter context, `--describe`. .NET, Python and Node side by side. +3. [The CLI](cli.md): `init`, `build`, `test`, `run`, `publish` and the rest. +4. [The manifest](manifest.md): `adapter.json`. +5. [Packaging and storage](packaging-and-storage.md): what a package contains and where it goes. -You get, without asking for it: +## Reference -* **`ILogger`** routed onto the adapter's log channel, so anything your services — or the - libraries they use — log reaches the host under `serverless.adapters.{id}`. -* **`IConfiguration`** built from startup values, with cloud metadata namespaced under - `AdapterValues:` so it can never shadow them. -* **`IAdapterContext`** for pushing events and metrics, injectable anywhere — and for reading and - writing small durable state the host holds on the adapter's behalf (`GetStateAsync` / - `SetStateAsync`), which is where a polling receiver keeps its cursor, and for per-call - configuration on a shared instance (`ValueOf` / `InvocationValues`). -* **`AdapterSession.Id`** — ambient per-invocation identity, and the boundary a pooled adapter - needs so state cannot leak between checkouts. +- [The CLI](cli.md): every command, flag, default, environment variable and exit code. +- [The manifest](manifest.md): every `adapter.json` field and its validation rules. +- [The protocol](protocol.md): the stdin handshake, the gRPC stream and its frames, the encoding + rules, the `--describe` output. For writing an SDK in another language or an `exec` adapter. +- [Compatibility](compatibility.md): what keeps working with older hosts, SDKs and packages. -**This also removes the constructor footgun.** The container is built *after* startup values -arrive, so injecting `IOptions` into a constructor is safe — unlike -`Runner.Run(new Handler())`, where the handler exists before argv has been parsed. +## Samples in this repository -Only `SWSL_`-prefixed environment variables are bound, and startup values outrank them. Binding -the environment unprefixed is a trap: a setting named `Path` picks up the machine's `PATH`. - -## Run the samples - -Two hosts, for two kinds of look. Build the solution first — both package adapters from their -build output. - -For the RabbitMQ pair, start a broker first — without one those two adapters simply do not start -and the rest of the dashboard is unaffected, which is the intended failure mode: - -```bash -docker run -d --rm -p 5672:5672 -p 15672:15672 rabbitmq:3.13-management -``` - -**Web dashboard** — live observability, both lifecycles side by side: - -```bash -dotnet run --project SW.Serverless.SampleWeb -``` - -**Console** — the same runtime with no UI, if you would rather read a log: - -```bash -dotnet run --project SW.Serverless.Samples.Host -``` - -Both start by packaging the sample adapters into a local-filesystem cloud store -(`AddLocalTestsCloudFiles`, no credentials) and then starting them **by adapter id only** — so -the download, extract and launch steps are exercised, not skipped. That is the -"install a provider without redeploying" claim actually running. - -| Sample | What it is for | +| Project | What it shows | |---|---| -| `SW.Serverless.Samples.Ticker` | Smallest resident adapter. Proves attach, push/ack, heartbeat, runtime reconfiguration and typed command errors. No dependencies. | -| `SW.Serverless.Samples.FolderSource` | The reference **data source** shape — ingress, egress, topology, discovery, test-connection, real status — using a folder instead of a broker. A RabbitMQ or Kafka adapter is this class with a different client. | -| `SW.Serverless.Samples.RabbitMq` | Shared connection handling for the two broker samples. Everything is a startup value — host, vhost, exchange type, queue arguments — because a provider must not be opinionated about the broker's own model. Reconnection is deliberately **not** retried in a loop: the adapter reports itself disconnected and lets the supervisor decide, where backoff and crash-loop quarantine already live. | -| `SW.Serverless.Samples.RabbitPublisher` | **Egress.** Publishes every 10 ms (~100/s) with publisher confirms and `mandatory: true`, so unroutable messages come back through `BasicReturn` instead of vanishing. `SetInterval` changes the rate while running. | -| `SW.Serverless.Samples.RabbitConsumer` | **Ingress.** Declares a queue and binding, consumes with `autoAck: false`, and **only calls `BasicAck` after the host has acknowledged**. A host rejection becomes `BasicNack(requeue: true)`. This is the ordering to copy for a Kafka offset commit. `SetPrefetch` changes the broker-side backpressure dial at runtime. | -| `SW.Serverless.Samples.LargeFiles` | **Streaming, and what visibility looks like under load.** Streams a file in configurable chunks and pushes each to the host, so resident memory tracks the *chunk* size and not the file size. Reports percent, MB/s, ETA and its own working set on the heartbeat, which the dashboard renders as a live progress bar. `GenerateTestFile` makes a file of any size on demand; `Pause` / `Resume` hold a transfer mid-file. Built entirely on constructor injection — options, a reader service, a throughput meter and an `ILogger`. | -| `SW.Serverless.Samples.Carrier` | **A typical adapter, resident.** Does what a Traxis agent adapter does — takes the host's shipment type, translates it to a carrier's gRPC API, calls it with deadlines and retries, returns the host's result type. The host invokes it *exactly* as it invokes a classic adapter; what changes is underneath. Built on constructor injection: options, a pooled gRPC client, a session-scoped call log, an `ILogger`. Implements `IResettable`, so it is safely poolable. | -| `SW.Serverless.Samples.CarrierContract` | The carrier's own `.proto`. It belongs to the carrier, not to SW.Serverless — the adapter's job is translating between it and the host's SDK types. | -| `SW.Serverless.Samples.Classic` | A conventional **non-resident** adapter — `Runner.Run`, no `Protocol` metadata key, so the host takes the v1 path for it byte for byte. Shows startup values and the `{{expected}}` schema, typed commands, `AdapterLogger`, failures and timeouts, and process-static state. | -| `SW.Serverless.Samples.Host` | Console host. Implements `IAdapterEventSink`, starts both adapters, prints events and heartbeats. | -| `SW.Serverless.SampleWeb` | Blazor Server dashboard plus minimal APIs. Live adapter health, event feed, adapter logs and metrics, failure injection, and a page for the classic per-invocation lifecycle to contrast against. | +| `SW.Serverless.Samples.Classic` | A classic .NET adapter: settings, typed commands, `AdapterLogger`, failures and timeouts. | +| `SW.Serverless.Samples.Ticker` | The smallest resident .NET adapter: attach, events, heartbeat status, commands that change it while it runs. | +| `SW.Serverless.Samples.FolderSource` | A resident adapter that reads files from a folder and publishes each as an event, archiving the file only after the host accepts it. | +| `SW.Serverless.Samples.LargeFiles` | Streaming a large file in chunks, with `AdapterHost` dependency injection. | +| `SW.Serverless.Samples.RabbitMq`, `.RabbitPublisher`, `.RabbitConsumer` | Resident adapters for a RabbitMQ broker: publishing with confirms, consuming with acknowledgement after the host accepts. | +| `SW.Serverless.Samples.Greedy` | An adapter that uses memory and CPU on demand, for exercising limits. | +| `SW.Serverless.SampleWeb` | A web host with a dashboard of resident adapters, events, logs and metrics. `dotnet run --project SW.Serverless.SampleWeb`. | +| `SW.Serverless.UnitTests/PythonAdapters`, `/NodeAdapters` | Small Python, JavaScript and TypeScript adapters, classic and resident, used by the tests. | +| `SW.Serverless.Installer.UnitTests/Contracts` | The sample "orders" contract used in [Contracts](contracts.md). | -### What the dashboard shows - -| Page | Why it is there | -|---|---| -| **Adapters** | Health from two independent sources — host-observed memory, CPU, threads, restarts and missed heartbeats, which keep working when an adapter is wedged; and adapter-reported state and provider detail from the heartbeat. Buttons invoke commands, toggle debug logging per instance at runtime, and kill a process so you can watch the supervisor restart it with backoff and then quarantine it. | -| **Events** | The push direction with its ack outcome and the host's reference. "Reject the next 3 events" on the Adapters page proves the ordering: a rejected event leaves the file in place, it is redelivered, and the dedupe key makes the second delivery recognisable. | -| **Logs & metrics** | Adapter log frames arriving as ordinary `ILogger` entries under `serverless.adapters.{id}`, and metric frames read back through a `MeterListener` on `System.Diagnostics.Metrics` — the same path any real exporter would use. | -| **Classic lifecycle** | The unchanged v1 path, running `SW.Serverless.Samples.Classic`. **Who am I?** returns the adapter's own pid: click it repeatedly and the classic pid changes every time while the resident pid beside it never moves. That spawn cost is what the pooled resident shape removes. Also covers the `{{expected}}` startup-value schema, typed in/out commands, a deliberate failure, and a timeout. | +The RabbitMQ samples need a broker, for example +`docker run -d --rm -p 5672:5672 rabbitmq:3.13-management`. -## The transport +## Running the tests -The adapter **dials the host** over a Unix domain socket (`/tmp/swsl-.sock`) or a named pipe -on Windows, and opens one bidirectional gRPC stream. A UDS is a filesystem path, not a network -address: **no port, no bind address, no firewall rule, and no long-lived network authentication -to configure.** The host validates a one-time handshake token on attach, so the endpoint is not -unauthenticated — it simply needs no credential management. The same contract binds to TCP + TLS for Kubernetes-orchestrated adapters, so both -modes share every line of code above the transport. See design section 15. - -The host writes the socket path and a one-time token to the child's **stdin**, not `argv` — which -is also how broker credentials stop showing up in `ps aux`. - -## The parts worth copying into a real provider - -* **Ack ordering** (`FolderSource.DeliverAsync`) — the file is archived only *after* the host - acknowledges. Crash in between and it is redelivered, which is exactly why every event carries - a dedupe key. Copy this shape for a Kafka offset commit or a RabbitMQ `basic.ack`. -* **Status, not liveness** (`GetStatusAsync`) — separates *disconnected* from *idle* from - *working*. A plain liveness probe conflates all three. -* **Bounded, droppable logs** — logging can never apply backpressure to the data path; events - never drop. -* **Credit window** — `MaxInFlight` bounds unacknowledged events, so an adapter reading faster - than the host persists cannot buffer its way to an OOM. - -## Calling a resident adapter like a classic one - -```csharp -// Classic — a process per call -using var scope = services.CreateScope(); -var serverless = scope.ServiceProvider.GetRequiredService(); -await serverless.StartAsync(adapterId, correlationId, settings); -var result = await serverless.InvokeAsync("CreateShipment", request); - -// Resident — a warm instance from the pool -await using var lease = await adapters.RentAsync(spec); -var result = await lease.InvokeAsync("CreateShipment", request); -var logs = await lease.InvokeAsync("GetLogs"); -``` - -Same command name, same JSON payload, same result type. What changes is underneath: no process -spawn, no JIT, no storage metadata check, and a gRPC channel that stays open across calls instead -of being dialled and discarded each time. - -**The session id is what makes the second form safe.** Traxis calls a command and then `GetLogs`, -and expects the command's upstream calls back — so both invocations have to land in the same -session. `lease.InvokeAsync` attaches the lease's session id; disposing the lease calls -`ResetAsync(sessionId)`, and the audit trail goes with it. Traxis's process-static `LogStore` -cannot do this: pool that process and one request's carrier calls appear in the next request's -audit record. - -## A footgun the classic sample encodes - -`Runner.Run(new Handler())` constructs the handler **before** `Runner` has parsed argv, so calling -`Runner.StartupValueOf(...)` from a constructor throws and the process dies before it can report -anything — the host only sees the stream close with "Received null data." Declare expectations in -the constructor; read values lazily, from the commands. `SW.Serverless.Samples.Classic` shows the -correct shape. - -## Tests - -```bash -dotnet test SW.Serverless.UnitTests/SW.Serverless.UnitTests.csproj +```sh +dotnet test ``` -The suite runs against a local-filesystem cloud store, so it needs no credentials and no cloud -account. The RabbitMQ tests additionally start a broker with Testcontainers; **without Docker -they report Inconclusive rather than failing**, and `SWSL_SKIP_BROKER_TESTS=1` skips them -deliberately — so `dotnet test` is safe to run anywhere. - -`RabbitAdapterTests` runs both broker adapters end to end: publish and confirm, publisher → -broker → consumer → host, `mandatory` returns, runtime prefetch and interval changes, purge, -staged test-connection, topology discovery, advertised commands, and heartbeat detail. The one -that matters most is `A_rejected_message_is_nacked_back_and_redelivered` — a host rejection -becomes `BasicNack(requeue: true)`, the message returns to the queue, and the redelivery carries -the same dedupe key so the host recognises it instead of persisting it twice. - -`LargeFileAdapterTests` proves the streaming claim rather than asserting it: a 24 MB file through -a 64 KB chunker must not grow the adapter's working set by 24 MB. Measured in the sample web, -512 MB streamed in 2048 chunks while resident memory held flat at 62–64 MB. -`AdapterHostingTests` covers the DI container, including a regression for the environment-variable -trap above. - -`CarrierAdapterTests` runs the carrier adapter against a real gRPC service on a loopback port: -DI-resolved upstream connection, the classic call shape, a rejection returned as a result rather -than thrown, retries on transient `Unavailable`, and the two that matter for pooling — -`GetLogs_after_a_command_sees_that_commands_calls` and -`One_leases_call_log_never_leaks_into_another`. - -`ResidentAdapterTests` covers installation from storage, typed command results and -typed failures, push/ack, and three things worth calling out: - -* **`A_timed_out_command_does_not_corrupt_the_next_call`** — the v1 regression. There a timed-out - command left the child running and its late reply resolved the *next* caller's completion, - which in Traxis reliably corrupted the `GetLogs` fetch issued right after it. -* **`Heartbeat_is_answered_while_a_command_is_running`** — proves the stream is multiplexed. If it - were not, a slow command would starve the heartbeat and the supervisor would restart a healthy - adapter. -* **`A_rejected_event_is_left_for_redelivery_and_then_deduplicated`** — ack ordering end to end. - -## What is deliberately not here yet - -The Kubernetes orchestrator and OTLP export. Both are called out in the design doc with the -phase they belong to. +The tests use a local-filesystem store, so they need no cloud account. Python and Node adapter +tests need `python3` and `node` on the `PATH`. The RabbitMQ tests start a broker with +Testcontainers and report Inconclusive without Docker; set `SWSL_SKIP_BROKER_TESTS=1` to skip them. diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..bbb33c7 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,413 @@ +# The CLI: sw-serverless + +`sw-serverless` creates, builds, tests, runs and publishes adapters in any supported language, and +manages published versions. + +- [Installing](#installing) +- [Commands at a glance](#commands-at-a-glance) +- [init](#init) · [build](#build) · [test](#test) · [run](#run) · [manifest validate](#manifest-validate) +- [publish](#publish) · [promote](#promote) · [versions](#versions) · [withdraw](#withdraw) +- [Publishing a .NET project directly (the original form)](#publishing-a-net-project-directly-the-original-form) +- [Storage](#storage) +- [How versions are chosen](#how-versions-are-chosen) +- [Exit codes](#exit-codes) + +## Installing + +Self-contained binaries are published on the +[GitHub releases](https://github.com/simplify9/SW-Serverless/releases) tagged `cli-v`: + +| Platform | Asset | +|---|---| +| Linux x64 | `sw-serverless-linux-x64.tar.gz` | +| Linux Arm64 | `sw-serverless-linux-arm64.tar.gz` | +| Linux x64, musl (Alpine) | `sw-serverless-linux-musl-x64.tar.gz` | +| macOS Intel | `sw-serverless-osx-x64.tar.gz` | +| macOS Apple silicon | `sw-serverless-osx-arm64.tar.gz` | +| Windows x64 | `sw-serverless-win-x64.zip` | + +Each release also has a `SHA256SUMS` file. On Linux and macOS the install script finds the newest +release, picks your platform's archive, checks it against `SHA256SUMS` and installs `sw-serverless` to +`~/.local/bin` (or `INSTALL_DIR`); `SW_SERVERLESS_VERSION` picks a version: + +```sh +curl -fsSL https://raw.githubusercontent.com/simplify9/SW-Serverless/main/scripts/install-cli.sh | sh +SW_SERVERLESS_VERSION=10.2.0 INSTALL_DIR=/usr/local/bin sh install-cli.sh +``` + +By hand, or on Windows: download your platform's archive and `SHA256SUMS` from the release, check +one against the other (`sha256sum -c` on Linux, `shasum -a 256 -c` on macOS, +`Get-FileHash` on Windows), unpack it, and put `sw-serverless` (`sw-serverless.exe`) on your `PATH`. + +To build it from source you need the .NET 10 SDK: + +```sh +git clone https://github.com/simplify9/SW-Serverless.git +cd SW-Serverless +dotnet run --project SW.Serverless.Installer -- build ../my-adapter # run any command without installing +dotnet publish SW.Serverless.Installer -c Release -o ./out # ./out/sw-serverless +``` + +What each command needs on the machine: + +| Command | Needs | +|---|---| +| `build`, `test`, `run` of a .NET adapter | The .NET 10 SDK (`dotnet`) | +| `build`, `test`, `run` of a Python adapter | `python3` 3.12 or later; `pip` if it has a `requirements.txt` | +| `build`, `test`, `run` of a JavaScript adapter | `node` 22 or later; `npm` if its `package.json` has dependencies | +| `build` of a TypeScript adapter | `node` 22.13 or later | +| `publish`, `promote`, `versions`, `withdraw` | Access to the storage | + +`sw-serverless --help` lists a command's options. + +## Commands at a glance + +| Command | What it does | +|---|---| +| `sw-serverless init ` | Writes a new adapter project. | +| `sw-serverless build [project]` | Builds a package: `bin/serverless/{id}-{version}.zip`. | +| `sw-serverless test [package]` | Runs the adapter as a host would and checks it: manifest, description, settings, contracts. | +| `sw-serverless run [package] --call ` | Starts the adapter, calls one command, prints the result. | +| `sw-serverless manifest validate [path]` | Checks an `adapter.json`. | +| `sw-serverless publish ` | Publishes a built package as a version. | +| `sw-serverless promote ` | Makes a published version the current one. | +| `sw-serverless versions ` | Lists published versions. | +| `sw-serverless withdraw ` | Marks a version as not to be used. | +| `sw-serverless ` | Builds and publishes a .NET project in one step (the original form). | + +**Options that take several values** (`--allow`, `--contract`) take them after one flag, separated +by spaces: `--contract a.json b.json`. Giving the flag twice is an error. Because such an option +takes every value that follows it, put positional arguments before it. + +## init + +```sh +sw-serverless init [--lang dotnet|python|node|typescript] [--id ] [--dir ] +``` + +Writes a new adapter in `/`: a greeter with two settings (`Greeting`, and a secret +`ApiKey`) and two commands (`Greet`, `Count`), an `adapter.json`, a `settings.example.json`, a +`.gitignore` and a `README.md`. + +| Option | Default | Meaning | +|---|---|---| +| `Name` | (required) | The project, folder and class name. Letters, digits and `_`, starting with a letter. | +| `--lang` | `dotnet` | `dotnet`, `python`, `node` (JavaScript) or `typescript`. | +| `--id` | from the name | The adapter id. `AcmeOrders` becomes `acme.orders`. | +| `--dir` | `.` | The folder to create the project in. The project folder must not exist or must be empty. | + +Files per language: + +| Language | Files | +|---|---| +| `dotnet` | `.csproj` (referencing `SimplyWorks.Serverless.Sdk` 10.1.0, with a NuGet lock file), `Program.cs` | +| `python` | `main.py`, `requirements.txt` | +| `node` | `main.js`, `package.json` | +| `typescript` | `main.ts`, `package.json` (`"type": "module"`), `tsconfig.json` | + +```sh +sw-serverless init AcmeOrders --lang python +# Made an adapter in /work/AcmeOrders: +# adapter.json +# main.py +# ... +``` + +## build + +```sh +sw-serverless build [project] [-o ] [--no-source] [--dry-run] [--allow ...] +``` + +Builds the adapter in `project` (default: the current folder), which must hold an `adapter.json` +with an `id`. The runtime in `adapter.json` decides how: + +- **.NET** (no `runtime`, or `"dotnet"`): `dotnet publish -c Release` of the one `.csproj`, + `.fsproj` or `.vbproj` in the folder. The adapter must use `SimplyWorks.Serverless.Sdk` 10.1.0 or + later. An adapter on an older SDK is published with the + [original form](#publishing-a-net-project-directly-the-original-form). +- **Python** (`"python"`): copies the adapter's files, copies the SDK under `_vendor/`, and installs + `requirements.txt` there with `pip`. +- **Node** (`"node"`): copies the adapter's files, turns TypeScript into JavaScript, installs + `package.json` dependencies with `npm`, and copies the SDK into `node_modules/`. + +Then it runs the adapter with `--describe`, writes the full `adapter.json` into the package, adds +the source under `source/`, and zips it. Details: [Packaging and storage](packaging-and-storage.md). + +| Option | Default | Meaning | +|---|---|---| +| `project` | `.` | The adapter's project folder. | +| `-o`, `--out` | `/bin/serverless` | Where the package folder (`package/`) and the zip go. | +| `--no-source` | off | Leave the source out of the package. | +| `--dry-run` | off | List the source files that would be carried, check them for secrets, and build nothing. | +| `--allow` | none | Source files (relative to the project) whose secret-scan finding is a false positive. | + +The zip is named `{id}-{version}.zip`, or `{id}.zip` when `adapter.json` has no version. + +The build prints the source files it carried and their sizes, and any warnings. It fails, before +building anything, if a source file looks like it holds a secret: + +``` +src/Client.cs:14 looks like a password in a connection string. Remove it, or pass --allow src/Client.cs if it isn't one +``` + +## test + +```sh +sw-serverless test [package] [--settings ] [--timeout ] [--allow-delete] [--contract ...] +``` + +Runs the conformance kit: starts the adapter the way a host does (from a temporary local store, +through the real host library) and checks it. `package` is a package zip, an unpacked package +folder, or a project folder, which is built first into a temporary folder. Default: the current +folder. + +| Option | Default | Meaning | +|---|---|---| +| `--settings` | none | A JSON file of settings: `{ "Name": "value" }`. The adapter really runs with them, so it calls whatever they point at. | +| `--timeout` | `60` | Seconds one call may take. | +| `--allow-delete` | off | Let methods a contract marks destructive run. Off by default because they change the real source the settings point at. | +| `--contract` | none | Contract files to check against. The CLI carries no contracts of its own. See [Contracts](contracts.md). | + +It prints one line per check, `PASS`, `FAIL` or `SKIP`, then `Conforms.` or the number of failed +checks: + +``` +PASS manifest +PASS describe — python SDK 10.2.0, 2 commands +PASS settings match the manifest — 2 settings +PASS starts — classic session +SKIP contracts — it declares no contract, so only what every adapter must do is checked +PASS an unknown command is refused — with an error, and it kept answering +Conforms. +``` + +What each check means is in [Contracts](contracts.md#what-the-conformance-kit-checks). The exit code +is 0 only if no check failed. + +## run + +```sh +sw-serverless run [package] --call [--input ] [--settings ] [--timeout ] +``` + +Starts the adapter as a host would, calls one command, prints its result, and stops it. `package` +is taken as for `test`. + +| Option | Default | Meaning | +|---|---|---| +| `--call` | (required) | The command name. | +| `--input` | none | The argument. Valid JSON is sent as JSON; anything else as text. `@file` reads it from a file. | +| `--settings` | none | A JSON file of settings. | +| `--timeout` | `60` | Seconds the call may take. | + +```sh +sw-serverless run --settings settings.json --call Greet --input Ada +# Hello, Ada! +sw-serverless run bin/serverless/acme.orders-1.2.0.zip --call Process --input @order.json +``` + +A string result is printed as it is; any other result is printed as JSON. If the call fails, the +error message is printed and the exit code is 1. + +## manifest validate + +```sh +sw-serverless manifest validate [path] +``` + +Checks an `adapter.json`, or the one in the folder `path` (default: the current folder), against +the [validation rules](manifest.md#validation). Prints each problem, then ` is valid.` or the +number of problems. + +## publish + +```sh +sw-serverless publish [-v ] [--no-promote] [--notes ] [--published-by ] [storage options] +``` + +Publishes a package made by `sw-serverless build`, in any language, as a version. + +| Option | Default | Meaning | +|---|---|---| +| `package` | (required) | The zip. | +| `-v`, `--version` | the manifest's `version` | An explicit version (`1.4.0`, `1.5.0-rc.1`) or `major`, `minor`, `patch` to bump the highest published release. See [How versions are chosen](#how-versions-are-chosen). | +| `--no-promote` | off | Publish the version without making it current. | +| `--notes` | none | Release notes (Markdown). Replaces `releaseNotes` in the manifest. | +| `--published-by` | `SWSL_PUBLISHED_BY`, then `GITHUB_ACTOR`, then your user name | Who published it, recorded in the catalog. | + +What it writes: + +- the package, with the version and publish time stamped into its manifest, to + `adapters-versions/{id}/{version}`; +- for a .NET adapter, unless `--no-promote`, the same package to `adapters/{id}`; +- the catalog entry `adapters-catalog/{id}.json`, with the new version, and as current unless + `--no-promote`. + +A Python, Node or `exec` package is never written to `adapters/{id}`: hosts from before manifests +would start whatever is there with `dotnet`. Newer hosts find its current version in the catalog. +For the same reason, an id once published as a .NET adapter cannot be published in another +runtime; use a new id. + +```sh +sw-serverless publish bin/serverless/acme.orders-1.2.0.zip -p s3 -b my-adapters -u https://s3.example.com +sw-serverless publish bin/serverless/acme.orders.zip -v minor --no-promote --notes "Retries on 503" +``` + +A package you assembled yourself, such as an [`exec`](packaging-and-storage.md#exec) adapter, can +be published too, as long as it has an `adapter.json` with a valid `id`. + +## promote + +```sh +sw-serverless promote [storage options] +``` + +Makes a published version the current one. Rolling back is promoting an older version; nothing is +rebuilt. + +It downloads the version, checks its SHA-256 against the one recorded when it was published +(refusing if they differ), and points the catalog at it. For a .NET adapter it also copies the +package over `adapters/{id}`, with the storage metadata older hosts read. A withdrawn or unknown +version is refused. + +Hosts pick up the change once their cached lookup expires (`AdapterMetadataCacheDuration`, 5 +minutes by default). + +## versions + +```sh +sw-serverless versions [storage options] +``` + +Lists the published versions, newest first. `*` marks the current one. + +``` +acme.orders: current 1.2.0 + VERSION PUBLISHED (UTC) BY SHA256 +* 1.2.0 2026-10-02 08:30 ci-bot 3f2a9c1b7d4e + 1.1.0 2026-09-20 14:05 ada 91c0e4a2b8f3 withdrawn + 1.0.0 2026-09-01 10:00 ada 0b7d14e6c2a9 +``` + +For an adapter published before the catalog existed, it reads the packages and their metadata +instead, and says so. + +## withdraw + +```sh +sw-serverless withdraw [storage options] +``` + +Marks a version as withdrawn in the catalog. It stays listed, `promote` refuses it, and +applications listing versions should not offer it. The package is not deleted, and a host asked +for that exact version still runs it. The current version cannot be withdrawn; promote another one +first. + +## Publishing a .NET project directly (the original form) + +```sh +sw-serverless [storage options] [-v ] [-k ] [--no-promote] [--notes ] + [--published-by ] [--no-probe] +``` + +The form the tool has always had, kept unchanged. It builds a .NET project with +`dotnet publish -c Release`, writes the manifest, zips and uploads it, in one step. It works with +every SDK version, including those before `--describe`. + +| Option | Meaning | +|---|---| +| `project` | The `.csproj` file. | +| `adapter-id` | The id to publish under. Uppercase is lowered. | +| `-v`, `--version` | An explicit version or `major`, `minor`, `patch`. **Without `-v` the upload is unversioned:** only `adapters/{id}` is replaced, and the catalog's current version is cleared. | +| `--no-promote` | With `-v` only: publish the version without making it current. | +| `-k`, `--kind` | Kinds, comma separated (`processor,source`). Replaces the kinds from `adapter.json` and `[AdapterKind]`. | +| `--notes`, `--published-by` | As for [publish](#publish). | +| `--no-probe` | Do not start the built adapter to ask for its settings. | + +The manifest is made from the `adapter.json` beside the project file, if there is one, and what +the tool reads from the built assembly: the entry assembly, the lifecycle (classic or resident), +the kinds, and the SDK version. If `adapter.json` lists no `properties`, the tool starts a classic +adapter and asks it for the settings it declares, unless `--no-probe`. If that fails, the publish +continues with a warning and no properties. + +```sh +sw-serverless -p s3 -b my-adapters -u https://s3.example.com -v patch ./Acme/Acme.csproj acme.orders +``` + +## Storage + +`publish`, `promote`, `versions`, `withdraw` and the original form all need to know where adapters +are stored. Each setting is taken from the first of: the command-line flag, the config file given +with `-c`, the environment variable. + +| Flag | Environment variable | Meaning | +|---|---|---| +| `-p`, `--provider` | `SWSL_PROVIDER` | `s3` (S3 or S3-compatible; the default), `as` (Azure Blob Storage), `gc` (Google Cloud Storage), `oc` (Oracle Object Storage), `local` (a folder) | +| `-b`, `--bucketname` | `SWSL_BUCKET` | Bucket or container name | +| `-a`, `--accesskey` | `SWSL_ACCESS_KEY` | Access key | +| `-s`, `--secret` | `SWSL_SECRET_KEY` | Secret key | +| `-u`, `--url` | `SWSL_SERVICE_URL` | Service URL. For `local`, the folder the store is kept in. | +| (config file only) | `SWSL_REGION` | Region | +| `-c`, `--cloudfilesconfigpath` | | A JSON config file (below) | +| `--published-by` | `SWSL_PUBLISHED_BY` | Who is publishing (then `GITHUB_ACTOR`, then the user name) | + +Google Cloud reads a service account from `SWSL_GC_PROJECT_ID`, `SWSL_GC_PRIVATE_KEY_ID`, +`SWSL_GC_PRIVATE_KEY`, `SWSL_GC_CLIENT_EMAIL`, `SWSL_GC_CLIENT_ID` and +`SWSL_GC_CLIENT_X509_CERT_URL`, or from the config file. `SWSL_GC_PRIVATE_KEY` may write its line +breaks as a literal `\n`, as they appear in a service-account JSON file. Oracle's settings +(`TenantId`, `UserId`, `FingerPrint`, `RSAKey`, `NamespaceName`) come only from the config file. + +The config file holds the same settings under `CloudFiles`: + +```json +{ + "CloudFiles": { + "Provider": "gc", + "BucketName": "my-adapters", + "ProjectId": "my-project", + "PrivateKeyId": "…", + "PrivateKey": "-----BEGIN PRIVATE KEY-----\n…\n-----END PRIVATE KEY-----\n", + "ClientEmail": "publisher@my-project.iam.gserviceaccount.com", + "ClientId": "…" + } +} +``` + +Its other keys are `AccessKeyId`, `SecretAccessKey`, `ServiceUrl`, `Region`, `TenantId`, `UserId`, +`FingerPrint`, `RSAKey`, `NamespaceName` and `ClientX509CertUrl`. + +In CI, keep keys in secrets and pass them as environment variables rather than flags, which end up +in shell history and process listings: + +```sh +export SWSL_PROVIDER=s3 SWSL_BUCKET=my-adapters SWSL_SERVICE_URL=https://s3.example.com +export SWSL_ACCESS_KEY=… SWSL_SECRET_KEY=… +sw-serverless publish bin/serverless/acme.orders-1.2.0.zip +``` + +For development, `-p local` keeps the store in a folder. A host reads it with +`AddLocalTestsCloudFiles` and the same bucket name and folder (see +[Hosting adapters](hosting.md#storage)): + +```sh +sw-serverless publish bin/serverless/greeter-0.1.0.zip -p local -b adapters-dev -u /tmp/swsl-store +``` + +The CLI always publishes under the root folder `adapters`. To publish under another root, use +`SW.Serverless.Tooling` from your own tool (see [Extending the tools](extending.md)). + +## How versions are chosen + +- An explicit version must be a semantic version (`1.4.0`, or a pre-release such as `1.5.0-rc.1`), + must not exist already, and its `major.minor.patch` must be higher than every published release. +- `major`, `minor` and `patch` bump the highest published release: `1.4.2` becomes `2.0.0`, + `1.5.0` or `1.4.3`. With no releases yet, the result is `1.0.0`. Pre-releases are not counted. +- A published version is never replaced. + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | The command did what it was asked: the build succeeded, every check passed, the publish completed. Also for `--help`. | +| `1` | Anything else: a bad command line, an invalid id or manifest, a failed build, a failed conformance check, a failed call, a storage error. The output says what went wrong. | diff --git a/docs/compatibility.md b/docs/compatibility.md new file mode 100644 index 0000000..671d19a --- /dev/null +++ b/docs/compatibility.md @@ -0,0 +1,141 @@ +# Compatibility + +SW-Serverless runs in deployments where hosts, adapters and publishing tools are upgraded at +different times. This page says what keeps working across versions, and how the protocol and +file formats are versioned so that it stays that way. + +- [Versions of the parts](#versions-of-the-parts) +- [Current hosts and older adapters](#current-hosts-and-older-adapters) +- [Older hosts and current packages](#older-hosts-and-current-packages) +- [The protocol](#the-protocol) +- [Manifests, catalog entries and descriptions](#manifests-catalog-entries-and-descriptions) +- [Storage](#storage) +- [The CLI](#the-cli) +- [Requiring a newer host or application](#requiring-a-newer-host-or-application) +- [How this is tested](#how-this-is-tested) + +## Versions of the parts + +| Part | Version line | Notes | +|---|---|---| +| Host, `SimplyWorks.Serverless` | 10.x | `HostInfo.Version` is the host's version, compared with an adapter's `minHostVersion`. | +| .NET SDK, `SimplyWorks.Serverless.Sdk` | 10.x | 10.1.0 added `--describe` (needed by `sw-serverless build`) and reading settings from standard input. | +| Python SDK `sw-serverless`, Node SDK `@simplyworks/sw-serverless` | 10.2.0 | Copied into each package by `sw-serverless build`. | +| Protocol | 1 and 2 | 1 is the .NET classic text protocol; 2 is gRPC. Hosts speak 2 to 2. | +| Manifest (`adapter.json`) | `manifestVersion` 1 | | +| Catalog entry | `catalogVersion` 1 | | +| Describe output | `describeVersion` 1 | | + +## Current hosts and older adapters + +A current host runs: + +- **packages without `adapter.json`**, published before manifests: as .NET adapters, from their + storage metadata, as before; +- **.NET classic adapters on SDKs before 10.1.0**: they receive their settings as command-line + arguments, as they expect. Adapters on 10.1.0 or later (known from the manifest's `sdkVersion`) + receive them on standard input instead; +- **.NET resident adapters on SDK 10.0.x**; +- **versions published by older tools** under `adapters/{id}/{version}`, when asked for + `{id}/{version}`; +- **packages carrying `source/`**, which it does not unpack, and packages from before `source/` + existed that happen to have a folder named `source`, which it unpacks as before (only a folder the + manifest declares as source is skipped). + +## Older hosts and current packages + +Hosts on `SimplyWorks.Serverless` 10.0.x, and applications that list adapters by reading the keys +under `adapters/`, do not know manifests, versions or the catalog. With them: + +- `adapters/{id}` always holds the current .NET package, with the metadata they need + (`EntryAssembly`, `Hash`), after every publish, promote or rollback. They run it. +- Nothing else is ever written under `adapters/`, so their listings see exactly the adapters that + exist. +- A package's `adapter.json` is just another file to them, and `source/` just another folder. +- A .NET adapter built on the current SDK still reads settings from command-line arguments when an + older host passes them that way. +- Python, Node and `exec` adapters are never written to `adapters/{id}`, so older hosts never see + them, rather than try to start them with `dotnet`. For the same reason such adapters must be + published with a version, and an id once published as a .NET adapter cannot move to another + runtime. +- They cannot pin a version published in the current layout: asked for `{id}/{version}`, they look + only at `adapters/{id}/{version}`. Current hosts look in both places. + +## The protocol + +Protocol 2 is designed to grow without breaking deployed adapters or hosts: + +- **Version negotiation.** The host offers its newest version in the handshake. The adapter answers + in Hello with the lower of that and its own newest. The host accepts any version in its range + (`ProtocolVersions.Min` to `Max`, both 2 today). A new protocol version raises the host's `Max` + and keeps `Min`, so adapters already deployed keep working. The SDKs refuse only a host older + than protocol 2. +- **New fields are optional.** Fields added to the messages (Hello's `sdk_language`, `settings`, + `kinds` and `contracts`; `CommandInfo`'s schemas; Invoke's `properties`) are ignored by older + hosts, and left empty by older adapters, which hosts handle. +- **New behaviour is announced.** The host sends `Cancel` only to adapters that list the `cancel` + capability in Hello, so an adapter built before it never receives a frame it does not know. + +Protocol 1 is frozen. The CLI and the host treat its markers (`#!#`, `{{null}}`, `{{newline}}`, +`{{error}}`, `{{expected}}`, `{{quit}}`) as fixed by every adapter already published. + +## Manifests, catalog entries and descriptions + +- Every field is optional when read. A missing manifest, or one from an older tool, means what it + always meant. +- Fields a reader does not know are kept and written back unchanged, at every level, in manifests, + catalog entries and `--describe` output. A manifest written by a newer tool survives a round trip + through an older one. +- `manifestVersion` is never lowered by a tool that rewrites a manifest. +- `compatibility.MinVersionOf(application)` also reads the older `minVersion` form. + +## Storage + +The layout is held to three rules (see [Packaging and storage](packaging-and-storage.md)): + +1. `adapters/{id}` always holds the current .NET package with the metadata an older host needs. +2. Nothing but `adapters/{id}` is written under `adapters/`. Versions older tools put at + `adapters/{id}/{version}` are read, never written. +3. Everything new lives beside it: `adapters-versions/` and `adapters-catalog/`. + +An adapter published only by older tools has no catalog entry. `versions` then reads its packages +instead, and its next `publish` or `promote` creates the entry from them. + +## The CLI + +- The original form, `sw-serverless [options] `, works as it always + has. Without `-v` it makes the same unversioned upload to `adapters/{id}`. +- Command words (`init`, `build`, `publish`, ...) are recognised only as the first argument, and + only when no file of that name exists, so a project file that happens to be named like a command + is still published. + +## Requiring a newer host or application + +When an adapter relies on something a host gained in a given release, say so in its +`adapter.json`: + +```json +{ "compatibility": { "minHostVersion": "10.1.0" } } +``` + +An older host refuses to install it, with a message naming both versions, instead of failing in a +less obvious way. Hosts before `minHostVersion` existed do not check it. + +`compatibility.applications` does the same for an application's own versions. Hosts ignore it; +the application checks its own entry with `MinVersionOf`. See +[The manifest](manifest.md#compatibility). + +## How this is tested + +`SW.Serverless.CompatibilityTests` runs these promises against released packages: + +- a host built on the released `SimplyWorks.Serverless` 10.0.0 installs and runs what the current + CLI publishes, follows promote and rollback, runs unversioned publishes, and still runs versions + in the old layout; +- the current host runs packages without manifests, versions in the old layout, and adapters built + on the released `SimplyWorks.Serverless.Sdk` 10.0.0, classic and resident; +- the old listing rule sees exactly the adapters that exist after versions, promote and withdraw; +- manifests and catalog entries with unknown fields round-trip through the current and the released + 10.0.2 parser; +- Python, Node and `exec` packages stay out of older hosts' sight, must be versioned, and cannot + take over a .NET adapter's id. diff --git a/docs/concepts.md b/docs/concepts.md new file mode 100644 index 0000000..9c3c9a1 --- /dev/null +++ b/docs/concepts.md @@ -0,0 +1,200 @@ +# Concepts + +This page explains the parts of SW-Serverless and the words the other pages use. It has no setup +steps; those are in [Hosting adapters](hosting.md) and [Writing adapters](writing-adapters.md). + +## Host and adapter + +A **host** is your application, with the `SimplyWorks.Serverless` library added. It knows how to +find adapters in storage, install them on its machine, start them, call them and stop them. + +An **adapter** is a small program that the host runs as a **separate child process**. It exposes +**commands**: named operations that take at most one argument and return at most one result. The +host calls a command by name, for example `Greet` with the argument `"Ada"`, and gets back +`"Hello, Ada!"`. + +Because an adapter is its own process: + +- a crash, a hang or a memory leak in it does not take the host down; +- it can be written in a different language from the host; +- a new version can be installed while the host keeps running. + +An adapter talks only to its host. It reads its configuration from the host, answers the host's +calls, and, if it is resident, hands events to the host. It does not need to know about the host's +database or internal services. + +## Adapter id + +Every adapter has an **id**, such as `acme.orders` or `greeter`. The id is how the host asks for +it and where it is stored. An id uses lowercase letters, digits, `.`, `_` and `-`, and starts with +a letter or digit. Uppercase is lowered when you publish. + +## Package + +A **package** is a zip file holding everything an adapter needs to run: + +- the adapter's runnable files: a .NET publish output, Python or JavaScript files, or a binary; +- its dependencies (for Python and Node, copied into the package; see + [Packaging and storage](packaging-and-storage.md)); +- an `adapter.json` [manifest](#manifest) at the root; +- optionally, its **source**, under `source/` (see [Source in packages](#source-in-packages)). + +`sw-serverless build` makes packages. A package runs on any host that has its runtime: nothing is +fetched when it is installed. + +## Manifest + +The **manifest** is the `adapter.json` file at the root of every package. It says how to run the +adapter (runtime, entry file, lifecycle, protocol), what it needs configured (its settings, called +properties in the manifest), what it is for (kinds and contracts) and how to present it (name, +summary, icon, publisher, tags). + +You write a short `adapter.json` beside your project: an id, a version, a display name and so on. +`sw-serverless build` asks the built adapter to [describe itself](#describe) and writes the full +manifest into the package from both. Field reference: [The manifest](manifest.md). + +## Storage, catalog and versions + +Adapters are published to **cloud storage**: S3 or an S3-compatible store, Azure Blob Storage, +Google Cloud Storage, Oracle Object Storage, or a local folder for development. The host reads the +same storage. Inside it, everything lives under one root folder, `adapters` by default: + +| Key | What it holds | +|---|---| +| `adapters/{id}` | The **current** package of a .NET adapter: the one that runs when no version is asked for. Older hosts read only this. | +| `adapters-versions/{id}/{version}` | One package per published **version**. Never overwritten. | +| `adapters-catalog/{id}.json` | The **catalog entry**: which version is current, every version with its manifest, digest, date and publisher, and the icon. | + +A **version** is a semantic version such as `1.4.0` or `1.5.0-rc.1`. Publishing a version never +replaces an existing one. **Promoting** a version makes it current; rolling back is promoting an +older one. **Withdrawing** a version marks it as not to be used, without deleting it. + +The **catalog** lets an application list adapters, with names, icons and settings, without opening +a single package. `AdapterCatalogStore` reads it. + +A host asked for a plain id, such as `greeter`, runs the current version. Asked for +`greeter/1.4.0`, it runs that version. This is **pinning**. + +Details: [Packaging and storage](packaging-and-storage.md). + +## Lifecycles: classic and resident + +An adapter has one of two **lifecycles**. + +A **classic** adapter runs for one **session**. The host starts a process, calls one or more +commands, one at a time, and then stops the process. Use classic for request-and-reply work: each +use gets a fresh process with the settings for that use. In .NET a classic adapter calls +`Runner.Run`. + +A **resident** adapter is started once and runs until it is stopped. It can keep connections open +(to a broker, a database, a partner's API), push **events** to the host, keep a small piece of +**state** with the host (such as a cursor), and report its **status** on every heartbeat. The host +supervises it: it restarts it if it crashes or stops answering, and stops restarting it +(**quarantines** it) after repeated crashes. In .NET a resident adapter calls `Runner.RunResident`; +in Python and Node an adapter class with a `start` method is resident. + +The host can run resident instances in two ways: + +- **exclusive**: exactly one process for a given key, for example one per broker connection; +- **pooled**: a few warm processes shared by many short uses, each use **renting** one and + returning it. When it is returned, the adapter is told to forget that use (a **reset**). + +## Protocols + +The host and an adapter talk over one of two **protocols**. + +**Protocol 1** is the original **classic text protocol**, used only by .NET adapters that call +`Runner.Run`. Commands and results are single lines on the process's standard input and output. + +**Protocol 2** is **gRPC over a local socket**. The host writes a one-line JSON **handshake** on +the adapter's standard input: a socket path (a Unix domain socket on Linux and macOS, a named pipe +on Windows) and a one-time token. The adapter connects to that socket and opens one gRPC stream, +over which every command, result, event, log line and heartbeat travels. Nothing listens on a +network port. + +Which protocol is used: + +| Adapter | Protocol | +|---|---| +| .NET, `Runner.Run` (classic) | 1 | +| .NET, `Runner.RunResident` (resident) | 2 | +| .NET, `Runner.RunResident` with `"lifecycle": "classic"` in its `adapter.json` | 2, run as a classic session | +| Python, Node, `exec` | 2, classic or resident | + +Classic adapters on protocol 2 are still called with `IServerlessService`, exactly like protocol 1 +ones; the host runs them on the resident machinery underneath. Details: [The protocol](protocol.md). + +## Runtimes + +The manifest's `runtime` says how the host starts the adapter. The host maps each name to a +launcher it controls; a package cannot name an arbitrary program. + +| Runtime | Started as | Needs on the host | +|---|---|---| +| `dotnet` (the default) | `dotnet .dll` | the .NET runtime the adapter targets | +| `python` | `python3 -u ` | Python 3.12 or later, unless the manifest asks otherwise | +| `node` | `node ` | Node 22 or later, unless the manifest asks otherwise | +| `exec` | the entry file itself | nothing: the package carries a self-contained binary | + +The host checks that a Python or Node runtime is present, and at a version the manifest accepts, +before starting anything. It also refuses a package built for other platforms (`linux-x64`, +`osx-arm64` and so on) than its own. Where to find `python3`, `node` and `dotnet` is configurable; +see [Hosting adapters](hosting.md#adapterruntimeoptions-addadapterruntimes). + +## Settings + +An adapter declares the **settings** it reads: their names, whether each is required, whether it is +secret, a default and a description. In .NET this is `Runner.Expect`; in Python `sw.expect`; in +Node `expect`. The old .NET name for settings is **startup values**, and the manifest calls them +**properties**. + +The host passes the values when it starts the adapter. Commands read them with +`Runner.StartupValueOf`, `sw.value_of` or `valueOf`. + +Declaring settings in code means the adapter itself is the one place that says what it needs. +`sw-serverless build` writes the declarations into the manifest, so an application can show a form +for an adapter's settings without starting it. The conformance kit checks that the two agree. + +A resident instance can also receive **per-call values** with each command. A command reads them +through the same functions; a per-call value wins over a startup value of the same name. + +The host always adds a `CorrelationId` value for classic sessions, from the `correlationId` it was +given. + +## Kinds and contracts + +A **kind** is a label for what an adapter is for, such as `processor` or `source`. SW-Serverless +does not interpret kinds; the application decides what they mean. + +A **contract** is a document an application writes to say what its adapters must do: for each kind, +the commands it calls, the shape of their arguments and results (as JSON Schema), example inputs, +which kinds are called as an ordered session, and which commands are destructive. An adapter +declares the contracts it implements, with a version (`orders` version 1), and the kinds it +implements. + +SW-Serverless ships no contract of its own. The conformance kit (`sw-serverless test`) checks an +adapter against any contract it is given. See [Contracts](contracts.md). + +## Describe + +Every SDK answers the `--describe` command-line flag: started with it, the adapter prints one JSON +document with its SDK, lifecycle, protocol, settings, commands (with JSON Schemas of their +arguments and results), kinds and contracts, and exits. The tools use this to learn about an +adapter in any language without reading its code. The format is in +[The protocol](protocol.md#describe). + +## Source in packages + +By default `sw-serverless build` puts the adapter's source in the package under `source/`, and +lists every source file with its SHA-256 in the manifest. This lets anyone holding a published +version read, compare, audit or rebuild it. The build leaves out build output, dependencies, +editor folders and files that usually hold secrets, follows `.gitignore` and `.serverlessignore`, +and refuses to build if a source file looks like it contains a secret. + +Hosts do not unpack `source/` when they install a package. `--no-source` leaves the source out. + +## Compatibility + +Hosts and SDKs of different ages work together. Packages without a manifest, hosts from before +versions and the catalog, and adapters built on older SDKs all keep working. See +[Compatibility](compatibility.md). diff --git a/docs/contracts.md b/docs/contracts.md new file mode 100644 index 0000000..eb60a68 --- /dev/null +++ b/docs/contracts.md @@ -0,0 +1,330 @@ +# Contracts + +A **contract** is a JSON document in which an application says what its adapters must do: the +kinds of adapter it has, the commands it calls on each kind, the shape of their arguments and +results, and examples to call them with. Adapters declare which contracts and kinds they implement. +The conformance kit (`sw-serverless test`) then checks an adapter against the contract, by running +it. + +SW-Serverless ships no contract. Each application writes its own. This page uses the sample +**orders** contract from the test suite +(`SW.Serverless.Installer.UnitTests/Contracts/`) as the worked example. + +- [The contract document](#the-contract-document) +- [The orders example](#the-orders-example) +- [Declaring a contract in an adapter](#declaring-a-contract-in-an-adapter) +- [What the conformance kit checks](#what-the-conformance-kit-checks) +- [Running the checks](#running-the-checks) +- [Shipping your contract with your own tool](#shipping-your-contract-with-your-own-tool) +- [Versioning a contract](#versioning-a-contract) + +## The contract document + +| Field | Meaning | +|---|---| +| `contract` | The contract's name, e.g. `orders`. Matched without regard to case. | +| `version` | An integer, 1 or more. An adapter declares the name and the version it implements. | +| `description` | Free text. | +| `encoding` | Free text explaining the encoding rules to adapter authors. Not read by the kit. | +| `types` | The payload types, by name. | +| `types..schema` | A JSON Schema for the type: either inline, or the name of a schema file beside the contract. | +| `types..encoding` | `"string"` when the type travels as raw text rather than JSON. | +| `types..description` | Free text. | +| `kinds` | The kinds, by name. | +| `kinds..description` | Free text. | +| `kinds..session` | Present (as text describing the order) when the host calls this kind's methods as one ordered session. | +| `kinds..methods` | The commands the host calls on this kind, in order. | +| `methods[].name` | The command name. Case-sensitive. | +| `methods[].input` | The argument's type name, or `null` for no argument. | +| `methods[].output` | The result's type name, or `null` for no result. | +| `methods[].examples` | Example arguments, as JSON. Strings for a `"string"` type. | +| `methods[].destructive` | `true` when calling it changes the outside world in a way a test must not do by default (deleting a file at the source, say). | + +Payloads follow the [encoding rules](writing-adapters.md#encoding): a string type is raw text, +anything else is JSON with property names exactly as the schema gives them. + +## The orders example + +An order-processing application has two kinds of adapter: a **processor** takes an order and says +whether it was accepted; a **source** is where orders come from, read in one session per run. + +`orders-adapter-contract.v1.json`: + +```json +{ + "contract": "orders", + "version": 1, + "description": "What an order-processing application calls on its adapters.", + "encoding": { + "string": "A string argument or result is the raw UTF-8 text, not a JSON string.", + "object": "Any other argument or result is JSON, with property names exactly as its schema gives them.", + "none": "A method with no argument receives an empty payload; one with no result returns an empty payload." + }, + "types": { + "Order": { "schema": "order.schema.json" }, + "Receipt": { "schema": "receipt.schema.json" }, + "OrderId": { "encoding": "string", "description": "An id a source gave in List, passed back exactly as given." }, + "OrderIdList": { "schema": { "type": "array", "items": { "type": "string" } } } + }, + "kinds": { + "processor": { + "description": "Takes an order and says whether it was accepted.", + "methods": [ + { + "name": "Process", + "input": "Order", + "output": "Receipt", + "examples": [ + { "OrderId": "SO-1001", "Lines": 2 } + ] + } + ] + }, + "source": { + "description": "Where orders come from, read in one session per run.", + "session": "Open, List, then for each order Fetch and — once it is safely taken — Remove, and finally Close, which is also called after a failure.", + "methods": [ + { "name": "Open", "input": null, "output": null }, + { "name": "List", "input": null, "output": "OrderIdList" }, + { "name": "Fetch", "input": "OrderId", "output": "Order" }, + { "name": "Remove", "input": "OrderId", "output": null, "destructive": true }, + { "name": "Close", "input": null, "output": null } + ] + } + } +} +``` + +`order.schema.json`, beside it: + +```json +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Order", + "type": "object", + "required": ["OrderId"], + "properties": { + "OrderId": { "type": "string" }, + "Lines": { "type": "integer" } + }, + "additionalProperties": true +} +``` + +`receipt.schema.json`: + +```json +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Receipt", + "type": "object", + "required": ["Accepted"], + "properties": { + "Accepted": { "type": "boolean" }, + "Reference": { "type": ["string", "null"] } + }, + "additionalProperties": true +} +``` + +## Declaring a contract in an adapter + +An adapter declares the contract name and version, and the kinds it implements. These reach the +manifest through `--describe` when the adapter is built; you can also list them in `adapter.json` +under `contracts` and `kinds`. + +A processor for the orders contract: + +**.NET** + +```csharp +using SW.Serverless.Sdk; + +public class Order +{ + public string OrderId { get; set; } + public int Lines { get; set; } +} + +public class Receipt +{ + public bool Accepted { get; set; } + public string Reference { get; set; } +} + +[AdapterKind("processor")] +[AdapterContract("orders", 1)] +public class Processor +{ + public Processor() => Runner.Expect("Partner", description: "Who receives the orders."); + + public Task Process(Order order) => + Task.FromResult(order.Lines == 0 + ? new Receipt { Accepted = false } + : new Receipt { Accepted = true, Reference = $"{Runner.StartupValueOf("Partner")}:{order.OrderId}" }); +} + +static class Program +{ + static Task Main() => Runner.Run(new Processor()); +} +``` + +**Python** + +```python +import sw_serverless as sw + + +@sw.implements("orders", 1, "processor") +class Processor: + def __init__(self): + sw.expect("Partner", description="Who receives the orders.") + + @sw.command("Process", description="Takes an order and says whether it was accepted.") + def process(self, order: dict) -> dict: + if not order.get("Lines"): + return {"Accepted": False, "Reference": None} + return {"Accepted": True, "Reference": f"{sw.value_of('Partner')}:{order['OrderId']}"} + + +if __name__ == "__main__": + sw.run(Processor) +``` + +**Node** + +```js +const sw = require("@simplyworks/sw-serverless"); + +class Processor { + static kinds = ["processor"]; + static contracts = { orders: 1 }; + static commands = { + Process: { method: "process", input: "json", output: "json", description: "Takes an order and says whether it was accepted." }, + }; + + constructor() { + sw.expect("Partner", { description: "Who receives the orders." }); + } + + process(order) { + return order.Lines + ? { Accepted: true, Reference: `${sw.valueOf("Partner")}:${order.OrderId}` } + : { Accepted: false, Reference: null }; + } +} + +sw.run(Processor); +``` + +Command names must match the contract exactly, including case. + +An application can make this easier for its adapter authors by publishing base classes that carry +the declarations and the method names, in each language. In Python, `@sw.implements` on a base +class is inherited; in Node, `static kinds`, `contracts` and `commands` are merged down the class +chain; in .NET, `[AdapterKind]` and `[AdapterContract]` are inherited. + +## What the conformance kit checks + +`sw-serverless test` (and `ConformanceRunner` in `SW.Serverless.Tooling`) runs these checks, in +this order. Each is reported as `PASS`, `FAIL` or `SKIP`, and the run passes when nothing failed. + +| Check | Passes when | +|---|---| +| `manifest` | The package has an `adapter.json` that can be read and passes [validation](manifest.md#validation). | +| `entry` | The manifest's entry for this platform exists in the package. Reported only on failure; nothing else runs after it fails. | +| `describe` | The adapter answers `--describe` with a description that names its SDK language. | +| `settings match the manifest` | Every setting the adapter declares is in the manifest's `properties` and the reverse, with the same `required`, `secret` and (for non-secrets) `default`. | +| `starts` | It installs and starts the way a host does (from a temporary store, through the real host library), as a classic session or a resident instance. Nothing else runs after it fails. | +| `contracts` | Skipped when it declares no contract. For each contract it declares (in the manifest or the description): the contract is known (given with `--contract` or registered), and the adapter implements at least one of its kinds. | +| ` : methods` | Every method of the kind is one of the adapter's commands (case-sensitive). | +| ` : answers example ` | For a kind without `session`: each example is sent; the call succeeds; if the method has an output with a schema, the answer is JSON valid against it. Skipped for a method with no examples. | +| ` : ` | For a kind with `session`: the methods are called once each, in the contract's order. A method without input is called with none. A method with input is given the first id from the first list a previous method returned; skipped if there was none. A `destructive` method is skipped unless `--allow-delete`. Outputs are checked against their schemas. | +| `an unknown command is refused` | Calling a command the adapter does not have fails with an error, and the adapter still answers afterwards. | + +The kit really runs the adapter with the settings you give it, so it calls whatever they point at. +For a source, point it at test data. + +## Running the checks + +```sh +sw-serverless test ./OrdersProcessor --settings settings.json --contract contracts/orders-adapter-contract.v1.json +``` + +Schema files named in the contract are read from the contract file's folder. Several contracts go +after one flag: `--contract a.json b.json`. + +For the orders processor above: + +``` +PASS manifest +PASS describe — python SDK 10.2.0, 1 commands +PASS settings match the manifest — 1 settings +PASS starts — classic session +PASS orders processor: methods — Process +PASS orders processor: Process answers example 1 — a valid Receipt +PASS an unknown command is refused — with an error, and it kept answering +Conforms. +``` + +If the adapter declares a contract the kit was not given: + +``` +FAIL contract orders v1 — the kit doesn't know this contract; pass it with --contract +``` + +From code: + +```csharp +using SW.Serverless.Tooling.Conformance; + +var report = await new ConformanceRunner().RunAsync(new ConformanceOptions +{ + PackageDirectory = "/build/acme.orders/package", // an unpacked package + Settings = new Dictionary { ["Partner"] = "acme" }, + Contracts = { ContractDocument.FromFile("contracts/orders-adapter-contract.v1.json") }, + CommandTimeoutSeconds = 30, + Log = Console.WriteLine, +}); + +foreach (var check in report.Checks) + Console.WriteLine($"{check.Outcome} {check.Name} {check.Detail}"); +return report.Passed ? 0 : 1; +``` + +## Shipping your contract with your own tool + +The `sw-serverless` CLI knows no contracts, so adapter authors pass yours with `--contract`. An +application that ships its own tool built on `SW.Serverless.Tooling` can instead register its +contract once, so every adapter declaring it is checked without being handed the file: + +```csharp +using System.Reflection; +using SW.Serverless.Tooling.Conformance; + +static string Embedded(string name) +{ + using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream($"OrdersTool.Contracts.{name}")!; + return new StreamReader(stream).ReadToEnd(); +} + +ContractDocument.Register(ContractDocument.FromJson( + Embedded("orders-adapter-contract.v1.json"), + readSibling: Embedded)); // how to read schema files the contract names +``` + +`ContractDocument.FromFile(path)` reads a contract from disk with its schemas beside it. +`ContractDocument.FromJson(json, readSibling)` reads one from a string, with `readSibling` reading +the schema files it names. `Register` replaces an earlier registration of the same name and +version. A contract passed in `ConformanceOptions.Contracts` is used before a registered one. + +See [Extending the tools](extending.md) for building such a tool. + +## Versioning a contract + +The version is a whole number. When a change would break existing adapters (a renamed method, a +new required property in a result), publish the contract under the next version and keep the old +one: adapters keep declaring the version they implement, and your application can support both +for as long as it needs. Adding an optional property or a new kind usually does not need a new +version. diff --git a/docs/extending.md b/docs/extending.md new file mode 100644 index 0000000..de6b210 --- /dev/null +++ b/docs/extending.md @@ -0,0 +1,369 @@ +# Extending the tools + +Everything `sw-serverless` does is in the library `SimplyWorks.Serverless.Tooling`. An application +with adapters of its own can build its own command-line tool or server on it, so its adapter +authors get the application's templates, base classes and contract without extra steps, while the +build, the conformance kit and the storage layout stay exactly those of `sw-serverless`. + +This page shows the pieces with an example: an "orders" application shipping an `orders-adapters` +tool. + +- [Referencing the library](#referencing-the-library) +- [Templates: Scaffolder](#templates-scaffolder) +- [Your own packages: BuildRequest.Packages](#your-own-packages-buildrequestpackages) +- [Your contract: ContractDocument.Register](#your-contract-contractdocumentregister) +- [Running and testing: LocalAdapterHost and ConformanceRunner](#running-and-testing-localadapterhost-and-conformancerunner) +- [Publishing: PackagePublisher and AdapterRepository](#publishing-packagepublisher-and-adapterrepository) +- [Putting it together](#putting-it-together) + +## Referencing the library + +```sh +dotnet add package SimplyWorks.Serverless.Tooling +``` + +It brings `SimplyWorks.Serverless` (the host, which the conformance kit runs adapters on) and +`SimplyWorks.Serverless.Contract` (the manifest and catalog models), and the storage providers. + +| Namespace | Types | +|---|---| +| `SW.Serverless.Tooling.Scaffolding` | `Scaffolder`, `ScaffoldRequest`, `ScaffoldResult` | +| `SW.Serverless.Tooling.Building` | `PackageBuilder`, `BuildRequest`, `BuildResult`, `VendoredPackage`, `IgnoreRules`, `SecretScanner` | +| `SW.Serverless.Tooling.Conformance` | `ConformanceRunner`, `ConformanceOptions`, `ConformanceReport`, `ContractDocument` | +| `SW.Serverless.Tooling` | `LocalAdapterHost`, `PackagePublisher`, `PublishPackageRequest`, `AdapterRepository`, `CloudFilesFactory` | +| `SW.Serverless.Installer` | `ServerlessUploadOptions` (the storage settings `CloudFilesFactory` takes) | +| `SW.Serverless.Contract.Catalog` | `AdapterManifest`, `AdapterCatalogEntry`, `AdapterCatalogStore`, `AdapterCatalogPaths`, `AdapterSelfDescription` | + +## Templates: Scaffolder + +`Scaffolder.Scaffold(request)` writes the generic greeter that `sw-serverless init` writes. +`Scaffolder.Scaffold(request, templates)` writes your files instead, after the same checks: a name +that can be a project and class name, a language the SDKs support (`dotnet`, `python`, `node`, +`typescript`), a valid id, and an empty folder. + +A template set is a function from the name, id, language and kind to a list of files: + +```csharp +public delegate IEnumerable<(string File, string Content)> Templates(string name, string id, string language, string kind); +``` + +`kind` is `ScaffoldRequest.Kind`, which the generic templates ignore and yours can use. + +```csharp +using SW.Serverless.Tooling.Scaffolding; + +static IEnumerable<(string File, string Content)> OrdersTemplates(string name, string id, string language, string kind) +{ + if (language != "python") yield break; // this example only has Python templates + + yield return ("adapter.json", $$""" + { + "id": "{{id}}", + "version": "0.1.0", + "displayName": "{{Scaffolder.Spaced(name)}}", + "runtime": "python", + "entry": "main.py" + } + """); + + yield return ("main.py", kind == "source" + ? $$""" + import acme_orders + import sw_serverless as sw + + + class {{name}}(acme_orders.Source): + def list(self) -> list[str]: + return [] + + def fetch(self, order_id: str) -> dict: + return {"OrderId": order_id} + + + if __name__ == "__main__": + sw.run({{name}}) + """ + : $$""" + import acme_orders + import sw_serverless as sw + + + class {{name}}(acme_orders.Processor): + def process(self, order: dict) -> dict: + return {"Accepted": True, "Reference": order["OrderId"]} + + + if __name__ == "__main__": + sw.run({{name}}) + """); + + yield return ("requirements.txt", "acme-orders==1.0.0\n"); + yield return (".gitignore", "__pycache__/\nbin/\nsettings.json\n"); +} + +var result = Scaffolder.Scaffold(new ScaffoldRequest +{ + Name = "PartnerFeed", + Language = "python", + Kind = "source", + ParentDirectory = ".", +}, OrdersTemplates); + +foreach (var problem in result.Problems) Console.WriteLine(problem); +``` + +Helpers for writing templates: `Scaffolder.IdFrom(name)` (`AcmeOrders` to `acme.orders`), +`Scaffolder.Spaced(name)` (`Acme Orders`), `Scaffolder.NodePackageJson(id, typeScript, dependencies)`, +`Scaffolder.TsConfig`, and `Scaffolder.InLanguage(language)` for README text. + +## Your own packages: BuildRequest.Packages + +An application usually gives its adapter authors a small package of its own: base classes that +declare the contract's kinds and command names, and the payload types. For .NET this is a NuGet +package like any other. For Python and Node, which `sw-serverless build` builds without PyPI or +npm, the build can **vendor** it from files you hand it, exactly as it vendors the SDK: + +```csharp +using SW.Serverless.Tooling.Building; + +var request = new BuildRequest +{ + ProjectDirectory = "./PartnerFeed", + Log = Console.WriteLine, + Packages = + { + new VendoredPackage + { + Runtime = "python", + Name = "acme-orders", // as requirements.txt names it + Files = { ["acme_orders/__init__.py"] = File.ReadAllBytes("python/acme_orders/__init__.py") }, + }, + new VendoredPackage + { + Runtime = "node", + Name = "@acme/orders", // as package.json names it + Files = + { + ["@acme/orders/package.json"] = File.ReadAllBytes("node/package.json"), + ["@acme/orders/index.js"] = File.ReadAllBytes("node/index.js"), + }, + }, + }, +}; + +var result = await PackageBuilder.BuildAsync(request); +if (!result.Succeeded) foreach (var problem in result.Problems) Console.Error.WriteLine(problem); +else Console.WriteLine($"Built {result.ZipPath}"); +``` + +- `Files` are keyed by path under `_vendor/` (Python) or `node_modules/` (Node). A path that + would land outside that folder is refused. +- `Name` is the name an adapter's `requirements.txt` or `package.json` uses for it. The build skips + that name when it calls `pip` or `npm`, so naming it there (for editors and readers) does not + send the build to a registry that does not have it. +- Packages for one runtime are ignored when building another, and for .NET. + +The Python package above could be: + +```python +# acme_orders/__init__.py +import sw_serverless as sw + + +@sw.implements("orders", 1, "processor") +class Processor: + @sw.command("Process", description="Takes an order and says whether it was accepted.") + def process(self, order: dict) -> dict: + raise NotImplementedError + + +@sw.implements("orders", 1, "source") +class Source: + @sw.command("Open") + def open(self) -> None: ... + + @sw.command("List") + def list(self) -> list[str]: + raise NotImplementedError + + @sw.command("Fetch") + def fetch(self, order_id: str) -> dict: + raise NotImplementedError + + @sw.command("Remove") + def remove(self, order_id: str) -> None: ... + + @sw.command("Close") + def close(self) -> None: ... +``` + +A subclass overrides the methods; the command names, kinds and contract come from the base class. + +Other `BuildRequest` options: + +| Property | Default | Meaning | +|---|---|---| +| `ProjectDirectory` | (required) | The adapter's folder, with its `adapter.json`. | +| `OutputDirectory` | `/bin/serverless` | Where `package/` and the zip go. | +| `IncludeSource` | `true` | Carry the source under `source/`. | +| `AllowedFiles` | empty | Source files whose secret-scan finding is a false positive. | +| `DryRun` | `false` | Collect and scan the source only. | +| `Runtimes` | `python3`, `node`, `dotnet` on the `PATH` | Where the build finds Python and Node. | +| `Log` | nothing | Progress lines. | + +`BuildResult` has `Succeeded`, `Problems`, `Warnings`, `Manifest`, `PackageDirectory`, `ZipPath`, +`SourceFiles` and secret `Findings`. + +## Your contract: ContractDocument.Register + +Register your contract once when your tool starts, and the conformance kit checks every adapter +that declares it, without `--contract`: + +```csharp +using SW.Serverless.Tooling.Conformance; + +ContractDocument.Register(ContractDocument.FromJson( + Embedded("orders-adapter-contract.v1.json"), + readSibling: Embedded)); // reads order.schema.json and the other schema files it names +``` + +The contract format and what the kit checks are in [Contracts](contracts.md). + +## Running and testing: LocalAdapterHost and ConformanceRunner + +`LocalAdapterHost` runs an unpacked package on this machine exactly as a host would: it puts it in +a temporary local store, installs it from there, and starts it, classic or resident. + +```csharp +using SW.Serverless.Tooling; + +await using var adapter = await LocalAdapterHost.StartAsync( + result.PackageDirectory, + settings: new Dictionary { ["Folder"] = "/data/test-orders" }, + commandTimeoutSeconds: 30); + +Console.WriteLine(await adapter.CallAsync("List", null)); // the answer as text: raw string, or JSON +await adapter.CallVoidAsync("Close", null); + +var (description, problem) = await LocalAdapterHost.DescribeAsync( + Path.Combine(result.PackageDirectory, result.Manifest.Entry), result.Manifest.Runtime); +``` + +`ConformanceRunner` runs the checks `sw-serverless test` runs: + +```csharp +var report = await new ConformanceRunner().RunAsync(new ConformanceOptions +{ + PackageDirectory = result.PackageDirectory, + Settings = settings, + AllowDelete = false, + CommandTimeoutSeconds = 60, + Log = Console.WriteLine, +}); +``` + +`ConformanceOptions` also takes `Contracts` (used before registered ones), `WorkDirectory` and +`Runtimes`. + +## Publishing: PackagePublisher and AdapterRepository + +Get an `ICloudFilesService` for your storage, either the one your application already registers, +or one built from settings: + +```csharp +using SW.Serverless.Installer; // ServerlessUploadOptions +using SW.Serverless.Tooling; + +var files = CloudFilesFactory.Create(new ServerlessUploadOptions +{ + Provider = "s3", // s3, as, gc, oc or local + BucketName = "orders-adapters", + AccessKeyId = Environment.GetEnvironmentVariable("ADAPTERS_ACCESS_KEY"), + SecretAccessKey = Environment.GetEnvironmentVariable("ADAPTERS_SECRET_KEY"), + ServiceUrl = "https://s3.example.com", +}); +``` + +Publish a built package. Unlike the CLI, you can choose the root folder in storage; hosts must be +configured with the same one (`ServerlessOptions.AdapterRemotePath`): + +```csharp +var published = await PackagePublisher.PublishPackageAsync(files, new PublishPackageRequest +{ + PackagePath = result.ZipPath, + Version = "minor", // or "1.4.0", or null for the manifest's version + Promote = true, + ReleaseNotes = "Retries on 503.", + PublishedBy = "orders-adapters", + RemotePath = "orders-adapters", // the root folder; "adapters" unless set +}, log: Console.WriteLine); + +Console.WriteLine($"{published.Manifest.Id} {published.Version} {published.Sha256}"); +``` + +Manage versions with `AdapterRepository`, given the same root: + +```csharp +var repository = new AdapterRepository(files, Console.WriteLine, root: "orders-adapters"); + +var listing = await repository.ListVersionsAsync("acme.orders"); // Current, Versions, FromCatalog +await repository.PromoteAsync("acme.orders", "1.3.0", workDirectory: Path.GetTempPath()); +await repository.WithdrawAsync("acme.orders", "1.4.0"); +var next = await repository.ResolveVersionAsync("acme.orders", "patch"); + +var entry = await repository.Catalog.GetAsync("acme.orders"); // the catalog entry +``` + +These are the operations behind `publish`, `promote`, `withdraw` and `versions`, with the same +rules (see [The CLI](cli.md) and [Packaging and storage](packaging-and-storage.md)). Failures the +caller can act on are thrown as `SWException` (from `SimplyWorks.PrimitiveTypes`) with a message +that says what to do; an invalid or existing version as `ArgumentException`. + +## Putting it together + +A minimal `orders-adapters` tool, with `init`, `build`, `test` and `publish`: + +```csharp +using SW.Serverless.Installer; +using SW.Serverless.Tooling; +using SW.Serverless.Tooling.Building; +using SW.Serverless.Tooling.Conformance; +using SW.Serverless.Tooling.Scaffolding; + +ContractDocument.Register(ContractDocument.FromJson(Embedded("orders-adapter-contract.v1.json"), Embedded)); + +switch (args[0]) +{ + case "init": // orders-adapters init + { + var made = Scaffolder.Scaffold(new ScaffoldRequest { Name = args[1], Kind = args[2], Language = "python" }, OrdersTemplates); + made.Problems.ForEach(Console.WriteLine); + return made.Succeeded ? 0 : 1; + } + case "build": // orders-adapters build + { + var built = await PackageBuilder.BuildAsync(new BuildRequest { ProjectDirectory = args[1], Packages = { OrdersPython() }, Log = Console.WriteLine }); + built.Problems.ForEach(Console.WriteLine); + return built.Succeeded ? 0 : 1; + } + case "test": // orders-adapters test + { + var report = await new ConformanceRunner().RunAsync(new ConformanceOptions { PackageDirectory = args[1], Log = Console.WriteLine }); + report.Checks.ForEach(c => Console.WriteLine($"{c.Outcome} {c.Name} {c.Detail}")); + return report.Passed ? 0 : 1; + } + case "publish": // orders-adapters publish + { + var files = CloudFilesFactory.Create(new ServerlessUploadOptions { Provider = "local", BucketName = "dev", ServiceUrl = "/tmp/orders-store" }); + await PackagePublisher.PublishPackageAsync(files, new PublishPackageRequest { PackagePath = args[1], RemotePath = "orders-adapters" }); + return 0; + } + default: + return 1; +} +``` + +`OrdersTemplates` is the template function from [Templates](#templates-scaffolder), `OrdersPython()` +returns the Python `VendoredPackage` from [Your own packages](#your-own-packages-buildrequestpackages), +and `Embedded` reads a resource embedded in the tool, as in +[Contracts](contracts.md#shipping-your-contract-with-your-own-tool). diff --git a/docs/hosting.md b/docs/hosting.md new file mode 100644 index 0000000..61bcfa8 --- /dev/null +++ b/docs/hosting.md @@ -0,0 +1,467 @@ +# Hosting adapters + +This page is for developers adding SW-Serverless to an application that runs adapters. Read +[Concepts](concepts.md) first if the words host, session, resident or protocol are new. + +- [Packages to add](#packages-to-add) +- [Registering the host](#registering-the-host) +- [Options](#options) +- [Storage](#storage) +- [Classic sessions: IServerlessService](#classic-sessions-iserverlessservice) +- [Resident adapters: IResidentAdapterHost](#resident-adapters-iresidentadapterhost) +- [The event sink](#the-event-sink) +- [The state store](#the-state-store) +- [Logs and metrics](#logs-and-metrics) +- [Pinning versions](#pinning-versions) +- [How installation works](#how-installation-works) +- [Limits and supervision](#limits-and-supervision) +- [What the host machine needs](#what-the-host-machine-needs) +- [Security](#security) + +## Packages to add + +```sh +dotnet add package SimplyWorks.Serverless +``` + +and one storage provider, from the `SimplyWorks.CloudFiles` family: + +| Provider | Package | Registration | +|---|---|---| +| S3 and S3-compatible | `SimplyWorks.CloudFiles.S3.Extensions` | `AddS3CloudFiles` | +| Azure Blob Storage | `SimplyWorks.CloudFiles.AS.Extensions` | `AddAsCloudFiles` | +| Google Cloud Storage | `SimplyWorks.CloudFiles.GC.Extensions` | `AddGoogleCloudFiles` | +| Oracle Object Storage | `SimplyWorks.CloudFiles.OC.Extensions` | `AddOracleCloudFiles` | +| A local folder (development) | `SimplyWorks.CloudFiles.LocalTests.Extensions` | `AddLocalTestsCloudFiles` | + +The library targets .NET 10 and uses ASP.NET Core (it serves the gRPC endpoint adapters connect +to), so the application must be able to reference `Microsoft.AspNetCore.App`. + +## Registering the host + +```csharp +using SW.CloudFiles.Extensions; +using SW.Serverless; +using SW.Serverless.Resident; + +// 1. Where adapters are published. +services.AddS3CloudFiles(o => +{ + o.BucketName = "my-adapters"; + o.AccessKeyId = configuration["Adapters:AccessKey"]; + o.SecretAccessKey = configuration["Adapters:SecretKey"]; + o.ServiceUrl = "https://s3.eu-central-1.amazonaws.com"; +}); + +// 2. Optional: where python3, node and dotnet are. Must come before AddServerless. +services.AddAdapterRuntimes(o => o.PythonExecutable = "/usr/bin/python3.12"); + +// 3. Classic sessions, and the installer both lifecycles share. +services.AddServerless(o => +{ + o.AdapterLocalPath = "/var/lib/myapp/adapters"; + o.CommandTimeout = 60; +}); + +// 4. Resident adapters, and protocol 2 sessions (every Python, Node and exec adapter). +services.AddResidentAdapters(o => +{ + o.HeartbeatInterval = TimeSpan.FromSeconds(15); + o.SoftMemoryLimitBytes = 512L * 1024 * 1024; +}); +``` + +What each call registers: + +| Call | Registers | +|---|---| +| `AddServerless(Action)` | `IServerlessService` (transient), `ServerlessOptions`, the shared `AdapterInstaller` and `AdapterRuntimes`, a memory cache. | +| `AddAdapterRuntimes(Action)` | Where the host finds `python3`, `node` and `dotnet`. The other calls register defaults only if none is registered, so call this one first. | +| `AddResidentAdapters(Action)` | `IResidentAdapterHost` (a singleton and a hosted service), your `IAdapterEventSink`, and an in-memory `IAdapterStateStore`. | +| `AddResidentAdapters(...)` | The same, with your own `IAdapterStateStore`. | + +Rules to know: + +- `AddResidentAdapters` works on its own when every adapter is given by path + (`AdapterSpec.EntryAssemblyPath`). To install adapters from storage by id, register + `AddServerless` and an `ICloudFilesService` as well: the resident host installs through the same + installer, which needs both, and says so if they are missing when it first needs them. +- Register `AddResidentAdapters` whenever you run anything other than .NET classic adapters. A + Python, Node or `exec` adapter, and a .NET adapter on protocol 2, is run on the resident host even + when it is called as a classic session. Without it, `StartAsync` fails with a message saying so. +- The resident host is an `IHostedService`. It opens its socket when the application's host starts, + so use it from a running host (`WebApplication`, `Host.CreateDefaultBuilder`, and so on). + +## Options + +### ServerlessOptions (`AddServerless`) + +| Option | Default | Meaning | +|---|---|---| +| `AdapterRemotePath` | `"adapters"` | The root folder in storage. Must match the root adapters were published under; the CLI always publishes under `adapters`. | +| `AdapterLocalPath` | `"./adapters"` | Where packages are unpacked on this machine, one folder per package. | +| `AdapterMetadataCacheDuration` | `5` | Minutes the host remembers which package an adapter id resolves to. A newly promoted version is picked up once this expires. | +| `CommandTimeout` | `30` | Seconds a classic session's command may take, unless the call passes its own. | +| `IdleTimeout` | `300` | Seconds a protocol 1 (.NET classic) adapter waits for its next command before it exits. | + +### ResidentOptions (`AddResidentAdapters`) + +| Option | Default | Meaning | +|---|---|---| +| `SocketPath` | `/tmp/swsl-{pid}.sock` | The Unix domain socket adapters connect to (Linux, macOS). macOS limits the path to about 104 bytes. | +| `PipeName` | `swsl-{pid}` | The named pipe adapters connect to (Windows). | +| `HandshakeTimeout` | 30 s | How long a started adapter has to connect and say Hello. | +| `InvokeTimeout` | 300 s | Default timeout for a command on a resident instance. | +| `HeartbeatInterval` | 15 s | How often each instance is pinged and its process sampled. | +| `MissedHeartbeatsBeforeRestart` | `3` | Unanswered pings in a row before the process is killed and restarted. | +| `MaxInFlight` | `16` | How many events one instance may have waiting for the event sink. | +| `CrashLoopThreshold` / `CrashLoopWindow` | `5` / 5 min | Crashes within the window before the instance is quarantined (no more restarts). | +| `SoftMemoryLimitBytes` | `0` (off) | Memory above which the adapter is asked to drain and is then restarted. | +| `HardMemoryLimitBytes` | `0` (off) | Memory above which the process is killed. Also given to the runtime as a heap limit. | +| `CpuPercentLimit` | `0` (off) | Sustained CPU, as a share of the whole machine, above which the adapter is asked to drain. | +| `CpuLimitSamples` | `4` | Heartbeats in a row above `CpuPercentLimit` before it trips. | +| `DrainDeadline` | 60 s | How long an adapter asked to drain has to exit before it is killed. | +| `IdleTimeout` | 10 min | How long a pooled instance may sit unused before it is stopped. `TimeSpan.Zero` disables this. | +| `DiagnosticBufferLines` | `200` | Lines of an adapter's stdout and stderr kept for error messages. | +| `UseWorkstationGc` | `true` | Start .NET adapters with workstation GC, which uses less memory than server GC. | + +### AdapterRuntimeOptions (`AddAdapterRuntimes`) + +| Option | Default | +|---|---| +| `PythonExecutable` | `"python3"` | +| `NodeExecutable` | `"node"` | +| `DotnetExecutable` | `"dotnet"` | + +Each is looked up on the `PATH` unless you give a full path. `DotnetExecutable` is used for .NET +adapters on protocol 2; protocol 1 sessions always start `dotnet` from the `PATH`. + +## Storage + +The host reads adapters from the `ICloudFilesService` you register, under `AdapterRemotePath`. Use +the same bucket and credentials (read access is enough) that adapters are published to. + +For development, publish to a folder and point the host at it: + +```sh +sw-serverless publish bin/serverless/greeter-0.1.0.zip -p local -b adapters-dev -u /tmp/swsl-store +``` + +```csharp +services.AddLocalTestsCloudFiles(o => +{ + o.BucketName = "adapters-dev"; + o.StoragePath = "/tmp/swsl-store"; +}); +``` + +The layout inside storage is described in [Packaging and storage](packaging-and-storage.md). + +## Classic sessions: IServerlessService + +`IServerlessService` (namespace `SW.PrimitiveTypes`) runs one **session**: it starts the adapter's +process, calls commands one at a time, and stops the process when it is disposed. Resolve one per +session from a scope, and let the scope end the session. + +```csharp +using Microsoft.Extensions.DependencyInjection; +using SW.PrimitiveTypes; + +public class OrderService(IServiceScopeFactory scopes) +{ + public async Task ProcessAsync(Order order, string partnerKey) + { + using var scope = scopes.CreateScope(); + var serverless = scope.ServiceProvider.GetRequiredService(); + + await serverless.StartAsync( + adapterId: "acme.orders", + correlationId: order.Id, + startupValues: new Dictionary { ["ApiKey"] = partnerKey }); + + var receipt = await serverless.InvokeAsync("Process", order); + var log = await serverless.InvokeAsync("GetLog", null); // same process, same session + return receipt; + } // disposing the scope stops the adapter +} +``` + +| Member | What it does | +|---|---| +| `StartAsync(adapterId, correlationId, startupValues = null)` | Installs the adapter if needed and starts it with these settings. `adapterId` may pin a version: `"acme.orders/1.4.0"`. The host adds `CorrelationId` to the values. | +| `StartAsync(adapterId, correlationId, adapterPath, startupValues = null)` | Starts a .NET protocol 1 adapter from a `.dll` on disk, without storage. For development and tests. | +| `InvokeAsync(command, input, commandTimeout = 0)` | Calls a command and converts its result to `TResult`. `commandTimeout` is in seconds; `0` uses `CommandTimeout`. | +| `InvokeAsync(command, input, commandTimeout = 0)` | Calls a command that returns nothing. | +| `GetExpectedStartupValues()` | The settings the adapter declares: name, `Optional`, `Default`, `Type`, `Private`, `Description`. | +| `Dispose()` | Stops the adapter. `ServerlessService` implements `IDisposable`; the scope calls it. | + +Rules of a session: + +- One command at a time. Wait for each call to finish before the next. Use one session per + concurrent use. +- How `input` is sent: a `string` as its raw text, `null` as no argument, anything else as JSON. A + `string` result is the raw text; anything else is read as JSON into `TResult`. See + [encoding](writing-adapters.md#encoding). +- A failing command throws. On protocol 2 the exception is `AdapterInvocationException` (namespace + `SW.Serverless.Resident`) with `AdapterExceptionType` (the adapter's error type), `Message` and + `Detail` (its stack trace). On protocol 1 it is an `Exception` whose message carries the + adapter's exception text. The session stays usable after a failed command. +- A command that runs past its timeout throws `TimeoutException`. On protocol 1 the process is + then killed and the session cannot be used again. On protocol 2 the adapter is told the call was + cancelled, and the session stays usable. + +## Resident adapters: IResidentAdapterHost + +`IResidentAdapterHost` (namespace `SW.Serverless.Resident`) is a singleton that owns every +long-running adapter process on this machine. + +### Exclusive instances + +An exclusive instance is one process per `(AdapterId, InstanceKey)`. Starting a key that is +already running returns the running instance. + +```csharp +var instance = await adapters.StartExclusiveAsync(new AdapterSpec +{ + AdapterId = "acme.queue-reader", + InstanceKey = "subscription-42", + StartupValues = { ["Host"] = "broker.internal", ["Queue"] = "orders" }, + SoftMemoryLimitBytes = 256L * 1024 * 1024, +}); + +var stats = await instance.InvokeAsync("GetStats"); +await instance.InvokeAsync("SetPrefetch", 50); + +await adapters.StopAsync("acme.queue-reader", "subscription-42", drain: true); +``` + +`AdapterSpec`: + +| Property | Meaning | +|---|---| +| `AdapterId` | The id, optionally pinned: `"acme.queue-reader/2.1.0"`. | +| `InstanceKey` | The exclusive instance's key. Left empty for pooled rentals. | +| `StartupValues` | The adapter's settings. Sent over the socket, never on the command line. | +| `AdapterValues` | Extra values the adapter can read (in .NET, `IAdapterContext.AdapterValues`). Merged over the package's storage metadata. Pools read `PoolSize` and `IdleTimeoutSeconds` from here. | +| `EntryAssemblyPath` | Start this file instead of installing from storage. For development. | +| `Runtime` | With `EntryAssemblyPath`: `dotnet`, `python`, `node` or `exec`. Filled in from the manifest when installing from storage. | +| `PoolKey` | For pooled rentals: which pool. Defaults to the id plus a hash of `StartupValues`, so different settings never share a process. | +| `SoftMemoryLimitBytes`, `HardMemoryLimitBytes`, `CpuPercentLimit`, `CpuLimitSamples` | Limits for this instance; `0` uses the host default. | + +`IResidentAdapterHost`: + +| Member | What it does | +|---|---| +| `StartExclusiveAsync(spec)` | Starts the instance, or returns it if it is already running. Waits until the adapter has connected. | +| `StopAsync(adapterId, instanceKey, drain = true)` | Asks the adapter to stop (with `drain`, to finish what it is doing first: up to 30 s, otherwise 5 s), then ends the process. | +| `RestartAsync(adapterId, instanceKey, drain = true)` | Stops and starts it again under the same key. | +| `UpdateLimitsAsync(adapterId, instanceKey, ResourceLimits)` | Changes its limits while it runs. Soft memory and CPU limits apply at once; a hard memory change returns `RestartRequired = true`, because the runtime reads it at launch. | +| `RentAsync(spec)` | Rents a pooled instance. See below. | +| `Get(adapterId, instanceKey)`, `List()` | The running `ResidentAdapterInstance` objects. | +| `Describe()` | An `InstanceHealth` for each instance (see [Limits and supervision](#limits-and-supervision)). | + +`ResidentAdapterInstance`: + +| Member | What it does | +|---|---| +| `InvokeAsync(command, input = null, timeoutSeconds = 0, cancellationToken = default, sessionId = null, properties = null)` | Calls a command. Several calls may run at once. `properties` are per-call values the adapter reads like settings. `0` seconds uses `InvokeTimeout`. | +| `InvokeAsync(command, byte[] payload, ...)` | The same with raw bytes in and out. | +| `PingAsync(timeout)` | Pings it now and returns its status (`Pong`). | +| `ResetAsync(sessionId)` | Tells it to forget a session. | +| `SetLogLevelAsync(LogLevel)` | Changes the lowest level it sends logs at. | +| `State`, `Commands`, `CommandDetails`, `Settings`, `Kinds`, `Contracts`, `SdkVersion`, `SdkLanguage`, `ProtocolVersion` | What it said about itself when it connected. | + +### Pooled instances + +A pool keeps a few warm processes of one adapter with one set of settings, and hands them out for +short uses. It replaces starting a process per use. + +```csharp +await using (var lease = await adapters.RentAsync(new AdapterSpec +{ + AdapterId = "acme.pricing", + StartupValues = { ["Region"] = "eu" }, + AdapterValues = { ["PoolSize"] = "4" }, +})) +{ + var quote = await lease.InvokeAsync("Price", basket); + var audit = await lease.InvokeAsync("GetAudit"); // same session as the call above +} // returned to the pool; the adapter is told to reset this session +``` + +- Calls through a lease carry the lease's session id. Returning the lease sends a reset for that + session, and waits for the adapter to confirm it before the process is rented again. A pooled + adapter must forget per-session state on reset: in .NET implement `IResettable`, in Python and + Node a `reset(session_id)` method. +- `PoolSize` (in `AdapterValues`, default 4, at most 64) caps the processes in one pool. + `IdleTimeoutSeconds` overrides `ResidentOptions.IdleTimeout` for this adapter. +- Every distinct set of `StartupValues` is its own pool, unless you set `PoolKey`. + +## The event sink + +A resident adapter pushes work into the host by publishing **events**: a message read from a queue, +a file found in a folder. You implement `IAdapterEventSink` to receive them. + +```csharp +public class OrderEventSink(OrdersDb db) : IAdapterEventSink +{ + public async Task OnEventAsync(InboundEvent e, CancellationToken cancellationToken) + { + if (await db.HasSeenAsync(e.DedupeKey, cancellationToken)) + return EventOutcome.Ok(reference: "duplicate"); + + var id = await db.SaveIncomingAsync(e.AdapterId, e.Endpoint, e.ContentType, e.Payload, cancellationToken); + return EventOutcome.Ok(reference: id.ToString()); + } +} +``` + +`InboundEvent` has `AdapterId`, `InstanceKey`, `Endpoint` (where it came from: a queue, a topic, a +folder), `DedupeKey`, `ContentType`, `Payload` (bytes), `Headers` and `Traceparent`. + +How it works: + +1. The adapter publishes an event and waits. +2. The host calls `OnEventAsync`. Return `EventOutcome.Ok(reference)` once the event is stored + durably, or `EventOutcome.Rejected(error)` to refuse it. An exception is treated as a rejection. +3. The adapter receives the outcome. Only after an accepted outcome should it acknowledge its source + (delete the file, ack the message). After a rejection or a crash, the source still has the + message and the adapter will deliver it again. + +Delivery is therefore at least once. Use `DedupeKey` to recognise a repeat. At most `MaxInFlight` +events per instance wait for the sink at once. + +## The state store + +A resident adapter may keep a small piece of durable state with the host: the cursor of a polling +reader, the time of its last run. The adapter cannot keep this itself, because it is restarted, may +run on another machine next time, and a pooled one is not the same process twice. + +```csharp +public class OrderStateStore(OrdersDb db) : IAdapterStateStore +{ + public Task GetAsync(AdapterStateKey key, CancellationToken cancellationToken) => + db.GetAdapterStateAsync(key.AdapterId, key.InstanceKey, key.Name, cancellationToken); // null when absent + + public Task SetAsync(AdapterStateKey key, string value, CancellationToken cancellationToken) => + value == null + ? db.DeleteAdapterStateAsync(key.AdapterId, key.InstanceKey, key.Name, cancellationToken) + : db.SaveAdapterStateAsync(key.AdapterId, key.InstanceKey, key.Name, value, cancellationToken); +} +``` + +- Keys are scoped by adapter id, instance key and a name the adapter chooses. +- `SetAsync` must be durable before it returns. A `null` value deletes the entry. +- Values are meant to be small. You may refuse a large one by throwing; the adapter receives the + error. +- The default `InMemoryAdapterStateStore` keeps state in memory, per process. Replace it in any real + deployment with `AddResidentAdapters`. + +## Logs and metrics + +- Adapter log lines arrive as ordinary `ILogger` entries in the category + `serverless.adapters.{adapter id}`. Configure their level and output like any other logs. Use + `instance.SetLogLevelAsync` to change what a resident adapter sends. +- Metrics that resident adapters record are published as counters on the + `System.Diagnostics.Metrics` meter named `SW.Serverless.Adapters`, tagged `adapter.id` and + `adapter.instance`. Export them as you export other metrics. The host keeps at most 200 metric + names and 10 tags per metric. + +## Pinning versions + +Pass `"{id}/{version}"` wherever an adapter id is taken: + +```csharp +await serverless.StartAsync("acme.orders/1.4.0", correlationId); +await adapters.StartExclusiveAsync(new AdapterSpec { AdapterId = "acme.orders/1.4.0", InstanceKey = "main" }); +``` + +A plain id runs the current version. A pinned version is fetched from `adapters-versions/` (or from +where older publishing tools put versions). Withdrawing a version does not stop a host that pins it; +the package stays in storage. + +## How installation works + +When the host is asked for an adapter id it: + +1. Resolves the id to a package in storage: `adapters/{id}`, or for an adapter with no package + there (Python, Node, `exec`), the catalog's current version; or the pinned version. The answer + is cached for `AdapterMetadataCacheDuration` minutes. +2. Downloads the zip to a temporary file and unpacks it into `AdapterLocalPath/{package hash}`, into + a temporary folder first and then renamed into place, so a half-unpacked package is never used. + `source/` is not unpacked. An entry that would land outside the folder fails the install, and so + does a package that unpacks to more than 4 GB (`AdapterInstaller.MaxExtractedBytes`). +3. Reads `adapter.json`, and refuses to start the adapter if this host does not have its runtime, + has the runtime at a version outside the manifest's `runtimeVersion`, is not one of its + `platforms`, or is older than its `compatibility.minHostVersion`. The error says which. +4. Deletes older unpacked versions of the same adapter that no running process uses. + +A package with no `adapter.json` (from before manifests) runs as a .NET adapter, as it always did. + +## Limits and supervision + +Supervision applies to processes on the resident host: resident instances, and classic sessions +on protocol 2 (Python, Node, `exec`, and .NET adapters that opt in). Protocol 1 .NET sessions have +the command timeout and the idle timeout, and nothing else. + +On every heartbeat the host samples each process's memory (resident set size) and CPU, and pings +the adapter: + +| Finding | What the host does | +|---|---| +| Memory above the soft limit | Asks the adapter to drain (finish in-flight work and exit), then restarts it. Killed if it hasn't exited after `DrainDeadline`. | +| Memory above the hard limit | Kills the process tree and restarts it. | +| CPU above `CpuPercentLimit` for `CpuLimitSamples` heartbeats | Asks it to drain, then restarts it. There is no hard CPU kill. | +| `MissedHeartbeatsBeforeRestart` pings unanswered | Kills it and restarts it. | +| The process exits unasked | Restarts it after a backoff. After `CrashLoopThreshold` crashes in `CrashLoopWindow`, quarantines it: it stays stopped until started again. | + +Notes: + +- The CPU figure is a share of the whole machine. On a 16-core machine one busy core is about 6%. +- The hard memory limit is also given to the runtime: `DOTNET_GCHeapHardLimit` for .NET and + `--max-old-space-size` for Node, so an allocation past it fails inside the adapter. Python has no + such setting; only the host's kill applies. +- On Linux the host sets each adapter's `oom_score_adj` to 500, so under memory pressure the kernel + kills an adapter before the host. +- SW-Serverless does not set operating-system limits such as cgroups. For a hard guarantee, run the + host in a container with its own limits. + +`Describe()` reports, per instance, what the host observes (`ProcessId`, `WorkingSetBytes`, +`CpuPercent`, `ThreadCount`, `Uptime`, `RestartCount`, `MissedHeartbeats`, `Quarantined`, +`DrainRequested`, `LastHeartbeatOn`, `IdleSince`), what the adapter reports in its status +(`Connected`, `ReportedState`, `LastMessageOn`, `InFlight`, `LastError`, `Details`), what it +declared when it connected (commands, settings, kinds, contracts, SDK), and the last lines of its +output (`Diagnostics`). + +## What the host machine needs + +| Adapters you run | The host machine needs | +|---|---| +| .NET | `dotnet` and the .NET runtime the adapters target. | +| Python | `python3`, version 3.12 or later (or what the adapter's `runtimeVersion` asks for). Nothing from PyPI: packages carry their dependencies. | +| JavaScript, TypeScript | `node`, version 22 or later (or what the adapter's `runtimeVersion` asks for). Nothing from npm. | +| `exec` | Nothing; the package is the program. It must be built for the host's platform. | + +The Python and Node SDKs connect over a Unix domain socket only, so Python and Node adapters run +on Linux and macOS hosts, not Windows. .NET adapters run on all three. + +## Security + +- **An adapter is not sandboxed.** It runs as the same operating-system user as the host, with the + same file and network access. Run only adapters you trust, or run the host in a container with + the access the adapters should have. +- **No network port.** Adapters connect to the host over a Unix domain socket or a named pipe. On + Unix the socket file is made readable and writable by its owner only, and its folder, unless it + is `/tmp`, by its owner only. +- **A one-time token.** The host writes the socket path and a random token on the adapter's + standard input. The adapter must present the token in its first message; an unknown or reused + token is refused. +- **Settings stay off the command line.** On protocol 2, settings travel over the socket. On + protocol 1 they are written on the adapter's standard input when its SDK is 10.1.0 or later + (known from the manifest's `sdkVersion`). Adapters built on older SDKs receive them as + base64-encoded command-line arguments, which other processes on the machine can read; rebuild + them on a current SDK. +- **Runtimes are names, not paths.** A manifest says `python` or `node`; the host decides which + program that means. A package's entry, and every file in it, must lie inside the package. +- **Secrets in source.** `sw-serverless build` refuses to package source that looks like it holds a + key or password. Mark secret settings as secret, so applications mask them. diff --git a/docs/manifest.md b/docs/manifest.md new file mode 100644 index 0000000..c60b8c4 --- /dev/null +++ b/docs/manifest.md @@ -0,0 +1,227 @@ +# The manifest: adapter.json + +Every package has an `adapter.json` at its root: the **manifest**. It tells a host how to run the +adapter, and tells an application what the adapter needs and how to present it. + +There are two files with that name: + +- **The one you write**, beside your project. It holds what only you know: the id, the version, + names and descriptions, the icon, and for Python and Node the runtime and entry file. It can be + short. +- **The one in the package**, which `sw-serverless build` writes. It starts from yours and adds what + the build learns from the built adapter: entry, lifecycle, protocol, SDK version, settings, kinds, + contracts, source files. Where yours and the adapter's own description disagree, the adapter + wins. + +You never edit the second one by hand. + +## A complete example + +What you write, for a Python adapter: + +```json +{ + "id": "acme.orders", + "version": "1.2.0", + "displayName": "Acme Orders", + "summary": "Sends orders to Acme and returns their receipts.", + "description": "Sends each order to Acme's order API.\n\nNeeds an **API key** from Acme.", + "publisher": { "name": "Acme Integrations", "url": "https://acme.example", "email": "dev@acme.example" }, + "license": "MIT", + "homepage": "https://acme.example/adapters/orders", + "repository": "https://github.com/acme/orders-adapter", + "icon": "icon.png", + "tags": ["orders", "partners"], + "categories": ["Partners"], + "runtime": "python", + "entry": "main.py", + "compatibility": { + "minHostVersion": "10.1.0", + "applications": { "orders-app": "3.2.0" } + }, + "properties": [ + { "name": "Endpoint", "displayName": "API URL", "group": "Connection" }, + { "name": "ApiKey", "displayName": "API key", "group": "Connection" }, + { "name": "Mode", "type": "select", "options": ["test", "live"] } + ] +} +``` + +What the build writes into the package (abridged), given an adapter that declares the settings +`Endpoint`, `ApiKey` (secret) and `Mode` (default `test`) and implements the `orders` contract's +`processor` kind: + +```json +{ + "manifestVersion": 1, + "id": "acme.orders", + "version": "1.2.0", + "displayName": "Acme Orders", + "summary": "Sends orders to Acme and returns their receipts.", + "kinds": ["processor"], + "runtime": "python", + "language": "python", + "entry": "_serverless_entry.py", + "runtimeVersion": ">=3.12", + "contracts": { "orders": 1 }, + "source": { + "path": "source", + "files": { "adapter.json": "9c1f…", "icon.png": "51d0…", "main.py": "a775…", "requirements.txt": "a43a…" }, + "buildCommand": "sw-serverless build", + "lockfiles": ["requirements.txt"] + }, + "lifecycle": "classic", + "protocol": { "min": 2, "max": 2 }, + "sdkVersion": "10.2.0", + "compatibility": { "minHostVersion": "10.1.0", "applications": { "orders-app": "3.2.0" } }, + "properties": [ + { "name": "Endpoint", "displayName": "API URL", "description": "Where orders go.", "type": "text", "required": true, "secret": false, "group": "Connection" }, + { "name": "ApiKey", "displayName": "API key", "type": "text", "required": true, "secret": true, "group": "Connection" }, + { "name": "Mode", "type": "select", "required": false, "secret": false, "default": "test", "options": ["test", "live"] } + ] +} +``` + +`sw-serverless publish` then sets `version` (if you passed `-v`) and `publishedOn`. + +## Fields + +"You" means the field comes from the `adapter.json` you write. "Build" means `sw-serverless build` +sets it, whatever you wrote. + +### Identity + +| Field | Set by | Meaning | +|---|---|---| +| `manifestVersion` | build | The manifest format version: `1`. | +| `id` | you (required) | The adapter id: lowercase letters, digits, `.`, `_` and `-`, starting with a letter or digit. | +| `version` | you, or `publish -v` | A semantic version, such as `1.4.0` or `1.5.0-rc.1`. `publish` uses it unless `-v` says otherwise. | +| `publishedOn` | publish | When it was published. | + +### Presentation + +All optional, all yours. Applications use them to list and describe the adapter. + +| Field | Meaning | +|---|---| +| `displayName` | Its name, for people. | +| `summary` | One line, for a list or a card. | +| `description` | A longer description, in Markdown. | +| `publisher` | `{ "name", "url", "email" }`. | +| `license`, `homepage`, `repository` | Strings. | +| `icon` | A PNG, JPEG or SVG file inside the package, as a path from the package root. Icons of 64 KB or less are also copied into the catalog. For .NET, make sure the file reaches the publish output (for example with `CopyToPublishDirectory`); `publish` fails if it is missing. | +| `tags`, `categories` | Lists of strings. | +| `releaseNotes` | What changed in this version, in Markdown. `publish --notes` replaces it. | + +### Running + +| Field | Set by | Meaning | +|---|---|---| +| `runtime` | you (Python, Node, `exec`); build | `dotnet` (the default), `python`, `node` or `exec`. | +| `entry` | you (Python, Node, `exec`); build | The file the runtime starts, as a path from the package root. For Python you name your script (default `main.py`) and the build replaces it with `_serverless_entry.py`, a small script it writes that sets up the vendored packages and runs yours. For Node you name your script (default `main.js`; a `.ts` entry becomes `.js`). For .NET the build finds the entry assembly. | +| `entries` | you | A different entry per platform, for a package that carries one build per platform: `{ "linux-x64": "bin/linux-x64/adapter", "osx-arm64": "bin/osx-arm64/adapter" }`. Every platform named must also be in `platforms`. `entry` stays the default. | +| `runtimeVersion` | you; build sets a default | The runtime versions it needs. Comparisons separated by commas or spaces: `>=3.12`, `>=22, <24`. A bare version matches that line: `3.12` is any `3.12.x`. Python defaults to `>=3.12`, Node to `>=22`. For .NET it is checked only when you set it. | +| `platforms` | you; build | The platforms it runs on: `linux-x64`, `linux-arm64`, `osx-x64`, `osx-arm64`, `win-x64`. Absent means anywhere its runtime runs. The build sets it when a Python requirement or a Node dependency has native code. A host on another platform refuses the package. | +| `lifecycle` | build | `classic` or `resident`, from what the adapter itself reports. For a .NET adapter that calls `Runner.RunResident`, write `"lifecycle": "classic"` to have it run as classic sessions on protocol 2; see [Writing adapters](writing-adapters.md#a-net-adapter-on-protocol-2-run-as-a-classic-session). | +| `protocol` | build | `{ "min": 2, "max": 2 }` for an adapter on protocol 2. Absent for a .NET adapter on protocol 1. | +| `language` | you; build sets a default | For listing: `csharp`, `fsharp`, `vb`, `python`, `javascript`, `typescript`. | +| `sdkVersion` | build | The SDK version the adapter was built with. Hosts use it to decide whether a .NET classic adapter reads its settings from standard input. | + +### What it is for + +| Field | Set by | Meaning | +|---|---|---| +| `kinds` | you and the adapter | The kinds it implements. The build combines yours with those the adapter declares in code. | +| `contracts` | you and the adapter | The contracts it implements, with versions: `{ "orders": 1 }`. Combined the same way; where both name a contract, the adapter's version wins. | + +### Properties + +`properties` lists the adapter's settings, with what a form needs to ask for them. + +| Property field | Meaning | +|---|---| +| `name` | The setting's name. Letters, digits, `_`, `.`, `:` and `-`, starting with a letter or `_`. | +| `displayName` | A label. | +| `description` | Help text. | +| `type` | `text` (the default), `multiline`, `number`, `boolean`, `select` or `json`. | +| `required` | Whether it must be given. | +| `secret` | Masked wherever it is shown. A secret's default is never written into the manifest. | +| `default` | The value used when none is given. | +| `options` | The choices of a `select`. | +| `group` | A heading to group related settings under, such as `Connection`. | + +The settings themselves come from the adapter's code. The build writes one property per setting +the adapter declares: `name`, `required`, `secret`, `default` and `description` come from the code. +From a property of the same name in your file it keeps `displayName`, `group` and `options`, and +`type` unless your type is `text`. A property in your file that the code does not declare is +dropped. The conformance kit fails a package whose properties and declared settings disagree. + +### Source + +| Field | Set by | Meaning | +|---|---|---| +| `source.path` | build | The folder in the package that holds the source: `source`. | +| `source.files` | build | Every source file, as a path within that folder, with its SHA-256 in hex. | +| `source.buildCommand` | build | How the package was built: `dotnet publish -c Release` or `sw-serverless build`. | +| `source.lockfiles` | build | The dependency lockfiles among the source: `packages.lock.json`, `requirements.txt`, `poetry.lock`, `package-lock.json`, `yarn.lock` and others. | + +Absent when built with `--no-source`. See [Packaging and storage](packaging-and-storage.md#source). + +### Compatibility + +| Field | Meaning | +|---|---| +| `compatibility.minHostVersion` | The oldest SW-Serverless host that may run the adapter. A host older than this refuses to install it. Set it when the adapter relies on host behaviour added in a given release. A host's own version is `SW.Serverless.HostInfo.Version`. | +| `compatibility.applications` | The oldest version of each application that may use the adapter, by application name: `{ "orders-app": "3.2.0" }`. Hosts ignore it. An application reads its own entry and decides. | + +An application reads its minimum with `MinVersionOf`: + +```csharp +var minimum = manifest.Compatibility?.MinVersionOf("orders-app"); +if (minimum != null && Version.Parse(minimum) > myVersion) + throw new InvalidOperationException($"This adapter needs orders-app {minimum} or later."); +``` + +`MinVersionOf(name)` also reads the older form `"minVersion"` written directly under +`compatibility`, for manifests from before `applications` existed: for an application named +`Orders`, `"minOrdersVersion": "3.2.0"`. It matches names without regard to case. + +### Fields the model does not know + +A manifest may contain fields this version does not know, written by a newer tool. They are kept +and written back unchanged by every tool that reads and rewrites the manifest. + +## Validation + +`sw-serverless manifest validate`, `build`, `publish` and the conformance kit all check: + +- `manifestVersion` is 1 or more; +- `id`, if present, uses only lowercase letters, digits, `.`, `_` and `-`, starting with a letter or + digit; +- `version`, if present, is a semantic version (`1.4.0`, `1.4.0-beta.1`); +- `entry`, `icon`, every `entries` value and `source.path` are paths inside the package: relative, + no leading `/` or `\`, no `:`, no `..`; +- `runtime` is a lowercase name (letters, digits, `.`, `-`); +- every platform looks like `linux-x64`; +- every platform named in `entries` is in `platforms`; +- every contract has a name and a version of 1 or more; +- every source file has a 64-digit hex SHA-256; +- `lifecycle` is `classic` or `resident`; +- `protocol.min` is not above `protocol.max`; +- `compatibility.minHostVersion` and every `compatibility.applications` value are versions; +- every property has a valid name, no name appears twice, every `type` is known, and every + `select` has `options`. + +`build` also requires an `id`, and `publish` requires a version, from the manifest or `-v`. + +## The original publishing form + +When a .NET project is published directly with `sw-serverless `, the +manifest is made from the `adapter.json` beside the project file, if any, and the build output: + +- `id`, `version`, `runtime` (`dotnet`), `entry`, `lifecycle`, `protocol`, `sdkVersion` and + `publishedOn` are set by the tool; +- `kinds` come from `-k` if given, else from your file, else from `[AdapterKind]` attributes; +- `properties` come from your file if it lists any; otherwise the tool starts a classic adapter and + asks it for its declared settings (skip with `--no-probe`); +- everything else is yours, as written. diff --git a/docs/packaging-and-storage.md b/docs/packaging-and-storage.md new file mode 100644 index 0000000..0c31c62 --- /dev/null +++ b/docs/packaging-and-storage.md @@ -0,0 +1,312 @@ +# Packaging and storage + +This page describes what is inside a package, how `sw-serverless build` makes one for each +language, and where packages, versions and the catalog are kept in storage. + +- [What a package contains](#what-a-package-contains) +- [.NET](#net) · [Python](#python) · [Node and TypeScript](#node-and-typescript) · [exec](#exec) +- [Source](#source) +- [Storage layout](#storage-layout) +- [The catalog](#the-catalog) +- [How a host finds a package](#how-a-host-finds-a-package) +- [Older hosts](#older-hosts) + +## What a package contains + +A package is a zip. At its root: + +- the adapter's runnable files; +- its dependencies, so that installing it fetches nothing; +- `adapter.json`, the full [manifest](manifest.md); +- `source/`, the source it was built from, unless built with `--no-source`. + +`sw-serverless build` writes the unpacked package to `bin/serverless/package/` and the zip to +`bin/serverless/{id}-{version}.zip` in the project (or under `--out`). + +## .NET + +The build runs `dotnet publish -c Release` and packages its output. The adapter must reference +`SimplyWorks.Serverless.Sdk` 10.1.0 or later, because the build asks the built adapter to describe +itself with `--describe`. Dependencies come from NuGet as usual and are part of the publish output. + +``` +acme.orders-1.2.0.zip +├── AcmeOrders.dll entry +├── AcmeOrders.deps.json +├── AcmeOrders.runtimeconfig.json +├── SW.Serverless.Sdk.dll and the other dependencies +├── adapter.json +└── source/ + ├── AcmeOrders.csproj + ├── Program.cs + ├── packages.lock.json + └── adapter.json +``` + +Projects the adapter references with `` are carried in `source/` too, so the +source rebuilds. The build warns when one lies outside the repository. + +## Python + +Python runs from source, so the package is the adapter's own files, as the [source rules](#source) +select them, plus: + +- `_vendor/sw_serverless/`: the SDK, copied from the copy the CLI carries. No PyPI access is + needed for it. +- `_vendor/...`: what `requirements.txt` names, installed by `pip` from wheels. A line naming the + SDK (`sw-serverless`) is skipped. +- `_serverless_entry.py`: a small script the build writes. It puts `_vendor` (and the folder for + this platform, if there is one) on the import path and runs your entry script. It becomes the + manifest's `entry`. + +``` +acme.orders-1.2.0.zip +├── _serverless_entry.py entry +├── main.py +├── requirements.txt +├── _vendor/ +│ ├── sw_serverless/ +│ └── requests/ … pure-Python requirements +├── adapter.json +└── source/ +``` + +**Native code.** The build downloads wheels only (`--only-binary=:all:`), for CPython 3.12, and +never compiles anything. If every requirement is pure Python, it is installed once into `_vendor/` +and the package runs anywhere. If any requirement has native code, every requirement is installed +once per target platform, into `_vendor//`, and the manifest's `platforms` lists those +platforms, so a host on any other refuses the package instead of failing on import. The targets +are `linux-x64` and `linux-arm64` unless `adapter.json` names others in `platforms`. Supported +targets are `linux-x64`, `linux-arm64`, `osx-x64`, `osx-arm64` and `win-x64`. A requirement with no +wheel for a target fails the build. + +The build machine needs `python3` with `pip`, and network access to your package index when +`requirements.txt` names anything besides the SDK. + +## Node and TypeScript + +Node runs JavaScript from source, so the package is the adapter's own files, plus: + +- `node_modules/`, from `npm ci` (when `package-lock.json` exists) or `npm install`, with + `--omit=dev` and `--ignore-scripts`. Install scripts of dependencies are not run. + `devDependencies` are left out. A `@simplyworks/sw-serverless` entry is removed before `npm` runs. +- `node_modules/@simplyworks/sw-serverless/`: the SDK, copied from the copy the CLI carries. + +**TypeScript.** Each `.ts`, `.mts` or `.cts` file (except `.d.ts`) is turned into `.js`, `.mjs` or +`.cjs` by Node's own type stripping, and the TypeScript file is removed from the package (it stays +in `source/`). Nothing is compiled: syntax that cannot simply be erased, such as `enum`, +`namespace` or constructor parameter properties, fails the build with its file and line. This needs +Node 22.13 or later on the build machine. An entry of `main.ts` becomes `main.js` in the manifest. + +**Native addons.** A dependency with a native addon (`*.node`) is built by `npm` for the machine +it runs on, and cannot be built for another platform from there. The build refuses it unless +`adapter.json` has `"platforms"` set to exactly the platform you are building on; build on the +platform the adapter will run on. + +``` +acme.orders-1.2.0.zip +├── main.js entry (from main.ts) +├── package.json +├── node_modules/ +│ ├── @simplyworks/sw-serverless/ +│ └── … +├── adapter.json +└── source/ +``` + +## exec + +An `exec` adapter is a self-contained executable that speaks [protocol 2](protocol.md): a Go or +Rust program, a native-compiled .NET program, or anything else. The host starts the entry file +itself. `sw-serverless build` does not build `exec` adapters; you make the zip yourself, with the +binary and an `adapter.json`: + +```json +{ + "id": "acme.fast-parser", + "version": "1.0.0", + "runtime": "exec", + "lifecycle": "resident", + "protocol": { "min": 2, "max": 2 }, + "entry": "fast-parser", + "platforms": ["linux-x64"] +} +``` + +To carry one binary per platform, list them in `entries`: + +```json +{ + "id": "acme.fast-parser", + "version": "1.0.0", + "runtime": "exec", + "lifecycle": "classic", + "protocol": { "min": 2, "max": 2 }, + "entry": "linux-x64/fast-parser", + "platforms": ["linux-x64", "linux-arm64", "osx-arm64"], + "entries": { + "linux-x64": "linux-x64/fast-parser", + "linux-arm64": "linux-arm64/fast-parser", + "osx-arm64": "osx-arm64/fast-parser" + } +} +``` + +Then check and publish it like any other package: + +```sh +(cd package && zip -r ../acme.fast-parser-1.0.0.zip .) +sw-serverless test acme.fast-parser-1.0.0.zip +sw-serverless publish acme.fast-parser-1.0.0.zip -p s3 -b my-adapters +``` + +The host sets the execute bit on the entry when it installs the package on Linux and macOS. For +`sw-serverless test` to pass, the binary must also answer `--describe` +([format](protocol.md#describe)). + +## Source + +Unless you pass `--no-source`, the build copies the project's source into the package under +`source/` and records each file's SHA-256 in the manifest's `source.files`. Anyone holding a +published version can then read it, compare two versions, or rebuild it. Hosts never unpack +`source/`. + +Which files are carried: + +1. Everything in the project folder (and, for .NET, the folders of referenced projects), +2. minus a built-in list that is always left out: + - build output and dependencies: `bin/`, `obj/`, `dist/`, `build/`, `out/`, `target/`, + `__pycache__/`, `*.pyc`, `.venv/`, `venv/`, `node_modules/`, `.pytest_cache/`, + `.mypy_cache/`, `packages/`, `*.nupkg`; + - version control and editors: `.git/`, `.vs/`, `.idea/`, `.vscode/`, `*.user`, `*.suo`, + `.DS_Store`; + - files that usually hold secrets: `.env`, `.env.*`, `*.pem`, `*.key`, `*.pfx`, `*.p12`, `*.jks`, + `id_rsa*`, `id_ed25519*`, `id_ecdsa*`, `secrets.json`, `*.publishsettings`, `*.pubxml`, + `*.pubxml.user`, `appsettings.*.json`; +3. minus what the project's `.gitignore` and `.serverlessignore` exclude, with gitignore's rules + (`*`, `**`, a trailing `/` for folders, a leading `/` to anchor, `!` to bring a file back). + +For Python and Node, the same rules decide which of your files go into the package to run. + +Before building, every text file carried is scanned for things that look like secrets: private +keys, cloud and service tokens (AWS, GitHub, Slack, Google, Stripe, Azure storage keys), passwords +in connection strings, JSON web tokens, and secrets assigned in code. Obvious placeholders such as +`changeme` or `${API_KEY}` are ignored. A finding fails the build and names the file and line. If it +is a false positive, pass `--allow `. Use `--dry-run` to see what would be carried. + +The build warns when the source is larger than 10 MB, which usually means build output slipped in. + +## Storage layout + +Everything is under one root folder in the bucket, `adapters` unless a deployment uses another +(`ServerlessOptions.AdapterRemotePath` on the host). + +| Key | What | Written by | +|---|---|---| +| `adapters/{id}` | The current package of a .NET adapter. What hosts from before versions run. | `publish` and `promote` of a .NET adapter (unless `--no-promote`); the original publishing form | +| `adapters-versions/{id}/{version}` | One package per version. Never overwritten. | `publish`; the original form with `-v` | +| `adapters-catalog/{id}.json` | The catalog entry. | Every command except `versions` | +| `adapters/{id}/{version}` | Where publishing tools from before this layout put versions. | Read only, never written | + +Ids and versions in keys are lowercase. + +Each package object carries storage metadata, which older hosts read: + +| Metadata | Value | +|---|---| +| `EntryAssembly` | The entry file. | +| `Hash`, `Sha256` | The hex SHA-256 of the zip. Hosts name the unpacked folder after `Hash`. | +| `Lifecycle` | `classic` or `resident`. | +| `Kind` | The kinds, comma separated. | +| `Version` | The version, or empty for an unversioned upload. | +| `Timestamp` | When it was uploaded, UTC. | +| `Lang` | `dotnet`. | + +Versions and the catalog are kept beside `adapters/`, not inside it, for two reasons: older hosts +and applications treat every key under `adapters/` as an adapter, and storage backed by a file +system cannot hold `adapters/{id}` as a file and a folder at once. + +## The catalog + +`adapters-catalog/{id}.json` describes everything published of one adapter, so an application can +list adapters, their versions and their settings without opening a package: + +```json +{ + "catalogVersion": 1, + "id": "greeter", + "current": "1.1.0", + "manifest": { "…": "the current package's manifest" }, + "sha256": "421ff604eef7abe8c288ab3f04e2914a801df2e0a590953946a89217a86ce5e2", + "iconDataUri": "data:image/png;base64,…", + "updatedOn": "2026-10-09T15:24:57.854628+00:00", + "versions": [ + { + "version": "1.0.0", + "sha256": "0b7d14e6c2a9…", + "publishedOn": "2026-09-01T10:00:00+00:00", + "publishedBy": "ada", + "manifest": { "…": "that version's manifest" }, + "withdrawn": false + }, + { + "version": "1.1.0", + "sha256": "421ff604eef7…", + "publishedOn": "2026-10-09T15:24:57.815776+00:00", + "publishedBy": "ci-bot", + "manifest": { "…": "…" }, + "withdrawn": false + } + ] +} +``` + +- `current` is null after an unversioned upload (the original form without `-v`): an unversioned + package is running. +- `versions` is oldest first. A record never changes once written, except `withdrawn`. +- `iconDataUri` is set when the icon is a PNG, JPEG or SVG of 64 KB or less. + +Read it from .NET with `AdapterCatalogStore` (in `SimplyWorks.Serverless.Contract`): + +```csharp +using SW.Serverless.Contract.Catalog; + +var catalog = new AdapterCatalogStore(cloudFiles); // root "adapters" +foreach (var entry in await catalog.ListAsync()) + Console.WriteLine($"{entry.Id} {entry.Current}: {entry.Manifest?.DisplayName}"); + +var one = await catalog.GetAsync("greeter"); // null if it has no entry +var pinnable = one.Versions.Where(v => !v.Withdrawn).Select(v => v.Version); +``` + +An adapter published only by tools from before the catalog has no entry until its next publish or +promote, which create one from the packages and their metadata. + +`AdapterCatalogPaths` builds the keys: `Current(root, id)`, `Version(root, id, version)`, +`Catalog(root, id)`, and `Ref(id, version)` / `Split(ref)` for pinned ids. + +## How a host finds a package + +| Asked for | The host reads | +|---|---| +| `greeter` | `adapters/greeter`; if there is none (Python, Node, `exec`), the catalog's `current` version at `adapters-versions/greeter/{current}` | +| `greeter/1.4.0` | `adapters-versions/greeter/1.4.0`; if there is none, `adapters/greeter/1.4.0` | + +It then unpacks the package (without `source/`) and checks its manifest, as described in +[Hosting adapters](hosting.md#how-installation-works). + +## Older hosts + +Hosts on `SimplyWorks.Serverless` 10.0.x, and applications that list adapters by reading +`adapters/`, predate manifests, versions and the catalog. What is published stays usable by them: + +- `adapters/{id}` always holds the current .NET package, with the metadata above, after every + publish, promote or rollback. +- Nothing is written under `adapters/` except `adapters/{id}`. +- A Python, Node or `exec` package is never written to `adapters/{id}`, because an older host would + start it with `dotnet`. Such adapters are invisible to older hosts. This is also why they must be + published with a version, and why an id once used for a .NET adapter cannot switch runtime. +- An older host asked for `{id}/{version}` looks only at `adapters/{id}/{version}`, so it cannot + pin versions published in the current layout. + +More in [Compatibility](compatibility.md). diff --git a/docs/protocol.md b/docs/protocol.md new file mode 100644 index 0000000..de324d0 --- /dev/null +++ b/docs/protocol.md @@ -0,0 +1,350 @@ +# The protocol + +This page describes how a host and an adapter talk, in enough detail to write an SDK in another +language, or an `exec` adapter that speaks the protocol directly. Adapter authors using the .NET, +Python or Node SDK do not need it. + +The message definitions are in +[`SW.Serverless.Contract/Protos/adapter.proto`](../SW.Serverless.Contract/Protos/adapter.proto) +(package `sw.serverless.v1`). This page explains how they are used. + +- [Overview](#overview) +- [Starting: the stdin handshake](#starting-the-stdin-handshake) +- [The stream](#the-stream) +- [Hello and Ready](#hello-and-ready) +- [Frames](#frames) +- [Encoding](#encoding) +- [Errors](#errors) +- [Stopping](#stopping) +- [Classic sessions on protocol 2](#classic-sessions-on-protocol-2) +- [Describe](#describe) +- [Protocol 1](#protocol-1) +- [Checklist for a new SDK](#checklist-for-a-new-sdk) + +## Overview + +``` +host adapter process + | start process (stdin, stdout, stderr piped) | + | stdin: {"socket": ..., "token": ...}\n -------->| + | | connect to the socket + |<------------- gRPC Attach stream opened --------| + |<------------- Hello (token, commands, ...) -----| + |------------- Ready (settings, max in flight) -->| + |------------- Invoke #1 ------------------------>| + |<------------ InvokeResult #1 -------------------| + |------------- Ping #2 -------------------------->| + |<------------ Pong #2 ---------------------------| + |<------------ Event #1 (adapter's id) -----------| + |------------- EventAck #1 ---------------------->| + |------------- Shutdown ------------------------->| + | | finish, close the stream, exit +``` + +This is **protocol 2**. The host speaks versions 2 to 2 today. **Protocol 1**, the classic text +protocol, is used only by .NET adapters on `Runner.Run`; see [Protocol 1](#protocol-1). + +## Starting: the stdin handshake + +The host starts the adapter as a child process, with its working directory set to the folder of +the entry file: + +| Runtime | Command | +|---|---| +| `dotnet` | `dotnet ` | +| `python` | `python3 -u ` (with `PYTHONUNBUFFERED=1`) | +| `node` | `node [--max-old-space-size=] ` | +| `exec` | `` | + +Immediately after starting it, the host writes **one line of JSON** to the adapter's standard +input: + +```json +{"socket":"/tmp/swsl-4182.sock","pipe":null,"token":"8c0e0b0f6a7c4e5f9f4d2a2c7b1e9d33","adapterId":"acme.orders","instanceKey":"main","protocol":2} +``` + +| Field | Meaning | +|---|---| +| `socket` | The Unix domain socket to connect to (Linux, macOS). | +| `pipe` | The named pipe to connect to (Windows), as a pipe name. Exactly one of `socket` and `pipe` is set. | +| `token` | A one-time token. Send it back in Hello. | +| `adapterId` | The adapter id, possibly with a pinned version (`acme.orders/1.2.0`). Echo it in Hello. | +| `instanceKey` | Which instance this process is. Echo it in Hello. | +| `protocol` | The newest protocol version the host speaks. | + +Then: + +- **Keep reading standard input.** The host keeps it open while it lives. End of file means the host + has gone: stop and exit, rather than running on as an orphan. +- **Standard output and standard error** are not a channel. The host keeps the last lines (200 by + default) and shows them when the adapter fails to connect or exits unexpectedly. Write startup + failures there. +- Nothing secret is on the command line: settings arrive later, over the socket. + +## The stream + +Connect to the socket or pipe and speak **HTTP/2 without TLS** (prior knowledge, no upgrade). The +host serves one gRPC method: + +```proto +service AdapterHost { + rpc Attach (stream AdapterFrame) returns (stream HostFrame); +} +``` + +Open one call to `POST /sw.serverless.v1.AdapterHost/Attach` with `content-type: application/grpc` +and `te: trailers`. Each message in either direction is the gRPC framing: one byte `0` (not +compressed; compression is not used), a 4-byte big-endian length, then the protobuf-encoded +message. A message may be up to 64 MB. The `:authority` is not checked; the Python and Node SDKs +use `localhost`. + +The adapter has to connect, open the stream and send Hello within the host's handshake timeout (30 +seconds by default), or the host kills it. + +Everything travels on this one stream, in both directions, interleaved. The adapter must keep +reading frames while commands run: a command that takes a minute must not stop pings being +answered, or the host will restart the adapter. Writes to the stream must not interleave; use one +writer. + +## Hello and Ready + +The **first frame** the adapter sends must be `Hello`: + +| Field | Meaning | +|---|---| +| `token` | From the handshake. An unknown or already used token ends the call with `PERMISSION_DENIED`. | +| `adapter_id`, `instance_key` | From the handshake. | +| `protocol_version` | The version the adapter will use: the lower of the handshake's `protocol` and the newest it speaks. Outside the host's range, the call ends with `FAILED_PRECONDITION`. | +| `sdk_version`, `sdk_language` | Your SDK's version and language (`dotnet`, `python`, `node`, `go`, ...). | +| `capabilities` | Strings: `command:` for every command; `resident` if it has a start hook; `resettable` if it handles Reset meaningfully; `cancel` if it understands the Cancel frame. | +| `commands` | A `CommandInfo` per command: `name`, `description`, `returns_value`, `input_schema` and `output_schema` (JSON Schema documents as strings, empty when none), and `parameter_type` / `parameter_schema` (a type name, and a .NET-style property map; may be left empty by other languages). | +| `settings` | A `SettingInfo` per declared setting: `name`, `description`, `required`, `secret`, `default_value` (empty for none), `type` (`text`, `multiline`, `number`, `boolean`, `select`, `json`). | +| `kinds` | The kinds it implements. | +| `contracts` | The contracts it implements: name to version. | + +A first frame that is not Hello ends the call with `INVALID_ARGUMENT`. + +The host answers with `Ready`: + +| Field | Meaning | +|---|---| +| `max_in_flight` | How many events the adapter may have waiting for an EventAck at once. | +| `startup_values` | The settings, name to value. Includes `CorrelationId` for classic sessions. | +| `adapter_values` | Extra values from the package's storage metadata and the host. | + +After Ready the host considers the instance ready and may send Invoke at once. An SDK should build +the adapter and run its start hook on Ready, before running commands, and must keep reading frames +meanwhile or queue the Invokes that arrive. + +## Frames + +Every frame has an `id` (int64) and a `traceparent` (W3C trace context, may be empty), and one body. + +`HostFrame` bodies, host to adapter: + +| Body | Reply | Meaning | +|---|---|---| +| `Ready` | none | See above. Sent once. | +| `Invoke` | `InvokeResult` with the same `id` | Run a command. | +| `Cancel` | none | The host has given up on the Invoke with this `id`. Sent only to adapters with the `cancel` capability. | +| `Ping` | `Pong` with the same `id` | Heartbeat. | +| `Reset` | `InvokeResult` with the same `id` | End a session. | +| `SetLogLevel` | none | `level`: the lowest log level to send, 0 (Trace) to 5 (Critical). | +| `Shutdown` | none (close the stream and exit) | `reason`, `drain`. | +| `EventAck` | none | The answer to the adapter's Event with this `id`. | +| `StateResult` | none | The answer to the adapter's StateRequest with this `id`. | + +`AdapterFrame` bodies, adapter to host: + +| Body | Reply | Meaning | +|---|---|---| +| `Hello` | `Ready` | First frame. | +| `InvokeResult` | none | The answer to an Invoke or a Reset, with its `id`. | +| `Pong` | none | The answer to a Ping, with its `id`. | +| `Event` | `EventAck` with the same `id` | Hand an event to the host. | +| `StateRequest` | `StateResult` with the same `id` | Get, set or delete a piece of state. | +| `LogEntry` | none | A log line. `id` 0. | +| `Metric` | none | A metric value. `id` 0. | + +Ids for frames the host starts (Invoke, Ping, Reset) come from the host's counter. Ids for frames +the adapter starts (Event, StateRequest) come from the adapter's own counter. The two do not +collide because each side matches replies only against its own pending requests. + +**Invoke** + +| Field | Meaning | +|---|---| +| `command` | The command name. | +| `payload` | The argument, [encoded](#encoding). Empty for none. | +| `timeout_seconds` | How long the host will wait. The adapter may cancel its own work after it. | +| `session_id` | Groups several calls into one session (a pooled lease). Empty means the call stands alone. | +| `properties` | Per-call values, read like settings; they win over startup values of the same name. | + +Several Invokes may be in flight at once. Answer each with `InvokeResult { payload }` or +`InvokeResult { error { type, message, detail } }`, in any order. If the host has timed out or +cancelled the call, it discards a late answer. + +**Ping / Pong.** The host pings every heartbeat (15 seconds by default) and restarts the adapter +after three unanswered pings. Answer quickly, even while busy. `Pong` fields: `connected`, `state` +(free text), `last_message_unix_ms`, `in_flight`, `last_error`, `details` (string map). + +**Reset.** Answer with an `InvokeResult` with an empty payload once per-session state for +`session_id` is gone, or with an error. The host waits up to 15 seconds before handing the +process to another session. + +**Event / EventAck.** `Event` fields: `payload` (bytes), `dedupe_key`, `content_type`, `headers` +(string map), `endpoint`. The adapter waits for `EventAck { accepted, reference, error }`, and +acknowledges its own source only if `accepted`. It must not have more than `max_in_flight` events +unanswered at once. A `traceparent` on the Event frame is passed to the host's event sink. + +**StateRequest / StateResult.** `StateRequest { op, name, value }` with `op` `GET` (0), `SET` (1) +or `DELETE` (2); `value` is used by SET only. `StateResult { found, value, error }`: `found` is false +for a GET of a name with nothing stored. An `error` means the host could not do it; report it to +the adapter's code rather than ignoring it. + +**LogEntry.** `level` (0 Trace, 1 Debug, 2 Information, 3 Warning, 4 Error, 5 Critical), `message`, +`exception`, `properties` (string map), `timestamp_unix_ms`. Logs and metrics may be dropped under +load; never let them hold up results, pongs or events. + +**Metric.** `name`, `value` (double), `tags` (string map). The host adds the value to a counter. + +## Encoding + +Payloads (Invoke arguments, InvokeResult results, Event payloads) are bytes. Every SDK uses the +same rule: + +| Value | Bytes | +|---|---| +| A string | Its UTF-8 text, not a JSON string. | +| Bytes | As they are. | +| Nothing | Empty. | +| Anything else | JSON (UTF-8). | + +The .NET host serializes objects with Newtonsoft.Json, keeping property names as declared, and +reads a result as raw text when the caller asks for a string, otherwise as JSON. An SDK should do +the same: decode the argument according to the command's declared input type, and encode the result +the same way. + +## Errors + +An `Error` has `type`, `message` and `detail`. + +- `type` is the error's type as the adapter names it: an exception's full type name in .NET, a + qualified class name in Python, a constructor name in Node, or a type the author chose. Hosts show + it to callers as `AdapterInvocationException.AdapterExceptionType`. +- `message` is for people. +- `detail` is a stack trace or similar, for logs. + +Conventions: an unknown command fails with type `MissingMethodException` (or +`System.MissingMethodException`), and a cancelled call with `OperationCanceledException`. + +## Stopping + +`Shutdown { reason, drain }` asks the adapter to stop. With `drain`, it should stop taking new work, +let commands already running finish and send their results, and exit, within 30 seconds; without, +within 5. Run the adapter's stop hook first, then wait for running commands, then send what is +still queued, close the stream and exit. The host kills the process after the deadline. + +The host sends `Shutdown` with `drain` when it stops an instance by request (by default), when an +adapter crosses its soft memory or sustained CPU limit, and when the host itself stops. + +## Classic sessions on protocol 2 + +A Python, Node or `exec` adapter, or a .NET adapter on protocol 2, is also run for classic sessions +through `IServerlessService`. Underneath, the host: + +1. starts a process for the session with instance key `classic-`; +2. sends Ready with the session's settings plus `CorrelationId`; +3. sends one Invoke at a time, waiting for each answer; +4. sends `Shutdown` with `drain` false when the session is disposed. + +An adapter needs nothing special for this. `GetExpectedStartupValues()` answers from the `settings` +in Hello. + +## Describe + +Every SDK must handle the `--describe` command-line argument: print one JSON document to standard +output and exit with code 0, without reading the handshake or connecting anywhere. Tools run it +with standard input closed, and give up after 30 seconds. + +```json +{ + "describeVersion": 1, + "sdkLanguage": "python", + "sdkVersion": "10.2.0", + "lifecycle": "classic", + "protocol": { "min": 2, "max": 2 }, + "settings": [ + { "name": "Greeting", "description": "What to say before the name.", "type": "text", "required": false, "secret": false, "default": "Hello" } + ], + "commands": [ + { + "name": "Add", + "description": "Adds two numbers.", + "inputSchema": { "type": "object", "properties": { "A": { "type": "integer" }, "B": { "type": "integer" } }, "required": ["A", "B"] }, + "outputSchema": { "type": "integer" }, + "returnsValue": true + } + ], + "kinds": [], + "contracts": {}, + "warnings": [] +} +``` + +| Field | Meaning | +|---|---| +| `describeVersion` | `1`. | +| `sdkLanguage`, `sdkVersion` | As in Hello. `sdkLanguage` must be set. | +| `lifecycle` | `classic` or `resident`: how the adapter's entry point runs it. | +| `protocol` | `{ "min": 2, "max": 2 }` for protocol 2. .NET classic adapters report `{ "min": 1, "max": 1 }`. | +| `settings` | `name`, `description`, `type`, `required`, `secret`, `default`. Leave out a secret's default. | +| `commands` | `name`, `description`, `inputSchema` (JSON Schema, or null for no argument), `outputSchema` (null for no result), `returnsValue`. | +| `kinds`, `contracts` | As in Hello. | +| `warnings` | Anything that made the description incomplete, such as an adapter that could not be built without its settings. | + +Unknown fields are kept by readers, so an SDK may add its own. `sw-serverless build` writes the +manifest from this; the conformance kit compares it with the manifest. + +## Protocol 1 + +Protocol 1 is the original .NET text protocol, used by `Runner.Run`. Hosts use it only for .NET +adapters whose manifest has no `protocol` of 2 or more; it is not available to other languages. +In brief: + +- The host starts `dotnet --values-on-stdin` and writes three lines on standard input, each + base64-encoded JSON: host options, startup values, adapter values. (Adapters on SDKs older than + 10.1.0 get the same three values as command-line arguments instead.) +- A call is one line, `#!##!##!#`, with newlines in the argument written as + `{{newline}}` and a null argument as `{{null}}`. The answer is one line on standard output, + `#!##!#`, or `{{error}}`. +- `{{expected}}` as the command asks for the declared settings; `{{quit}}` ends the process. +- Log lines go to standard error, starting with `{{log.information}}`, `{{log.warning}}` or + `{{log.error}}`. +- One call at a time. A timed-out call kills the process. + +New SDKs should implement protocol 2. + +## Checklist for a new SDK + +1. On `--describe`, print the [description](#describe) and exit 0. +2. Otherwise read one line from standard input and parse the handshake. Refuse a `protocol` lower + than 2. +3. Keep watching standard input; on end of file, stop. +4. Connect to `socket` (or `pipe`), open `Attach`, send Hello with the token and your commands, + settings, kinds, contracts and capabilities. +5. On Ready, store the settings, build the adapter, run its start hook without blocking the frame + reader. +6. Run each Invoke on its own task; answer with its id; apply `properties` and `session_id` to that + call only; encode as above; report errors with a type. +7. Answer Ping with Pong promptly, from the adapter's status hook if it has one. +8. Answer Reset with an InvokeResult. +9. Send Events and StateRequests with your own ids, wait for the matching reply, and respect + `max_in_flight`. +10. Send logs and metrics on a droppable queue, after results and pongs. +11. On Cancel, cancel the call's work (if you listed the `cancel` capability). +12. On Shutdown, run the stop hook, let running commands finish within the deadline, flush, close, + exit. + +The Python SDK (`sdk/python/src/sw_serverless`) implements all of this with the standard library +alone, including HTTP/2 and protobuf; it is a compact reference. diff --git a/docs/writing-adapters.md b/docs/writing-adapters.md new file mode 100644 index 0000000..f95c5d6 --- /dev/null +++ b/docs/writing-adapters.md @@ -0,0 +1,619 @@ +# Writing adapters + +This page is for developers writing adapters. Every topic shows .NET, Python and Node +(JavaScript or TypeScript) side by side. Read [Concepts](concepts.md) first if the words setting, +command, classic or resident are new. + +- [The SDKs](#the-sdks) +- [A first adapter](#a-first-adapter) +- [Settings](#settings) +- [Commands](#commands) +- [Encoding](#encoding) +- [Errors](#errors) +- [Logging](#logging) +- [Resident adapters](#resident-adapters) +- [The adapter context: events, state, metrics](#the-adapter-context-events-state-metrics) +- [Kinds and contracts](#kinds-and-contracts) +- [Dependency injection in .NET](#dependency-injection-in-net) +- [Describe](#describe) +- [Building, testing and running locally](#building-testing-and-running-locally) + +## The SDKs + +| Language | SDK | Requires | +|---|---|---| +| C#, F#, VB | NuGet `SimplyWorks.Serverless.Sdk`, namespace `SW.Serverless.Sdk` | .NET 10. Version 10.1.0 or later for `sw-serverless build`. | +| Python | `sw-serverless`, `import sw_serverless as sw` | Python 3.12 or later. | +| JavaScript, TypeScript | `@simplyworks/sw-serverless` | Node 22 or later. Building TypeScript needs Node 22.13 or later. | + +The Python and Node SDKs are not on PyPI or npm yet. `sw-serverless build` copies the SDK into the +package itself, so a build does not need them there. They will be published to PyPI and npm later. +Until then, for your editor: + +- Python: `export PYTHONPATH=/sdk/python/src`. +- Node: `npm install --no-save /sdk/node`. A plain `npm install` of a + project made by `sw-serverless init` fails, because its `package.json` names the SDK, which the + npm registry does not have yet. The build leaves the SDK out when it installs dependencies. + +`sw-serverless init --lang dotnet|python|node|typescript` writes a working project to start +from. See [The CLI](cli.md#init). + +## A first adapter + +A classic adapter with one setting and two commands: one takes text, one takes a JSON object. + +**.NET** + +```csharp +using SW.Serverless.Sdk; + +public class Numbers +{ + public int A { get; set; } + public int B { get; set; } +} + +public class Greeter +{ + public Greeter() + { + Runner.Expect("Greeting", "Hello", description: "What to say before the name."); + } + + [AdapterCommand("Greets someone by name.")] + public Task Greet(string name) => + Task.FromResult($"{Runner.StartupValueOf("Greeting")}, {name}!"); + + [AdapterCommand("Adds two numbers.")] + public Task Add(Numbers numbers) => Task.FromResult(numbers.A + numbers.B); +} + +static class Program +{ + static Task Main() => Runner.Run(new Greeter()); +} +``` + +**Python** + +```python +from dataclasses import dataclass + +import sw_serverless as sw + + +@dataclass +class Numbers: + A: int + B: int + + +class Greeter: + def __init__(self): + sw.expect("Greeting", "Hello", description="What to say before the name.") + + @sw.command("Greet", description="Greets someone by name.") + def greet(self, name: str) -> str: + return f"{sw.value_of('Greeting')}, {name}!" + + @sw.command("Add", description="Adds two numbers.") + def add(self, numbers: Numbers) -> int: + return numbers.A + numbers.B + + +if __name__ == "__main__": + sw.run(Greeter) +``` + +**JavaScript** + +```js +const sw = require("@simplyworks/sw-serverless"); + +class Greeter { + static commands = { + Greet: { method: "greet", input: "string", output: "string", description: "Greets someone by name." }, + Add: { + method: "add", + input: { type: "object", properties: { A: { type: "integer" }, B: { type: "integer" } }, required: ["A", "B"] }, + output: "json", + description: "Adds two numbers.", + }, + }; + + constructor() { + sw.expect("Greeting", { default: "Hello", description: "What to say before the name." }); + } + + greet(name) { + return `${sw.valueOf("Greeting")}, ${name}!`; + } + + add({ A, B }) { + return A + B; + } +} + +sw.run(Greeter); +``` + +**TypeScript** is the same code with types. Node removes the types at build time rather than +compiling, so only syntax that can simply be erased is allowed: no `enum`, no `namespace`, no +constructor parameter properties. The `tsconfig.json` that `init` writes sets +`erasableSyntaxOnly` so your editor flags them, and its `package.json` sets `"type": "module"` +so `import` works. + +```ts +import { expect, run, valueOf } from "@simplyworks/sw-serverless"; + +interface Numbers { A: number; B: number } + +class Greeter { + static commands = { + Greet: { method: "greet", input: "string", output: "string", description: "Greets someone by name." }, + Add: { method: "add", input: "json", output: "json", description: "Adds two numbers." }, + }; + + constructor() { + expect("Greeting", { default: "Hello", description: "What to say before the name." }); + } + + greet(name: string): string { + return `${valueOf("Greeting")}, ${name}!`; + } + + add(numbers: Numbers): number { + return numbers.A + numbers.B; + } +} + +run(Greeter); +``` + +A host calls them the same way whatever the language: + +```csharp +await serverless.StartAsync("greeter", correlationId, new Dictionary { ["Greeting"] = "Hi" }); +var text = await serverless.InvokeAsync("Greet", "Ada"); // "Hi, Ada!" +var sum = await serverless.InvokeAsync("Add", new { A = 2, B = 3 }); // 5 +``` + +## Settings + +Declare every setting the adapter reads, once, when the adapter is built. Read values inside +commands. + +| | .NET | Python | Node | +|---|---|---|---| +| Declare | `Runner.Expect(name, optional = false, isPrivate = false, description = null)` or `Runner.Expect(name, defaultValue, isPrivate = false, description = null)` | `sw.expect(name, default=None, *, required=None, secret=False, description=None, type="text")` | `expect(name, { default, required, secret, description, type })` | +| Read | `Runner.StartupValueOf(name)`, `Runner.StartupValueOf(name)` | `sw.value_of(name, default=None)` | `valueOf(name, fallback)` | +| All values | `Runner.StartupValues` | `sw.startup_values()` | `startupValues()` | + +- **Required.** In .NET a setting is required unless `optional: true` or it has a default. In Python + and Node it is required unless it has a default or `required` is false. +- **Secret** (`isPrivate` in .NET). Applications mask it, and its default is never written into the + manifest. +- **Default.** Returned when the host sends no value. +- **Type.** Python and Node accept `text`, `multiline`, `number`, `boolean`, `select` and `json`, to + tell an application what kind of field to show. Values always arrive as strings. .NET settings are + `text`; set another type for a .NET adapter in `adapter.json` (see [The manifest](manifest.md#properties)). +- **CorrelationId.** In a classic session the host also sends `CorrelationId`. In .NET it is also + `Runner.CorrelationId`. +- **Per-call values.** A resident instance may receive values with each command + (`properties` on the host's `InvokeAsync`). The read functions return a per-call value first, then + the startup value, then the default. + +**.NET: read settings in commands, not in the constructor.** With `Runner.Run(new Greeter())` the +handler is built before the host's values have been read, so a value read in the constructor is +only the declared default. Declare in the constructor, read in the commands. If the constructor +needs values, pass a factory, which is called after the values arrive: + +```csharp +static Task Main() => Runner.Run(() => new Greeter(Runner.StartupValueOf("Greeting"))); +``` + +**.NET resident adapters and declarations.** A resident adapter (and any .NET adapter on +`Runner.RunResident`) reports its settings to the host when it connects. With +`Runner.RunResident(new Handler())` the constructor has already run by then, so declarations in it +are reported. When the handler is built later, by `AdapterHost` (see +[Dependency injection](#dependency-injection-in-net)), make the `Runner.Expect` calls in `Main` +before running, or the host will not see them. `--describe`, and so the build, sees them either way. + +## Commands + +A command takes at most one argument and returns at most one result. + +**.NET.** Every public instance method that returns `Task` or `Task` and has at most one +parameter is a command, under its own name. Names are matched without regard to case. +`[AdapterCommand("...")]` adds a description; it is optional. Under `Runner.RunResident` a command +may also take a `CancellationToken` as its last parameter, and the methods `StartAsync`, +`StopAsync`, `GetStatusAsync` and `ResetAsync` are never commands. + +**Python.** A method marked `@sw.command("Name", description=...)` is a command. Without a name, +the method's own name is used: `@sw.command` alone works too. Commands may be `def` or `async def`; +plain `def` commands run on a worker thread, so a slow one does not stop the adapter answering the +host. The type hints of the argument and result decide how they are decoded and what schema +`--describe` reports. A dataclass argument is built from a JSON object. + +**Node.** Commands are listed in `static commands`, keyed by the name the host calls: + +```js +static commands = { + Name: { method: "methodName", input: "string", output: "json", description: "…" }, +}; +``` + +`input` and `output` are `"string"`, `"bytes"`, `"json"`, or a JSON Schema object (JSON described by +that schema). Leave `input` out for a command with no argument, and `output` out for one that +returns nothing. Methods may be async. Subclasses inherit and extend their base class's `commands`. + +In Python and Node, command names are matched exactly, including case. + +## Encoding + +Arguments and results cross the process boundary as bytes. The rule is the same in every SDK: + +| Value | On the wire | +|---|---| +| A string | Its raw UTF-8 text. Not a JSON string: `Ada`, not `"Ada"`. | +| Bytes (.NET `byte[]`, Python `bytes`, Node `"bytes"` / `Buffer`) | As they are. | +| Nothing (`null`, `None`, no argument, `Task`) | An empty payload. | +| Anything else | JSON. | + +JSON property names are written as your types name them: .NET uses the property names as declared +(`OrderId`, not `orderId`), Python uses dataclass field names, Node uses your object's keys. When an +application's [contract](contracts.md) fixes the names, match them exactly. + +Protocol 1 (.NET classic adapters on `Runner.Run`) differs slightly: numbers and booleans travel as +.NET writes them with `ToString()` (`True`, not `true`), and `byte[]` is sent as JSON (base64). + +## Errors + +Throw or raise to fail a command. The caller gets an error with a type, a message and a detail +(the stack trace). + +| | Fail with a type you choose | Type otherwise | +|---|---|---| +| .NET | Throw your own exception class: the type is its full name, e.g. `Acme.Orders.RejectedException`. | The exception's full type name. | +| Python | `raise sw.AdapterError("no stock", type="Acme.Rejected")` | The exception class's name, prefixed with its module unless it is a built-in or defined in the main script. | +| Node | `throw new sw.AdapterError("no stock", { type: "Acme.Rejected" })` | The error's constructor name, e.g. `TypeError`. | + +On the host, a protocol 2 error is an `AdapterInvocationException` whose `AdapterExceptionType` is +that type. A protocol 1 error is an `Exception` carrying the adapter's exception text. Either way the +adapter keeps running and can take the next command. + +A command the adapter does not have fails with the type `MissingMethodException` (Python and +Node) or `System.MissingMethodException` (.NET on protocol 2). + +## Logging + +| | How | Reaches the host as | +|---|---|---| +| .NET classic | `AdapterLogger.LogInformation(...)`, `LogWarning`, `LogError` | Log entries in `serverless.adapters.{id}` | +| .NET resident | `context.LogInformation(...)`, `LogWarning`, `LogError`, `Log(level, ...)`, or `ILogger` with [dependency injection](#dependency-injection-in-net) | The same, with structured properties | +| Python | The standard `logging` module | The same | +| Node | `sw.log.trace`, `debug`, `info`, `warn`, `error`, `critical` (`message`, optional `error`) | The same | + +- Do not write your own output to standard output. Protocol 1 uses it for results. +- In .NET resident adapters `AdapterLogger` writes to standard error, which the host keeps only for + crash reports. Use the context or `ILogger` there. +- Python's root logger level is set to `INFO` if it was unset or higher, so `logging.info(...)` is + sent by default. +- The host can change the lowest level an instance sends at run time. In .NET check + `context.MinimumLogLevel` before building an expensive message. +- Logs are dropped, not queued without limit, if the adapter writes them faster than they can be + sent. Command results and events are never dropped. + +## Resident adapters + +A resident adapter has a start hook, and optional stop, status and reset hooks. + +| Hook | .NET (`IResidentAdapter`, `IResettable`) | Python | Node | +|---|---|---|---| +| Start | `Task StartAsync(IAdapterContext context, CancellationToken ct)` | `start(self)` | `start()` | +| Stop | `Task StopAsync(CancellationToken ct)` | `stop(self)` | `stop()` | +| Status | `Task GetStatusAsync()` | `status(self)` returning a dict | `status()` returning an object | +| Reset | `Task ResetAsync(string sessionId)` | `reset(self, session_id)` | `reset(sessionId)` | +| Entry point | `Runner.RunResident(handler)` | `sw.run(Adapter)` (resident because it has `start`) | `run(Adapter)` (resident because it has `start`) | + +Python and Node hooks may be sync or async. + +- **Start** is called once the host has sent the settings. Open connections, start your own loop + on a task of its own, and return. Start may ask the host for things — read state, publish an + event — and commands wait until it has returned. If it fails, the adapter stops, and the host + restarts it as it does after a crash. A stop that arrives while start is still running waits for + it first, within the stop's own deadline. +- **Stop** is called when the host stops the adapter. Stop fetching, finish or hand back what is in + flight, close connections. The host waits up to 30 seconds when it asked for a drain, 5 otherwise. + Commands still running are allowed to finish within the same time. +- **Status** is called on every heartbeat and must return quickly. It reports `connected`, a free + text `state` such as `Connected`, `Idle` or `Disconnected`, the number of items in flight, the + last error, the time of the last message, and provider details as name-value pairs. Python keys: `connected`, `state`, + `in_flight`, `last_error`, `last_message_unix_ms`, `details`. Node keys: `connected`, `state`, + `inFlight`, `lastError`, `lastMessageOn`, `details`. +- **Reset** is for pooled adapters: forget anything kept for the session that ends. +- Commands work exactly as in classic adapters, and several may run at once. + +An example: an adapter that publishes a numbered tick every few seconds, keeps the count in host +state so it survives restarts, and reports it in its status. + +**.NET** + +```csharp +using System.Text; +using SW.Serverless.Sdk; +using SW.Serverless.Sdk.Resident; + +public class Ticker : IResidentAdapter +{ + IAdapterContext context; + CancellationTokenSource stopping; + Task loop; + long sent; + + public Ticker() => Runner.Expect("IntervalSeconds", "5", description: "Seconds between ticks."); + + public Task StartAsync(IAdapterContext context, CancellationToken cancellationToken) + { + this.context = context; + stopping = new CancellationTokenSource(); + loop = Task.Run(() => TickAsync(stopping.Token)); // not awaited: start must return + return Task.CompletedTask; + } + + async Task TickAsync(CancellationToken ct) + { + sent = long.Parse(await context.GetStateAsync("sent", ct) ?? "0"); + var interval = TimeSpan.FromSeconds(int.Parse(context.StartupValueOf("IntervalSeconds"))); + while (!ct.IsCancellationRequested) + { + await Task.Delay(interval, ct); + var number = sent + 1; + var result = await context.PublishAsync(Encoding.UTF8.GetBytes($"tick {number}"), + dedupeKey: $"tick-{number}", contentType: "text/plain", cancellationToken: ct); + if (!result.Accepted) continue; // not stored: send the same number again + sent = number; + await context.SetStateAsync("sent", sent.ToString(), ct); + } + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + stopping.Cancel(); + try { await loop; } catch (OperationCanceledException) { } + } + + public Task GetStatusAsync() => + Task.FromResult(new AdapterStatus { Connected = true, State = "Ticking", Details = { ["sent"] = sent.ToString() } }); + + [AdapterCommand("Ticks published since the adapter was first started.")] + public Task Sent() => Task.FromResult(sent); +} + +static class Program +{ + static Task Main() => Runner.RunResident(new Ticker()); +} +``` + +**Python** + +```python +import asyncio + +import sw_serverless as sw + + +class Ticker: + def __init__(self): + sw.expect("IntervalSeconds", "5", description="Seconds between ticks.") + self.sent = 0 + self.loop = None + + async def start(self): + self.loop = asyncio.create_task(self.tick(sw.context())) # not awaited: start must return + + async def tick(self, ctx): + self.sent = int(await ctx.get_state("sent") or 0) + interval = float(sw.value_of("IntervalSeconds")) + while True: + await asyncio.sleep(interval) + number = self.sent + 1 + try: + await ctx.publish(f"tick {number}", dedupe_key=f"tick-{number}", content_type="text/plain") + except sw.AdapterError: + continue # not stored: send the same number again + self.sent = number + await ctx.set_state("sent", str(self.sent)) + + async def stop(self): + self.loop.cancel() + + def status(self): + return {"connected": True, "state": "Ticking", "details": {"sent": str(self.sent)}} + + @sw.command("Sent", description="Ticks published since the adapter was first started.") + def sent_count(self) -> int: + return self.sent + + +if __name__ == "__main__": + sw.run(Ticker) +``` + +**Node** + +```js +const sw = require("@simplyworks/sw-serverless"); + +class Ticker { + static commands = { + Sent: { method: "sentCount", output: "json", description: "Ticks published since the adapter was first started." }, + }; + + constructor() { + sw.expect("IntervalSeconds", { default: "5", description: "Seconds between ticks." }); + this.sent = 0; + this.stopped = false; + } + + start() { + this.loop = this.tick(sw.context()); // not awaited: start must return + } + + async tick(ctx) { + try { + this.sent = Number((await ctx.getState("sent")) ?? 0); + const interval = Number(sw.valueOf("IntervalSeconds")) * 1000; + while (!this.stopped) { + await new Promise((resolve) => setTimeout(resolve, interval)); + const number = this.sent + 1; + try { + await ctx.publish(`tick ${number}`, { dedupeKey: `tick-${number}`, contentType: "text/plain" }); + } catch (e) { + continue; // not stored: send the same number again + } + this.sent = number; + await ctx.setState("sent", String(this.sent)); + } + } catch (e) { + if (!this.stopped) sw.log.error("ticking stopped", e); + } + } + + stop() { + this.stopped = true; + } + + status() { + return { connected: true, state: "Ticking", details: { sent: this.sent } }; + } + + sentCount() { + return this.sent; + } +} + +sw.run(Ticker); +``` + +### A .NET adapter on protocol 2, run as a classic session + +A .NET handler run with `Runner.RunResident` that does not implement `IResidentAdapter` speaks +protocol 2 but has nothing to keep running. Put `"lifecycle": "classic"` in its `adapter.json`, and +`sw-serverless build` marks it as a classic adapter on protocol 2: hosts run it one session at a +time, as they do Python and Node classic adapters. Such hosts need `AddResidentAdapters`. + +## The adapter context: events, state, metrics + +The context is what a protocol 2 adapter can reach beyond its argument. + +| | .NET | Python | Node | +|---|---|---|---| +| Get it | The `IAdapterContext` passed to `StartAsync`, or injected | `sw.context()` | `sw.context()` | +| Publish an event | `await context.PublishAsync(payload, dedupeKey, endpoint, headers, contentType, ct)` returns `PublishResult { Accepted, Reference, Error }` | `await ctx.publish(payload, dedupe_key=, content_type=, headers=, endpoint=)` returns the reference, raises `AdapterError` if rejected | `await ctx.publish(payload, { dedupeKey, contentType, headers, endpoint })` returns the reference, throws `AdapterError` if rejected | +| Read state | `await context.GetStateAsync(name)` (null if none) | `await ctx.get_state(name)` (None if none) | `await ctx.getState(name)` (null if none) | +| Write state | `await context.SetStateAsync(name, value)` | `await ctx.set_state(name, value)` | `await ctx.setState(name, value)` | +| Delete state | `await context.SetStateAsync(name, null)` | `await ctx.delete_state(name)` | `await ctx.deleteState(name)` | +| Record a metric | `context.Metric(name, value, tags)` | `ctx.metric(name, value, tags)` | `ctx.metric(name, value, tags)` | +| Read a value | `context.ValueOf(name)`, `context.InvocationValues` | `ctx.value_of(name)` | `ctx.valueOf(name)` | +| The call was abandoned | `context.CallCancelled`, or a `CancellationToken` parameter | `ctx.cancelled` (an `asyncio.Event`); an async command's task is also cancelled | `ctx.signal` (an `AbortSignal`) | +| The adapter is stopping | `context.Stopping` | `ctx.stopping` (an `asyncio.Event`) | `ctx.stopping` (an `AbortSignal`) | +| Identity | `context.AdapterId`, `context.InstanceKey` | `ctx.adapter_id`, `ctx.instance_key`, `ctx.session_id`, `ctx.command` | `ctx.adapterId`, `ctx.instanceKey`, `ctx.sessionId`, `ctx.command` | + +**Events.** Publish, wait for the result, and acknowledge your source (delete the file, ack the +message) only when the host has accepted it. If the host rejects it, or the adapter crashes first, +the source still has it and it will be delivered again, so give every event a `dedupeKey` that +identifies it: a message id, an offset, a file name and its hash. `endpoint` says where it came +from: a queue, a topic, a folder. A payload is encoded like a command result: a string as UTF-8, +bytes as they are, anything else as JSON (in .NET, pass bytes). One event may be up to about 63 MB. The +host lets an instance have a limited number of events waiting at once (`MaxInFlight`, 16 by +default); further publishes wait their turn. + +**State** is a small string per name, kept by the host for this adapter instance, and kept across +restarts. Use it for a bookmark, such as a cursor or the time of the last run, and write it only +once the work it records has been accepted. It is not a data store; the host may refuse a large +value, and the error reaches you. + +**Metrics** are added to a counter of that name on the host. + +**A synchronous Python command** cannot be stopped from outside. When the call is abandoned it +keeps running in its thread; check `ctx.cancelled` in long loops. + +## Kinds and contracts + +If an application's contract defines kinds, declare which you implement. Details and the full +example are in [Contracts](contracts.md). + +| | Declare | +|---|---| +| .NET | `[AdapterKind("processor")]` and `[AdapterContract("orders", 1)]` on the handler class | +| Python | `@sw.implements("orders", 1, "processor")` on the class | +| Node | `static kinds = ["processor"];` and `static contracts = { orders: 1 };` on the class | +| Any language | `"kinds"` and `"contracts"` in `adapter.json` | + +## Dependency injection in .NET + +For a .NET adapter bigger than one class, `AdapterHost` (namespace `SW.Serverless.Sdk.Hosting`) +builds a service container once the settings have arrived, so constructor injection of +configuration is safe: + +```csharp +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using SW.Serverless.Sdk.Hosting; + +static Task Main() => AdapterHost.CreateBuilder() + .ConfigureServices((configuration, services) => + { + services.Configure(configuration); // bound from the settings + services.AddSingleton(); + }) + .Build() + .RunResidentAsync(); // or .RunAsync() for classic +``` + +The container provides: + +- `IConfiguration`: the settings, then the package's storage metadata under `AdapterValues:`, then + environment variables that start with `SWSL_` (prefix removed). Settings win. +- `ILogger`, sent to the host's logs. +- `IAdapterContext`, under `RunResidentAsync`. +- The handler, as a singleton. + +`AdapterSession.Id` and `AdapterSession.Command` give the current call's session id and command +name anywhere in the call. + +Under `RunResidentAsync`, make the `Runner.Expect` calls in `Main`, before +`AdapterHost.CreateBuilder()`: the adapter reports its settings to the host before the container +builds the handler. See [Settings](#settings). + +## Describe + +Every SDK answers `--describe`: it prints a JSON description of the adapter and exits, without a +host. `sw-serverless build` uses it to write the manifest, and the conformance kit uses it to check +the adapter. Try it on a built package: + +```sh +dotnet bin/serverless/package/Greeter.dll --describe +python3 bin/serverless/package/_serverless_entry.py --describe +node bin/serverless/package/main.js --describe +``` + +To describe the adapter, the SDK builds it without settings. If the constructor fails without +them, the description is still printed, with a warning that settings declared later may be missing. +Keep constructors free of work that needs settings. The output format is in +[The protocol](protocol.md#describe). + +## Building, testing and running locally + +```sh +sw-serverless build # the package: bin/serverless/{id}-{version}.zip +sw-serverless test --settings settings.json # runs it as a host would and checks it +sw-serverless run --settings settings.json --call Add --input '{"A":2,"B":3}' +``` + +`test` and `run` start the adapter on this machine through a real host, so they need the adapter's +runtime installed. `settings.json` is a flat JSON object of setting names to values. Keep it out of +version control once it holds real credentials; the `.gitignore` that `init` writes already does. +See [The CLI](cli.md). diff --git a/sdk/node/README.md b/sdk/node/README.md index 9cb4e2f..85df596 100644 --- a/sdk/node/README.md +++ b/sdk/node/README.md @@ -36,6 +36,8 @@ sw.run(Greeter); and `reset(sessionId)`. `sw.context()` publishes events, keeps small state and records metrics; its `signal` aborts when the host gives up on a call. - **Logs** go to the host with `sw.log.info(...)` and the other levels. -- `node main.js --describe` prints what the adapter is; `serverless build` writes it into the manifest. +- `node main.js --describe` prints what the adapter is; `sw-serverless build` writes it into the manifest. -For Bitween adapters, `@simplyworks/bitween` has the four kinds ready to extend. Tests: `npm test`. +An application with a contract of its own declares it on the adapter class with `static kinds` and +`static contracts` (`{ orders: 1 }`), and can give its adapter authors base classes that do it for them. +`sw-serverless init --lang node` (or `typescript`) starts an adapter. Tests: `npm test`. diff --git a/sdk/python/README.md b/sdk/python/README.md index 8ae9681..2efa3ae 100644 --- a/sdk/python/README.md +++ b/sdk/python/README.md @@ -36,10 +36,11 @@ if __name__ == "__main__": `sw.context()` publishes events (`await ctx.publish(...)`), keeps small state (`get_state`/`set_state`/`delete_state`) and records metrics. - **Logging** through Python's `logging` reaches the host. -- `python main.py --describe` prints what the adapter is; `serverless build` writes it into the +- `python main.py --describe` prints what the adapter is; `sw-serverless build` writes it into the manifest. -For Bitween adapters, `simplyworks-bitween` has the four kinds — `Handler`, `Mapper`, `Validator`, -`Receiver` — ready to subclass. `serverless init --lang python --kind handler` starts one. +An application with a contract of its own declares it with `@sw.implements("orders", 1, "processor")` +on the adapter class, and can give its adapter authors base classes that do it for them. +`sw-serverless init --lang python` starts an adapter. Tests: `PYTHONPATH=src python -m unittest discover -s tests`. From c487ff851e899286772085b3d7b197f9cbb7d75a Mon Sep 17 00:00:00 2001 From: Samerz Date: Fri, 9 Oct 2026 19:52:32 +0300 Subject: [PATCH 17/17] Release sw-serverless for musl Linux on arm64 too --- .github/workflows/cli-release.yml | 4 ++-- README.md | 2 +- docs/cli.md | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml index f3a4a37..c2f8fb8 100644 --- a/.github/workflows/cli-release.yml +++ b/.github/workflows/cli-release.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - rid: [linux-x64, linux-arm64, linux-musl-x64, osx-x64, osx-arm64, win-x64] + rid: [linux-x64, linux-arm64, linux-musl-x64, linux-musl-arm64, osx-x64, osx-arm64, win-x64] steps: - uses: actions/checkout@v4 - uses: actions/setup-dotnet@v4 @@ -68,4 +68,4 @@ jobs: v="${{ github.event.inputs.version }}" [ -z "$v" ] && v="${GITHUB_REF_NAME#cli-v}" gh release create "cli-v$v" dist/* --title "sw-serverless $v" --target "${{ github.sha }}" \ - --notes "The sw-serverless command, self-contained for each platform. Install: curl -fsSL https://raw.githubusercontent.com/${{ github.repository }}/main/scripts/install-cli.sh | sh — or download the archive for your platform and check it against SHA256SUMS." + --notes "The sw-serverless command, self-contained for each platform. Install: curl -fsSL https://raw.githubusercontent.com/${{ github.repository }}/${{ github.event.repository.default_branch }}/scripts/install-cli.sh | sh — or download the archive for your platform and check it against SHA256SUMS." diff --git a/README.md b/README.md index d275ad1..dfbb7d6 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ published to PyPI and npm later. Self-contained binaries are published on the [GitHub releases](https://github.com/simplify9/SW-Serverless/releases) tagged `cli-v`, one -per platform: `sw-serverless-.tar.gz` for `linux-x64`, `linux-arm64`, `linux-musl-x64`, +per platform: `sw-serverless-.tar.gz` for `linux-x64`, `linux-arm64`, `linux-musl-x64`, `linux-musl-arm64`, `osx-x64` and `osx-arm64`, and `sw-serverless-win-x64.zip`, with a `SHA256SUMS` file. On Linux or macOS, the install script picks your platform, checks the download and installs to `~/.local/bin`: diff --git a/docs/cli.md b/docs/cli.md index bbb33c7..fa2bc3d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -22,6 +22,7 @@ Self-contained binaries are published on the | Linux x64 | `sw-serverless-linux-x64.tar.gz` | | Linux Arm64 | `sw-serverless-linux-arm64.tar.gz` | | Linux x64, musl (Alpine) | `sw-serverless-linux-musl-x64.tar.gz` | +| Linux Arm64, musl (Alpine) | `sw-serverless-linux-musl-arm64.tar.gz` | | macOS Intel | `sw-serverless-osx-x64.tar.gz` | | macOS Apple silicon | `sw-serverless-osx-arm64.tar.gz` | | Windows x64 | `sw-serverless-win-x64.zip` |