feat: gem v0.26.0 updates (drop browserless, capture, mcp, botasaurus improvements)

Gemfile

@@ -2,5 +2,5 @@
 
 source 'https://rubygems.org'
 
-gem 'html2rss', git: 'https://github.com/html2rss/html2rss.git'
+gem 'html2rss', '~> 0.26'
 gem 'html2rss-configs', git: 'https://github.com/html2rss/html2rss-configs.git'

Gemfile.lock

@@ -1,36 +1,10 @@
 GIT
   remote: https://github.com/html2rss/html2rss-configs.git
-  revision: d9e3dc55b38514eb9cd4a17cb0285d9ef73bc9d5
+  revision: ebda084dfdb87627091f0336208553204d073c7b
   specs:
     html2rss-configs (0.2.0)
       html2rss
 
-GIT
-  remote: https://github.com/html2rss/html2rss.git
-  revision: aa974733c6e88a5c934f445516d9752dfa316067
-  specs:
-    html2rss (0.25.0)
-      addressable (~> 2.7)
-      brotli
-      dry-validation
-      faraday (> 2.0.1, < 3.0)
-      faraday-follow_redirects
-      faraday-gzip (~> 3)
-      kramdown
-      mcp (~> 1.0)
-      mime-types (> 3.0)
-      nokogiri (>= 1.10, < 2.0)
-      rack (~> 3.0)
-      rackup (~> 2.0)
-      regexp_parser
-      reverse_markdown (~> 3.0)
-      rss
-      sanitize
-      thor
-      tzinfo
-      webrick (~> 1.9)
-      zeitwerk
-
 GEM
   remote: https://rubygems.org/
   specs:
@@ -87,6 +61,27 @@ GEM
     faraday-net_http (3.4.4)
       net-http (~> 0.5)
     hana (1.3.7)
+    html2rss (0.26.0)
+      addressable (~> 2.7)
+      brotli
+      dry-validation
+      faraday (> 2.0.1, < 3.0)
+      faraday-follow_redirects
+      faraday-gzip (~> 3)
+      kramdown
+      mcp (~> 1.0)
+      mime-types (> 3.0)
+      nokogiri (>= 1.10, < 2.0)
+      rack (~> 3.0)
+      rackup (~> 2.0)
+      regexp_parser
+      reverse_markdown (~> 3.0)
+      rss
+      sanitize
+      thor
+      tzinfo
+      webrick (~> 1.9)
+      zeitwerk
     json (2.21.2)
     json_schemer (2.5.0)
       bigdecimal
@@ -138,7 +133,7 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
-  html2rss!
+  html2rss (~> 0.26)
   html2rss-configs!
 
 BUNDLED WITH

astro.config.mjs

@@ -14,7 +14,7 @@ export default defineConfig({
     "/components/html2rss-configs": "/creating-custom-feeds/",
     "/components": "/",
     "/web-application/how-to/deployment": "/web-application/deployment/",
-    "/web-application/how-to/automatic-updates": "/web-application/deployment/#auto-update-with-watchtower",
+    "/web-application/how-to/automatic-updates": "/web-application/deployment/",
     "/web-application/how-to/use-automatic-feed-generation":
       "/web-application/guides/use-the-feed-directory/",
     "/web-application/how-to": "/web-application/guides/",

src/components/docs/DockerComposeSnippet.astro

@@ -1,9 +1,9 @@
 ---
 import { Code } from "@astrojs/starlight/components";
-import { botasaurusImage, caddyImage, watchtowerImage, webImage } from "../../data/docker";
+import { botasaurusImage, caddyImage, webImage } from "../../data/docker";
 
 interface Props {
-  variant: "minimal" | "productionCaddy" | "secure" | "watchtower" | "resourceGuardrails";
+  variant: "minimal" | "productionCaddy" | "secure" | "resourceGuardrails";
 }
 
 const { variant } = Astro.props;
@@ -78,15 +78,6 @@ volumes:
   botasaurus:
     image: ${botasaurusImage}
     restart: unless-stopped`,
-  watchtower: `services:
-  watchtower:
-    image: ${watchtowerImage}
-    restart: unless-stopped
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-      # Optional for private registries only:
-      # - "\${HOME}/.docker/config.json:/config.json:ro"
-    command: --cleanup --interval 7200 html2rss-web botasaurus caddy`,
   resourceGuardrails: `services:
   html2rss-web:
     image: ${webImage}

src/content/docs/creating-custom-feeds.mdx

@@ -31,11 +31,12 @@ When existing feeds or auto-sourcing are not enough, write a YAML config for the
 ## Recommended Workflow
 
 1. **Inspect the live page** in your browser developer tools
-2. **Write the smallest useful config** that extracts items, titles, and links
-3. **Validate the config** with `html2rss validate your-config.yml`
-4. **Render the feed** with `html2rss feed your-config.yml`
-5. **Add it to `html2rss-web`** so you can use it through your normal instance
-6. **Escalate request strategy when needed**: use a browser-based rendering strategy only when troubleshooting requires it
+2. **Optionally draft with capture** — `html2rss capture https://example.com/articles > your-config.yml` (see [Capturing Feed Configs](/ruby-gem/guides/capturing-feed-configs/))
+3. **Write or refine the smallest useful config** that extracts items, titles, and links
+4. **Validate the config** with `html2rss validate your-config.yml`
+5. **Render the feed** with `html2rss feed your-config.yml`
+6. **Add it to `html2rss-web`** so you can use it through your normal instance
+7. **Escalate request strategy when needed**: use Botasaurus (`strategy: botasaurus` or `auto` with `BOTASAURUS_SCRAPER_URL`) only when troubleshooting requires browser rendering
 
 This order keeps iteration fast and makes it easier to see whether the problem is the page structure, your
 selectors, or the fetch strategy.
@@ -188,8 +189,34 @@ there.
 **Help the community by sharing your config:**
 
 1. Go to [html2rss-configs on GitHub](https://github.com/html2rss/html2rss-configs)
-2. Click "Fork" → "Add file" → Create `domain.com.yml`
-3. Paste your config → "Commit new file" → "Open pull request"
+2. Click "Fork" → "Add file" → Create `domain.com/name.yml` under `lib/html2rss/configs/`
+3. Include top-level `directory.topics` (required for Feed Directory configs) using the controlled vocabulary below
+4. Paste your config → "Commit new file" → "Open pull request"
+
+Example catalog metadata:
+
+<Code
+  code={`
+  directory:
+    topics:
+      - tech
+      - research
+  channel:
+    url: https://example.com/blog
+    language: en
+  selectors:
+    items:
+      selector: "article.post"
+    title:
+      selector: "h2 a"
+    url:
+      selector: "h2 a"
+      extractor: "href"
+`}
+  lang="yaml"
+/>
+
+Allowed `directory.topics` values (prefer 1–2 primary topics): `sports`, `energy`, `tech`, `science`, `news`, `entertainment`, `jobs`, `finance`, `security`, `travel`, `environment`, `consumer`, `civic`, `product`, `research`.
 
 **Need help?** See our [contribution guide](/get-involved/contributing/) for detailed instructions.
 

src/content/docs/feed-directory/index.mdx

@@ -14,7 +14,7 @@ import FeedDirectory from "../../../components/FeedDirectory.astro";
 
 ---
 
-Need the main onboarding path first? Start with [Getting Started](/web-application/getting-started/) and create a feed from your own page URL. The directory below is the packaged fallback/catalog path for fast demos, known sample sources, or cases where the catalog already covers your site.
+Need the main onboarding path first? Start with [Getting Started](/web-application/getting-started/) and create a feed from your own page URL. The directory below is the packaged fallback path for fast demos, known sample sources, or cases where a curated config already covers your site. Use the topic and language filters to narrow the list; export OPML when you want to subscribe to several feeds at once.
 
 Need a different instance? You can use the built-in default, self-host your own, or find more options on the [community-run wiki](https://github.com/html2rss/html2rss-web/wiki/Instances).
 

src/content/docs/get-involved/contributing.mdx

@@ -26,10 +26,12 @@ Are you missing an RSS feed for a website? You can create your own feed config a
 **The easiest way to contribute:**
 
 1. Go to [html2rss-configs on GitHub](https://github.com/html2rss/html2rss-configs)
-2. Click "Fork" → "Add file" → Create `domain.com.yml`
-3. Paste your config → "Commit new file" → "Open pull request"
+2. Click "Fork" → "Add file" → Create `domain.com/name.yml` under `lib/html2rss/configs/`
+3. Include top-level `directory.topics` (non-empty) from the controlled vocabulary (`tech`, `news`, `security`, `research`, … — see [Creating Custom Feeds](/creating-custom-feeds/#sharing-your-config))
+4. Prefer setting `channel.language` when the page language is clear
+5. Paste your config → "Commit new file" → "Open pull request"
 
-**Need help writing configs?** See our [Creating Custom Feeds](/creating-custom-feeds/) guide.
+**Need help writing configs?** Start from [Capturing Feed Configs](/ruby-gem/guides/capturing-feed-configs/) or the [Creating Custom Feeds](/creating-custom-feeds/) guide.
 
 **Want to test your config first?** Use the [Ruby gem](/ruby-gem/installation/) to test it locally:
 

src/content/docs/ruby-gem/guides/ai-agent-workflows.mdx

@@ -0,0 +1,103 @@
+---
+title: AI Agent Workflows with MCP
+description: Guide to integrating html2rss with AI agents, Claude Desktop, and Cursor using the Model Context Protocol (MCP).
+---
+
+import { Code } from "@astrojs/starlight/components";
+
+`html2rss` includes a native [Model Context Protocol (MCP)](https://modelcontextprotocol.io/) server. This allows AI assistants in tools like Cursor, Claude Desktop, and GitHub Copilot to scrape pages, capture feed configurations, and generate RSS feeds autonomously.
+
+## Why Use MCP?
+
+Without MCP, agents must invoke shell commands, manage output buffers, and parse unstructured text. With MCP:
+
+- The agent automatically discovers available tools, arguments, and return types.
+- Responses are structured (JSON Feed objects, configuration schemas, and RSS XML).
+- Agents can inspect, capture, validate, and verify feeds iteratively in a closed feedback loop.
+
+## Client Setup
+
+### 1. Version Manager Shims (mise, asdf, rbenv, chruby)
+
+Because version managers manage Ruby runtimes and gem paths through environment shims, configure your MCP client to invoke the version manager executable rather than a bare `html2rss` command.
+
+<Code
+  code={`
+  # Install the gem in your global environment
+  mise exec -- gem install html2rss
+  # Or add it to your project Gemfile
+  mise exec -- bundle add html2rss
+`}
+  lang="bash"
+/>
+
+### 2. Cursor Configuration
+
+Add `html2rss` to your Cursor MCP settings (`~/.cursor/mcp.json` or `.cursor/mcp.json`):
+
+<Code
+  code={`
+  {
+    "mcpServers": {
+      "html2rss": {
+        "command": "mise",
+        "args": ["exec", "--", "html2rss", "mcp"]
+      }
+    }
+  }
+`}
+  lang="json"
+/>
+
+_(If using `asdf`, replace `"command": "mise"` with `"asdf"` and `"args": ["exec", "html2rss", "mcp"]`.)_
+
+### 3. Claude Desktop Configuration
+
+Add `html2rss` to your Claude Desktop configuration (`~/Library/Application Support/Claude/claude_desktop_config.json` on macOS or `%APPDATA%\\Claude\\claude_desktop_config.json` on Windows):
+
+<Code
+  code={`
+  {
+    "mcpServers": {
+      "html2rss": {
+        "command": "mise",
+        "args": ["exec", "--", "html2rss", "mcp"]
+      }
+    }
+  }
+`}
+  lang="json"
+/>
+
+## Autonomous Workflow Patterns
+
+### Pattern A: One-Shot Content Scraping
+
+When an agent needs articles immediately without saving a feed configuration:
+
+1. The agent calls `scrape_url` with the target URL.
+2. `html2rss` runs auto-source extraction (Schema.org, JSON state, semantic HTML) and returns a JSON Feed items array.
+3. If the page is protected or rendered with JavaScript, the agent calls `inspect_url` to diagnose the structure, then retries `scrape_url` with `strategy: "botasaurus"`.
+
+### Pattern B: Iterative Feed Config Authoring
+
+When an agent is tasked with creating a durable YAML feed configuration:
+
+1. **Inspect:** The agent calls `inspect_url` to check content type, SST node counts, and eligible scrapers.
+2. **Capture:** The agent runs `capture_config` to derive CSS selectors for items, title, link, and description.
+3. **Refine:** The agent reviews the derived selectors or asks the human user for domain-specific adjustments.
+4. **Validate:** The agent passes the configuration to `validate_config` to verify schema conformance.
+5. **Apply:** The agent tests the final configuration with `apply_config` to produce and inspect live RSS XML.
+
+## JavaScript-Rendered Sites (Botasaurus)
+
+For dynamic JavaScript single-page applications or sites protected by anti-bot measures, launch the Botasaurus scrape service:
+
+<Code
+  code={`
+  docker compose -f docker-compose.botasaurus.yml up -d
+`}
+  lang="bash"
+/>
+
+Ensure `BOTASAURUS_SCRAPER_URL` is accessible (typically `http://127.0.0.1:4010`) in the environment where the MCP server runs. Agents can then pass `strategy: "botasaurus"` to `scrape_url`, `inspect_url`, and `capture_config`.

src/content/docs/ruby-gem/guides/backward-compatibility.mdx

@@ -7,6 +7,13 @@ import { Code } from "@astrojs/starlight/components";
 
 This page outlines recent breaking changes, purged legacy aliases, and migration steps for older feed configurations.
 
+## 0.26.0
+
+Upgrade notes for configs and integrators targeting gem **0.26.0**:
+
+1. **Browserless strategy removal** — The `browserless` strategy, `puppeteer-ruby` dependency, interaction budgets, and `request.browserless.*` options have been completely removed.
+2. **Botasaurus as browser transport** — Browser-based scraping and anti-bot bypassing now run through `botasaurus` (`BOTASAURUS_SCRAPER_URL`). Use `strategy: auto` (which tries `faraday` -> `botasaurus`) or pin `strategy: botasaurus` directly.
+
 ## 0.25.0
 
 Upgrade notes for configs and integrators targeting gem **0.25.0**:
@@ -67,11 +74,11 @@ Update any occurrences of `link`, `pubDate`, or `updated` in your `selectors` bl
   # Deprecated / Removed
   channel:
     url: "https://example.com/articles"
-    strategy: browserless
+    strategy: botasaurus
     headers:
       User-Agent: "CustomAgent/1.0"
   # Current & Required
-  strategy: browserless
+  strategy: botasaurus
   headers:
     User-Agent: "CustomAgent/1.0"
   channel:
@@ -84,10 +91,10 @@ Update any occurrences of `link`, `pubDate`, or `updated` in your `selectors` bl
 
 When upgrading to modern `html2rss` releases:
 
-1. **Rename URL selectors**: Ensure article URL selectors use `url` rather than `link`.
-2. **Rename date selectors**: Ensure date selectors use `published_at` rather than `updated` or `pubDate`.
-3. **Move channel-level transport keys**: Ensure `strategy` and `headers` are defined at the top level of the YAML file.
-4. **Pin Browserless when needed**: If you relied on `auto` falling through to Browserless, set `strategy: browserless` (or `--strategy browserless`).
+1. **Migrate Browserless to Botasaurus**: Update any feeds using `strategy: browserless` or `request.browserless` to `strategy: botasaurus` or `strategy: auto` with `BOTASAURUS_SCRAPER_URL`.
+2. **Rename URL selectors**: Ensure article URL selectors use `url` rather than `link`.
+3. **Rename date selectors**: Ensure date selectors use `published_at` rather than `updated` or `pubDate`.
+4. **Move channel-level transport keys**: Ensure `strategy` and `headers` are defined at the top level of the YAML file.
 5. **Use `enclosure` for RSS media**: Do not rely on `image` becoming an RSS `<enclosure>`; select podcast/media URLs with `enclosure`.
 6. **Prefer `feed_result` for dual-format / cache**: Integrators that need RSS + JSON Feed from one scrape (or Marshal caching) should use `Html2rss.feed_result` and `status.to_h`.
 7. **Validate configurations**: Run `html2rss validate config.yml` to ensure your YAML conforms to the current schema.