diff --git a/.github/workflows/integrations.yml b/.github/workflows/integrations.yml index 6e5e73594..e54afd9ff 100644 --- a/.github/workflows/integrations.yml +++ b/.github/workflows/integrations.yml @@ -141,7 +141,7 @@ jobs: strategy: fail-fast: false matrix: - host: ["2026.8.2", "2026.9.1", "latest"] + host: ["2026.11.1", "latest"] steps: - name: Checkout uses: actions/checkout@v4 @@ -374,7 +374,7 @@ jobs: strategy: fail-fast: false matrix: - host: ['2026.8.2', '2026.9.1', 'latest'] + host: ['2026.11.1', 'latest'] steps: - uses: actions/checkout@v6 - uses: actions/setup-node@v4 diff --git a/bun.lock b/bun.lock index a737e7979..f9b01dc7d 100644 --- a/bun.lock +++ b/bun.lock @@ -267,7 +267,7 @@ "name": "@inline-openclaw/inline", "version": "0.0.72", "dependencies": { - "@inline-chat/realtime-sdk": "0.0.19", + "@inline-chat/realtime-sdk": "0.0.21", "zod": "4.6.7", }, "devDependencies": { @@ -278,7 +278,7 @@ "vitest": "4.1.10", }, "peerDependencies": { - "openclaw": ">=2026.8.2", + "openclaw": ">=2026.11.1", }, "optionalPeers": [ "openclaw", diff --git a/plugins/openclaw/AGENTS.md b/plugins/openclaw/AGENTS.md index bcddde19c..2bf31019d 100644 --- a/plugins/openclaw/AGENTS.md +++ b/plugins/openclaw/AGENTS.md @@ -2,8 +2,9 @@ ## Compatibility -- Support OpenClaw from `2026.8.2` with a minimum-only range. Never add an upper bound to host peer or plugin API compatibility. -- Keep CI to at most three hosts: the minimum host, one September release, and npm `latest`; a pinned development host is build provenance, not a support cap. +- Support OpenClaw from `2026.11.1` with a minimum-only range. Never add an upper bound to host peer or plugin API compatibility. The `0.0.x` line (`0.0.72`) is the legacy line for `2026.8.2` through `2026.10.x`. +- 1.0 uses the host's async SDK APIs and `MessagePresentation` only. Don't reintroduce the deprecated synchronous calls or the InteractiveReply bridge; they are removed in the first January 2027 stable host. +- Keep CI to at most three hosts: the minimum host, one intermediate release when the floor and `latest` differ, and npm `latest`; a pinned development host is build provenance, not a support cap. - Keep the compatibility table in `README.md` updated whenever the plugin version, OpenClaw host requirement, or Inline realtime SDK dependency changes. - The table should include the current plugin line plus at most five past plugin version lines. Prune older rows instead of letting the table grow indefinitely. - Each compatibility row should state the plugin version, supported OpenClaw host range or minimum host version, Inline realtime SDK version, status, and any important migration note. diff --git a/plugins/openclaw/README.md b/plugins/openclaw/README.md index 1d93ab993..e08b0ebd0 100644 --- a/plugins/openclaw/README.md +++ b/plugins/openclaw/README.md @@ -30,10 +30,11 @@ Reply-thread behavior: - Threads the bot follows continue without a mention. A thread's first message decides who follows; a bot mentioned later answers once without following. - Authorized users can run `/follow` in a group or reply thread to keep eligible unmentioned activity relevant, and `/unfollow` to explicitly opt out. -## Upgrading from 0.0.72 +## Upgrading to 1.0 -Breaking changes in the next release. They match the rules Hermes 1.0 follows; see [How bots behave](https://inline.chat/docs/openclaw). +Breaking changes in 1.0. The behavior changes match the rules Hermes 1.0 follows; see [How bots behave](https://inline.chat/docs/openclaw). +- **OpenClaw `2026.11.1` or newer is required.** 1.0 uses the host's async SDK APIs, which older hosts don't have, and drops the synchronous ones that OpenClaw removes in January 2027. Stay on `0.0.72` for older hosts. - **`/threadreply` is now `/threads`**, with `auto`, `on`, `reset`, and `status`, and it works in DMs. - **`replyThreadMode` is Auto or On, per chat.** On a chat, `thread` reads as `on` and `main` as `auto`. A bot-wide or `groups."*"` On is ignored with a startup warning; run `/threads on` in each chat that should use threads. - **Words no longer open a thread.** The agent understands the request and opens one. @@ -46,18 +47,18 @@ Breaking changes in the next release. They match the rules Hermes 1.0 follows; s ## Install -The Inline plugin supports OpenClaw `2026.8.2` and newer, including -`2026.9.x`, with no upper version cap. It is built against `2026.11.1`. Install -the plugin version for your OpenClaw release line. The current stable release -is `0.0.72`. +The next major, `1.0`, supports OpenClaw `2026.11.1` and newer with no upper +version cap. Install the plugin version for your OpenClaw release line. The +current stable release is `0.0.72`. | OpenClaw line | Inline plugin | Exact install | | --- | --- | --- | -| `2026.8.2` and newer, including `2026.10.5` | `0.0.72` | `openclaw plugins install npm:@inline-openclaw/inline@0.0.72 --force --accept-capabilities` | +| `2026.11.1` and newer | `1.0` (unreleased) | Published with the 1.0 release | +| `2026.8.2` through `2026.10.x` | `0.0.72` | `openclaw plugins install npm:@inline-openclaw/inline@0.0.72 --force --accept-capabilities` | | `2026.7.x` (`>=2026.7.1`) | `0.0.63` | `openclaw plugins install @inline-openclaw/inline@0.0.63 --force` | | `2026.6.x` (`>=2026.6.11`, including extended-stable `2026.6.34`) | `0.0.63` | `openclaw plugins install @inline-openclaw/inline@0.0.63 --force` | -The minimum supported host remains `2026.8.2`. Newer host versions are not +The minimum supported host for 1.0 is `2026.11.1`. Newer host versions are not blocked by package metadata. OpenClaw 2026.9.3 and newer require Node 24.16+ on the 24.x line or Node 26.1+; Node 26 is recommended. @@ -65,7 +66,7 @@ on the 24.x line or Node 26.1+; Node 26 is recommended. | Plugin version | OpenClaw host | Inline realtime SDK | Status | Notes | | --- | --- | --- | --- | --- | -| `0.0.72` | `>=2026.8.2` | `0.0.19` | Stable | Loads on OpenClaw `2026.10.5` and newer, which removed `readJsonFileWithFallback`; `0.0.71` and older fail to register the channel there. Matches `toolsBySender` on numeric sender ids only. | +| `0.0.72` | `>=2026.8.2` | `0.0.19` | Stable | Last line for OpenClaw `2026.8.2`–`2026.10.x`; 1.0 requires `2026.11.1`. Loads on OpenClaw `2026.10.5` and newer, which removed `readJsonFileWithFallback`; `0.0.71` and older fail to register the channel there. Matches `toolsBySender` on numeric sender ids only. | | `0.0.71` | `>=2026.8.2` | `0.0.19` | Previous | Adds opt-in quiet agent activity timelines and the expanded chat-history protocol while retaining recovery and install provenance diagnostics. | | `0.0.70` | `>=2026.8.2` | `0.0.18` | Previous | Preserves native reply suppression and send-policy decisions, with September approvals and shutdown cancellation. | | `0.0.69` | `>=2026.8.2` | `0.0.18` | Previous | Added September system change approvals and shutdown cancellation. | @@ -91,7 +92,7 @@ on the 24.x line or Node 26.1+; Node 26 is recommended. - Carries forward native reply suppression, September approvals, cancellation-safe shutdown, and realtime SDK `0.0.19` recovery behavior. - Tests the extracted package against released hosts and checks that the bundled SDK matches the validated build. -The CI host matrix runs on `2026.8.2`, `2026.9.1`, and npm `latest`. +The CI host matrix runs on `2026.11.1` and npm `latest`. To check an installed host locally after building the SDK: ```sh diff --git a/plugins/openclaw/REVIEW-1.0.md b/plugins/openclaw/REVIEW-1.0.md new file mode 100644 index 000000000..6eb84e69b --- /dev/null +++ b/plugins/openclaw/REVIEW-1.0.md @@ -0,0 +1,105 @@ +# OpenClaw 1.0 review + +Reviewed 8 November 2026. Baseline: [PR #157](https://github.com/inline-chat/inline/pull/157), commit `3d8f57d53`. This draft is stacked on that commit; the original PR is unchanged. + +## Assessment + +The host migration is a useful foundation. The minimum-only host range, async API migration, SDK update, service-message filtering, and shared account client are sensible. The monitor split preserves behavior and makes the next work easier. The PR's OpenClaw source, packed consumer, and installed-host checks were green when inspected. + +It is not the complete 1.0 experience. In particular, reliable inbound acceptance, Inline-owned output settings, richer progress, and grant-backed approvals remain. Treat the foundation and the full 1.0 release as separate checkpoints. + +## Corrections in this draft + +| Priority | Problem at the reviewed head | Correction | +| --- | --- | --- | +| P1 | Typed `command` presentation actions lose their kind and become `agent:` callbacks. Pressing `/status` can start an ordinary agent turn instead of executing a native command. | Preserve the explicit command kind through rendering and use `system:` action IDs. Opaque callback data remains agent-owned even when it looks like a command. Applies to buttons and select options. | +| P1 | The monitor renders disabled presentation buttons as live actions. | Omit their native actions; retain their labels in fallback text. | +| P2 | The monitor returns authored text before rendering presentation fallback. A URL-only control, table, or other unsupported content can disappear. | Render canonical presentation content on the direct monitor path, honor an authored `presentationTextMode: fallback`, and strip consumed presentation metadata from the outbound renderer to prevent a second flattening. | +| P2 | A cancelled caller can borrow a shared client because acquisition previously checked cancellation only through a fresh connection. | Check cancellation before borrowing, after connecting, and before invoking the leased operation. Never close the monitor's client on a cancelled lease. | +| P2 | The monitor's direct sends do not wait for explicit server flood hints. Immediate fallback sends can encounter the same rejection. | One sender owns all 15 direct monitor send sites. Retry explicit rate-limit rejections at most twice, wait the complete server hint up to 60 seconds, preserve one `randomId`, and cancel waits on monitor shutdown. Longer waits and errors without a trustworthy hint remain failures. Transport errors and unknown commit outcomes are not retried by this helper. | +| Cleanup | The plugin pins SDK 0.0.21 but still expresses Working as numeric `6`. | Use the SDK's named `WORKING` enum. | + +The action parser is shared by outbound and monitor rendering; the duplicate monitor parser is removed. This remains a small mapping onto Inline's existing controls. It introduces no question framework, configuration key, protocol change, database change, or release/version bump. + +The flood sender is an in-process recovery improvement. It is not a durable monitor outbox. Native outbound delivery retains the host's existing outbox and stable delivery identities. Edits, uploads, and approval-runtime sends have their existing error handling; this draft does not promise to make every RPC retryable. + +## Remaining work before the full 1.0 release + +### 1. Preserve inbound receipts through retry-safe acceptance — P1 + +`monitor.ts` calls `scheduleInboundMessage` / `scheduleImmediateInboundMessage` and returns from the SDK `consumeEvents` handler. The SDK acknowledges handler completion. These schedules live in memory; `claimInlineInboundMessageInstance` also runs before routing and dispatch. `handleInboundNow` returns after an initial metadata failure, and falls back to a stale group snapshot after some refresh failures. + +Consequences: a process interruption can lose already-acknowledged scheduled work; a temporary chat/sender lookup failure can become a permanent drop; stale follow state can decide whether a bot wakes. This is existing debt carried through the PR, rather than a regression introduced by its helper extraction. + +Hermes keeps the SDK receipt pending through its sidecar/adapter handoff and retries routing failures before deduplication or effects. Bridge v2 records direct input in its ledger before acknowledgement. Adopt that ownership model here while moving to the host's public inbound/turn helpers: + +- Keep the receipt until the message is intentionally ignored or accepted by a durable host owner; in-memory scheduling alone is not acceptance. +- Retry unavailable metadata/provenance before marking the message handled. Preserve per-chat order while allowing other chats to proceed. +- Recheck live follow/access state before admission. Distinguish unavailable authority from a deliberate denial. +- Keep `/stop` and steering responsive. Simply awaiting an entire model turn in the SDK callback would serialize away required control behavior. +- Prove crash-before-admission replay, lookup failure then recovery, cancellation during debounce/voice wait, independent-chat progress, and acknowledgement-response loss with the real SDK event stream. + +The current tests strongly exercise routing and presentation, but do not establish this end-to-end recovery property. Exactly-once model execution or external side effects should not be claimed. + +### 2. Wire Inline-owned output settings and the composer — P1 for 1.0 UX + +The source protocol contains `GET_AGENT_SETTINGS`, settings pushes, and `OUTPUT_MODE`, but the published SDK remains `0.0.21` with protocol `0.0.13`. The public protocol tarball inspected during this review does **not** contain `GET_AGENT_SETTINGS` or `OUTPUT_MODE`; it does contain `WORKING` and `startedAt`. Workspace source overrides can hide this distinction. + +After the protocol/SDK publish: + +- Declare `OUTPUT_MODE`; fetch the bot's settings and apply revisions/pushes without allowing an old fetch to overwrite a newer update. +- Resolve thread override, then bot default, then Friendly. Freeze a mode for each turn so an update cannot create half of two presentations. +- Friendly: Working and a concise final answer, with a bounded prose update only when useful. Detailed: progress and long-answer edit streaming. +- Publish `AGENT_CONFIGURATION` from the host model/reasoning catalog. The Apple composer expects item IDs `agent.model` and `agent.reasoning`; this plugin currently uses `model` and `reasoning`. Change IDs, capabilities, and mutation handling together, retaining any required compatibility for already-open settings cards. + +Do not hand-code method numbers or widen SDK types to use unpublished fields. Verify a packed plugin against registry dependencies, not only the monorepo aliases. + +### 3. Finish Detailed progress using Inline primitives — P1 for 1.0 UX + +The richer timeline is still behind `experimentalAgentActivity`; ordinary progress is a temporary placeholder. Host events are flattened to Markdown. Port the bridge's event vocabulary and bounded document model, not its entire transport/runtime: + +- One progress disclosure with a real `startedAt`, prose interleaved with tool groups, and typed statuses. Use Steps for plans, Diff for patches, Fields for approvals, and Callout for actionable failures. +- Keep the established Working header, collapsible content, and local client timer. Preserve the agreed open-state/prose behavior; a bridge implementation detail is not automatically the intended UX. +- Split at steering, questions/approvals, and size boundaries. Clear the live thought on completion. Send the final answer separately with normal notification behavior. +- Coalesce edits, bound output tails and document size, and degrade unsupported content to Markdown/plain text. Reuse the host's live-preview lifecycle where it fits instead of adding another streaming owner. +- Test recorded event sequences from more than one host runtime, including stop during tools, late terminal events, failed edits, and answers following an approval. + +### 4. Make approvals genuinely grant-backed — P1 for the agreed 1.0 behavior + +Approval buttons currently use the host's command authorization/config approvers and `requiredGrant = 0`. Clearing the actions leaves the original request text. That is different from Inline APPROVE grants, a visible resolved outcome, and placement in the working thread. + +Implement authority and presentation together: server-enforced APPROVE requirements, current owner/delegate eligibility, the specified extra config allowance, and a resolved/expired/answered-elsewhere edit. Do not add `requiredGrant` cosmetically while leaving configured approvers unable to use the card. Keep a plain “continue” separate from approval. Continue using the host's pending approval authority rather than treating callback IDs as consent. + +Typed host approval actions are still unsupported by the generic presentation mapper; the plugin's dedicated native approval runtime is the current owner. Typed questions and richer approval semantics need deliberate integration with that owner, not conversion into model callbacks. + +### 5. Complete the behavior contract — P2 + +- **Sessions:** On-mode opener/follow-ups already share the child session. Auto-mode thread adoption still has the documented parent/child session split. Preserve privacy for guests in DM-rooted threads when addressing it. +- **Bot hand-backs:** prompt guidance and deliberate-silence handling exist, but verify the real host's no-visible-reply recovery does not force another bot response. Include a two-bot request/hand-back test and an authorized loop-cap recovery path. +- **`/new`:** a fresh host session is not automatically a fresh Inline reply thread. Implement the decided thread behavior explicitly. +- **Tool consolidation:** one `inline` tool plus `message` can reduce prompt schemas, but retain the host's read-authority fence and per-operation mutation checks. Consolidation must not create a second route around a blocked `message(read)`. +- **Status:** distinguish a successful socket/GET_ME probe from receiving, admitting, and answering messages. Report unresolved receipt/backlog and policy failures without declaring an unused chat unhealthy merely because it is quiet. +- **Setup/docs:** finish the CLI-assisted install/doctor path and migration documentation after the actual 1.0 behavior and dependency versions settle. + +### 6. Keep upstream-dependent work separate + +Current public SDK documentation describes more than the minimum supported release. Check both the released exports and current source before adopting question routing, fork preparation, display mentions, or child-placement bindings. Prefer public helpers and capability checks over private plugin files or version-string branches. + +The official catalog/trust work remains separate. Raising the host floor or declaring `providerOwnedReadGates` does not make an external install trusted. Do not promise broad cross-chat reads until a host release includes the trusted install/catalog contract. Bot agent-session/work-bar support remains the separately planned server/protocol work. + +## Validation and release qualification + +Baseline: 47 files, 748 tests passed locally. Draft: `bun run check` passes type checking, lint, 48 test files / 770 tests with coverage, and the runtime/declaration build. The suite includes the packed-artifact check. PR-host CI was also green when checked. + +Regressions cover rendered command dispatch through the monitor, opaque callback isolation, disabled controls, unsupported URL fallback, shared-client cancellation, bounded flood waits, stable identities, and uncertain-send suppression. The packed-artifact test checks the generated runtime bundle. + +A real gateway with live bots was not used for this review. Before releasing 1.0, qualify the packed artifact on the minimum and latest host, two-bot hand-backs, group/DM Auto and On threads, a guest in a DM-rooted thread, restart/replay, approval resolution from another client, Stop/steer, and both output modes on native clients. Use the actual published protocol/SDK versions. + +## Sources + +- [OpenClaw 2026.11.1 release](https://github.com/openclaw/openclaw/releases/tag/v2026.11.1). +- [Native Discord presentation mapping at the minimum host](https://github.com/openclaw/openclaw/blob/v2026.11.1/extensions/discord/src/shared-interactive.ts): command/callback kind, styles, disabled state, and typed approval routing. +- [Native Slack presentation mapping](https://github.com/openclaw/openclaw/blob/23f2cd68e9d38bbffe561bada8eab5a72b9af110/extensions/slack/src/blocks-render.ts) and [Telegram approval callbacks](https://github.com/openclaw/openclaw/blob/23f2cd68e9d38bbffe561bada8eab5a72b9af110/extensions/telegram/src/approval-callback-data.ts). +- [Message presentation contract](https://docs.openclaw.ai/plugins/message-presentation) and [channel inbound API](https://docs.openclaw.ai/plugins/sdk-channel-inbound). Online documentation is current-main guidance, not proof of a minimum-host export. +- [Public inbound exports at the inspected current main](https://github.com/openclaw/openclaw/blob/23f2cd68e9d38bbffe561bada8eab5a72b9af110/src/plugin-sdk/channel-inbound.ts). Local source comparison also used `b67fef2c7`. +- Inline references: `plugins/hermes-agent/src/sidecar/inbound-delivery.ts`, `plugins/hermes-agent/src/sidecar/inbound-stream.ts`, `plugins/hermes-agent/plugin/inline/adapter.py`, `cli/src/bridge/v2/runtime.rs`, `render.rs`, `doc.rs`, `settings.rs`, `work.rs`, and `apple/InlineKit/Sources/InlineKit/Agents/ComposeAgentChainModel.swift`. diff --git a/plugins/openclaw/docs/openclaw-setup.md b/plugins/openclaw/docs/openclaw-setup.md index 8c071210c..61a492388 100644 --- a/plugins/openclaw/docs/openclaw-setup.md +++ b/plugins/openclaw/docs/openclaw-setup.md @@ -8,7 +8,8 @@ Need a bot token first? See `docs/create-inline-bot.md`. Choose the Inline plugin version that matches the installed OpenClaw line: -- OpenClaw `2026.8.2` and newer, including `2026.9.x`: Inline plugin `0.0.70` — `openclaw plugins install npm:@inline-openclaw/inline@0.0.70 --force --accept-capabilities` +- OpenClaw `2026.11.1` and newer: Inline plugin `1.0` (unreleased; published with the 1.0 release) +- OpenClaw `2026.8.2` through `2026.10.x`: Inline plugin `0.0.72` — `openclaw plugins install npm:@inline-openclaw/inline@0.0.72 --force --accept-capabilities` - OpenClaw `2026.7.x` (`>=2026.7.1`): Inline plugin `0.0.63` — `openclaw plugins install @inline-openclaw/inline@0.0.63 --force` - OpenClaw `2026.6.x` (`>=2026.6.11`, including extended-stable `2026.6.34`): Inline plugin `0.0.63` — `openclaw plugins install @inline-openclaw/inline@0.0.63 --force` @@ -22,7 +23,7 @@ newer, use Node 24.16+ on 24.x or Node 26.1+ (Node 26 recommended). Install the current version: ```sh -openclaw plugins install npm:@inline-openclaw/inline@0.0.70 --force --accept-capabilities +openclaw plugins install npm:@inline-openclaw/inline@0.0.72 --force --accept-capabilities ``` If already installed on a supported host, update in place: diff --git a/plugins/openclaw/package.json b/plugins/openclaw/package.json index f7aec9fe5..526ccfaf6 100644 --- a/plugins/openclaw/package.json +++ b/plugins/openclaw/package.json @@ -36,11 +36,11 @@ "check:host": "bun scripts/check-host.mjs" }, "dependencies": { - "@inline-chat/realtime-sdk": "0.0.19", + "@inline-chat/realtime-sdk": "0.0.21", "zod": "4.6.7" }, "peerDependencies": { - "openclaw": ">=2026.8.2" + "openclaw": ">=2026.11.1" }, "peerDependenciesMeta": { "openclaw": { @@ -100,10 +100,10 @@ "npmSpec": "@inline-openclaw/inline", "localPath": "openclaw", "defaultChoice": "clawhub", - "minHostVersion": ">=2026.8.2" + "minHostVersion": ">=2026.11.1" }, "compat": { - "pluginApi": ">=2026.8.2" + "pluginApi": ">=2026.11.1" }, "build": { "openclawVersion": "2026.11.1" diff --git a/plugins/openclaw/src/inline/actions.test.ts b/plugins/openclaw/src/inline/actions.test.ts index 5a4fd4fc1..d2d82cdf7 100644 --- a/plugins/openclaw/src/inline/actions.test.ts +++ b/plugins/openclaw/src/inline/actions.test.ts @@ -4472,6 +4472,70 @@ describe("inline/actions", () => { expect(new TextDecoder().decode(data)).toBe("approve") }) + it("maps typed presentation actions in shared interactive payloads to inline callbacks", async () => { + vi.resetModules() + + const sendMessage = vi.fn(async () => ({ messageId: 1n })) + + vi.doMock("@inline-chat/realtime-sdk", () => ({ + Method: { + EDIT_MESSAGE: 8, + }, + InlineSdkClient: class { + constructor(_opts: unknown) {} + connect = vi.fn(async () => {}) + close = vi.fn(async () => {}) + sendMessage = sendMessage + invokeRaw = vi.fn(async () => ({ oneofKind: undefined })) + }, + })) + + const { inlineMessageActions } = await import("./actions") + + await inlineMessageActions.handleAction?.({ + channel: "inline", + action: "send", + cfg: { + channels: { + inline: { + token: "token", + baseUrl: "https://api.inline.chat", + }, + }, + } satisfies OpenClawConfig, + params: { + to: "user:99", + message: "typed actions", + interactive: JSON.stringify({ + blocks: [ + { + type: "buttons", + buttons: [ + { label: "Approve", action: { type: "callback", value: "approve" } }, + { label: "Docs", action: { type: "url", url: "https://inline.chat/docs" } }, + { label: "Status", action: { type: "command", command: "/status" } }, + ], + }, + { + type: "select", + options: [{ label: "Fast", action: { type: "callback", value: "model:fast" } }], + }, + ], + }), + }, + } as any) + + const rows = sendMessage.mock.calls[0]?.[0]?.actions?.rows + expect(rows).toHaveLength(2) + expect(rows?.[0]?.actions.map((action: any) => action.text)).toEqual(["Approve", "Status"]) + expect(rows?.[0]?.actions.map((action: any) => action.actionId)).toEqual(["agent:1:1", "system:1:2"]) + expect( + rows?.[0]?.actions.map((action: any) => new TextDecoder().decode(action.action.callback.data)), + ).toEqual(["approve", "/status"]) + expect(rows?.[1]?.actions?.[0]?.text).toBe("Fast") + expect(new TextDecoder().decode(rows?.[1]?.actions?.[0]?.action.callback.data)).toBe("model:fast") + }) + it("passes empty buttons on edit to clear existing actions", async () => { vi.resetModules() diff --git a/plugins/openclaw/src/inline/actions.ts b/plugins/openclaw/src/inline/actions.ts index 6eab08711..84dbaae54 100644 --- a/plugins/openclaw/src/inline/actions.ts +++ b/plugins/openclaw/src/inline/actions.ts @@ -6,9 +6,11 @@ import type { ChannelMessageToolSchemaContribution, } from "openclaw/plugin-sdk/channel-contract" import { - normalizeInteractiveReply, - reduceInteractiveReply, - type InteractiveReplyButton, + isMessagePresentationInteractiveBlock, + resolveMessagePresentationButtonAction, + resolveMessagePresentationOptionAction, + type MessagePresentation, + type MessagePresentationAction, } from "openclaw/plugin-sdk/interactive-runtime" import type { OpenClawConfig } from "openclaw/plugin-sdk/core" import { @@ -21,6 +23,7 @@ import { type User, } from "@inline-chat/realtime-sdk" import { listInlineAccountIds, resolveInlineAccount, resolveInlineToken } from "./accounts.js" +import { withInlineAccountClient } from "./client-registry.js" import { adoptInlineActiveThread, getInlineActiveThreadRoute } from "./active-thread-route.js" import { downloadInlineMediaFromUrl, uploadInlineMediaFromUrl } from "./media.js" import { summarizeInlineMessageContent } from "./message-content.js" @@ -37,7 +40,10 @@ import { sanitizeInlineActionLabel, sanitizeInlineVisibleText, } from "./outbound-sanitize.js" -import { resolveInlineInteractiveTextFallback } from "./interactive-fallback.js" +import { + normalizeInlineLegacyInteractivePresentation, + resolveInlineInteractiveTextFallback, +} from "./interactive-fallback.js" import { assertInlineReadAllowed, captureInlineReadFence } from "./read-authority.js" import { buildInlineUserDisplayName, getSpaceMembersWithUsers } from "./space-members.js" import { @@ -855,6 +861,8 @@ type InlineReplyMarkupButton = text: string kind: "callback" callbackData: string + callbackDataKind?: "command" + style?: number } | { text: string @@ -963,7 +971,10 @@ function resolveInlineSpecializationIdFromSessionKey(sessionKey: string | null | return sessionKey?.match(/:inline-agent:([^:]+)$/)?.[1] } -function normalizeReplyMarkupButtons(raw: unknown): InlineReplyMarkupButton[][] { +function normalizeReplyMarkupButtons( + raw: unknown, + mapCallbackData?: (value: string) => string, +): InlineReplyMarkupButton[][] { if (!Array.isArray(raw)) return [] const rows: InlineReplyMarkupButton[][] = [] for (const candidateRow of raw) { @@ -974,8 +985,9 @@ function normalizeReplyMarkupButtons(raw: unknown): InlineReplyMarkupButton[][] const text = sanitizeInlineActionLabel( typeof candidateButton.text === "string" ? candidateButton.text : "", ) + const rawCallbackData = typeof candidateButton.callback_data === "string" ? candidateButton.callback_data : "" const callbackData = sanitizeInlineActionCallbackData( - typeof candidateButton.callback_data === "string" ? candidateButton.callback_data : "", + mapCallbackData ? mapCallbackData(rawCallbackData) : rawCallbackData, ) const copyText = sanitizeInlineActionCopyText( typeof candidateButton.copy_text === "string" @@ -986,7 +998,13 @@ function normalizeReplyMarkupButtons(raw: unknown): InlineReplyMarkupButton[][] ) if (!text) continue if (callbackData) { - row.push({ text, kind: "callback", callbackData }) + row.push({ + text, + kind: "callback", + callbackData, + ...(candidateButton.callback_data_kind === "command" ? { callbackDataKind: "command" as const } : {}), + ...(candidateButton.style === 1 || candidateButton.style === 2 ? { style: candidateButton.style } : {}), + }) } else if (copyText) { row.push({ text, kind: "copyText", copyText }) } @@ -999,17 +1017,44 @@ function normalizeReplyMarkupButtons(raw: unknown): InlineReplyMarkupButton[][] return rows } -function chunkInteractiveButtons( - buttons: readonly InteractiveReplyButton[], +type InlinePresentationControl = { + label: string + action: MessagePresentationAction | undefined + disabled?: boolean | undefined + style?: string | undefined +} + +/** + * Preserve the action kind independently of callback data. A command is handled + * by the host; an opaque callback (even one containing /approve) belongs to the + * agent. Unsupported actions stay in the presentation fallback text. + */ +function resolveInlinePresentationCallbackData(action: MessagePresentationAction | undefined) { + if (action?.type === "callback") return action.value + if (action?.type === "command") return action.command + return undefined +} + +function chunkPresentationControls( + controls: readonly InlinePresentationControl[], rows: InlineReplyMarkupButton[][], ) { - for (let i = 0; i < buttons.length; i += INLINE_ACTION_MAX_PER_ROW) { + for (let i = 0; i < controls.length; i += INLINE_ACTION_MAX_PER_ROW) { const row: InlineReplyMarkupButton[] = [] - for (const button of buttons.slice(i, i + INLINE_ACTION_MAX_PER_ROW)) { - const text = sanitizeInlineActionLabel(button.label) - const callbackData = sanitizeInlineActionCallbackData(button.value) + for (const control of controls.slice(i, i + INLINE_ACTION_MAX_PER_ROW)) { + if (control.disabled) continue + const text = sanitizeInlineActionLabel(control.label) + const callbackData = sanitizeInlineActionCallbackData( + resolveInlinePresentationCallbackData(control.action), + ) if (!text || !callbackData) continue - row.push({ text, kind: "callback", callbackData }) + row.push({ + text, + kind: "callback", + callbackData, + ...(control.action?.type === "command" ? { callbackDataKind: "command" as const } : {}), + ...(control.style === "primary" ? { style: 1 } : control.style === "danger" ? { style: 2 } : {}), + }) } if (row.length === 0) continue @@ -1018,6 +1063,47 @@ function chunkInteractiveButtons( } } +function buildInlinePresentationRows(presentation: MessagePresentation): InlineReplyMarkupButton[][] { + const rows: InlineReplyMarkupButton[][] = [] + for (const block of presentation.blocks) { + if (rows.length >= INLINE_ACTION_MAX_ROWS) break + if (!isMessagePresentationInteractiveBlock(block)) continue + const controls: InlinePresentationControl[] = + block.type === "buttons" + ? block.buttons.map((button) => ({ + label: button.label, + action: resolveMessagePresentationButtonAction(button), + disabled: button.disabled, + style: button.style, + })) + : block.options.map((option) => ({ + label: option.label, + action: resolveMessagePresentationOptionAction(option), + })) + chunkPresentationControls(controls, rows) + } + return rows +} + +/** + * Render presentation buttons/selects as Inline `channelData.inline.buttons` + * rows, the same shape agent-authored `buttons` use. + */ +export function buildInlinePresentationButtons( + presentation: MessagePresentation, +): Array> { + return buildInlinePresentationRows(presentation).map((row) => + row.flatMap((button) => + button.kind === "callback" ? [{ + text: button.text, + callback_data: button.callbackData, + ...(button.callbackDataKind ? { callback_data_kind: button.callbackDataKind } : {}), + ...(button.style ? { style: button.style } : {}), + }] : [], + ), + ) +} + function resolveInlineInteractiveButtonsParam( params: Record, ): InlineReplyMarkupButton[][] | undefined { @@ -1038,30 +1124,12 @@ function resolveInlineInteractiveButtonsParam( } } - const interactive = normalizeInteractiveReply(rawInteractive) - if (!interactive) { + const presentation = normalizeInlineLegacyInteractivePresentation(rawInteractive) + if (!presentation) { return undefined } - const rows = reduceInteractiveReply(interactive, [] as InlineReplyMarkupButton[][], (state, block) => { - if (state.length >= INLINE_ACTION_MAX_ROWS) { - return state - } - if (block.type === "buttons") { - chunkInteractiveButtons(block.buttons, state) - return state - } - if (block.type === "select") { - chunkInteractiveButtons( - block.options.map((option) => ({ - label: option.label, - ...(option.value !== undefined ? { value: option.value } : {}), - })), - state, - ) - } - return state - }) + const rows = buildInlinePresentationRows(presentation) return rows.length > 0 ? rows : undefined } @@ -1072,9 +1140,13 @@ function toInlineMessageActions( return { rows: rows.map((row, rowIndex) => ({ actions: row.map((button, buttonIndex) => ({ - actionId: buildInlineMessageActionId(owner, rowIndex, buttonIndex), + actionId: buildInlineMessageActionId( + button.kind === "callback" && button.callbackDataKind === "command" ? "system" : owner, + rowIndex, + buttonIndex, + ), text: button.text, - style: 0, + style: button.kind === "callback" ? button.style ?? 0 : 0, requiredGrant: 0, action: button.kind === "callback" @@ -1097,7 +1169,7 @@ function toInlineMessageActions( export function resolveInlineMessageActionsParam( params: Record, - options?: { owner?: InlineMessageActionOwner }, + options?: { owner?: InlineMessageActionOwner; mapCallbackData?: (value: string) => string }, ): MessageActions | undefined { const owner = options?.owner ?? "agent" if (!Object.prototype.hasOwnProperty.call(params, "buttons")) { @@ -1125,7 +1197,7 @@ export function resolveInlineMessageActionsParam( throw new Error("inline action: buttons must be an array of button rows") } - const rows = normalizeReplyMarkupButtons(rawButtons) + const rows = normalizeReplyMarkupButtons(rawButtons, options?.mapCallbackData) return toInlineMessageActions(rows, owner) } @@ -2441,7 +2513,7 @@ async function findMessageById(params: { async function withInlineClient(params: { cfg: OpenClawConfig accountId: string | null | undefined - /** Host authority fence; runs after token resolution, right before connecting. */ + /** Host authority fence; runs after token resolution, before any Inline request (shared or fresh client). */ beforeConnect?: (() => void) | undefined fn: (client: InlineSdkClient, account: ReturnType) => Promise }): Promise { @@ -2450,17 +2522,11 @@ async function withInlineClient(params: { throw new Error(`Inline not configured for account "${account.accountId}" (missing token or baseUrl)`) } const token = await resolveInlineToken(account) - const client = new InlineSdkClient({ - baseUrl: account.baseUrl, - token, - }) params.beforeConnect?.() - await client.connect() - try { - return await params.fn(client, account) - } finally { - await client.close().catch(() => {}) - } + return await withInlineAccountClient( + { accountId: account.accountId, baseUrl: account.baseUrl, token }, + (client) => params.fn(client, account), + ) } function toJsonSafe(value: unknown): unknown { diff --git a/plugins/openclaw/src/inline/approval-handler.runtime.ts b/plugins/openclaw/src/inline/approval-handler.runtime.ts index bb83c64a4..a42757768 100644 --- a/plugins/openclaw/src/inline/approval-handler.runtime.ts +++ b/plugins/openclaw/src/inline/approval-handler.runtime.ts @@ -3,7 +3,7 @@ import type { ChannelApprovalKind, PendingApprovalView, } from "openclaw/plugin-sdk/approval-handler-runtime" -import { createChannelApprovalNativeRuntimeAdapter } from "openclaw/plugin-sdk/approval-handler-runtime" +import { createChannelApprovalNativeRuntimeAdapterAsync } from "openclaw/plugin-sdk/approval-handler-runtime" import { buildChannelApprovalNativeTargetKey } from "openclaw/plugin-sdk/approval-native-runtime" import { buildExecApprovalPendingReplyPayload, @@ -221,7 +221,17 @@ function buildPendingPayload(params: { } } -export const inlineApprovalNativeRuntime = createChannelApprovalNativeRuntimeAdapter< +// Named through the public factory because the SDK does not export the async adapter type. +type InlineApprovalNativeRuntimeAdapter = ReturnType< + typeof createChannelApprovalNativeRuntimeAdapterAsync< + InlinePendingDelivery, + InlinePreparedTarget, + InlinePendingApproval, + never + > +> + +export const inlineApprovalNativeRuntime: InlineApprovalNativeRuntimeAdapter = createChannelApprovalNativeRuntimeAdapterAsync< InlinePendingDelivery, InlinePreparedTarget, InlinePendingApproval, @@ -238,10 +248,10 @@ export const inlineApprovalNativeRuntime = createChannelApprovalNativeRuntimeAda }) : false }, - shouldHandle: (params) => { + shouldHandle: async (params) => { const resolved = resolveHandlerContext(params) return resolved - ? shouldHandleInlineExecApprovalRequest({ + ? await shouldHandleInlineExecApprovalRequest({ cfg: params.cfg, accountId: resolved.accountId, request: params.request, diff --git a/plugins/openclaw/src/inline/approval-native.test.ts b/plugins/openclaw/src/inline/approval-native.test.ts index a913fe287..b0c3fb59b 100644 --- a/plugins/openclaw/src/inline/approval-native.test.ts +++ b/plugins/openclaw/src/inline/approval-native.test.ts @@ -6,6 +6,7 @@ import { inlineApprovalCapability } from "./approval-native" import { getInlineExecApprovalApprovers, isInlineExecApprovalClientEnabled, + shouldHandleInlineExecApprovalRequest, } from "./exec-approvals" const SET_BOT_PRESENCE_STATE = 59 @@ -108,33 +109,38 @@ describe("inline/native approvals", () => { const request = buildExecRequest({ turnSourceThreadId: "8" }) expect(isInlineExecApprovalClientEnabled({ cfg })).toBe(true) - expect( - inlineApprovalCapability.authorizeActorAction?.({ + // Only the async (worker-owned) surfaces are registered. + for (const legacySurface of ["authorizeActorAction", "native", "nativeRuntime"] as const) { + expect(inlineApprovalCapability[legacySurface]).toBeUndefined() + } + expect(inlineApprovalCapability.nativeRuntimeAsync).toBeDefined() + await expect( + inlineApprovalCapability.authorizeActorActionAsync!({ cfg, accountId: "default", senderId: "user:51", action: "approve", approvalKind: "exec", }), - ).toEqual({ authorized: true }) - expect( - inlineApprovalCapability.authorizeActorAction?.({ + ).resolves.toEqual({ authorized: true }) + await expect( + inlineApprovalCapability.authorizeActorActionAsync!({ cfg, accountId: "default", senderId: "chat:51", action: "approve", approvalKind: "plugin", }), - ).toMatchObject({ authorized: false }) + ).resolves.toMatchObject({ authorized: false }) - expect( - inlineApprovalCapability.native?.describeDeliveryCapabilities?.({ + await expect( + inlineApprovalCapability.nativeAsync!.describeDeliveryCapabilities({ cfg, accountId: "default", approvalKind: "exec", request, }), - ).toMatchObject({ + ).resolves.toMatchObject({ enabled: true, preferredSurface: "both", supportsOriginSurface: true, @@ -142,7 +148,7 @@ describe("inline/native approvals", () => { notifyOriginWhenDmOnly: true, }) expect( - await inlineApprovalCapability.native?.resolveOriginTarget?.({ + await inlineApprovalCapability.nativeAsync!.resolveOriginTarget!({ cfg, accountId: "default", approvalKind: "exec", @@ -150,7 +156,7 @@ describe("inline/native approvals", () => { }), ).toEqual({ to: "chat:7", threadId: "8" }) expect( - await inlineApprovalCapability.native?.resolveApproverDmTargets?.({ + await inlineApprovalCapability.nativeAsync!.resolveApproverDmTargets!({ cfg, accountId: "default", approvalKind: "exec", @@ -159,6 +165,64 @@ describe("inline/native approvals", () => { ).toEqual([{ to: "user:51" }]) }) + it("matches approval requests to Inline accounts asynchronously", async () => { + const accounts = { + a: { token: "token-a", execApprovals: { approvers: ["51"] } }, + b: { token: "token-b", execApprovals: { approvers: ["52"] } }, + } + const cfg = { channels: { inline: { accounts } } } as unknown as OpenClawConfig + const handles = (config: OpenClawConfig, accountId: string, request: ExecApprovalRequest) => + shouldHandleInlineExecApprovalRequest({ cfg: config, accountId, request }) + + // Bound turn-source account wins. + const bound = buildExecRequest({ turnSourceAccountId: "b" }) + const boundResult = handles(cfg, "b", bound) + expect(boundResult).toBeInstanceOf(Promise) + await expect(boundResult).resolves.toBe(true) + await expect(handles(cfg, "a", bound)).resolves.toBe(false) + + // Explicit Inline forwarding targets scope delivery to their accounts. + const targeted = { + ...cfg, + approvals: { + exec: { + enabled: true, + mode: "targets", + targets: [{ channel: "inline", to: "user:52", accountId: "b" }], + }, + }, + } as unknown as OpenClawConfig + const unbound = buildExecRequest({ turnSourceAccountId: undefined }) + await expect(handles(targeted, "b", unbound)).resolves.toBe(true) + await expect(handles(targeted, "a", unbound)).resolves.toBe(false) + + // Foreign-channel requests without a bound account only route when exactly + // one Inline account is eligible. + const foreign = buildExecRequest({ + turnSourceChannel: "slack", + turnSourceAccountId: undefined, + turnSourceTo: "channel:C1", + }) + await expect(handles(cfg, "a", foreign)).resolves.toBe(false) + const single = { + channels: { inline: { accounts: { a: accounts.a } } }, + } as unknown as OpenClawConfig + await expect(handles(single, "a", foreign)).resolves.toBe(true) + + // The runtime availability gate awaits the same account matching. + const context = { client: {} } + await expect( + Promise.resolve(inlineApprovalNativeRuntime.availability.shouldHandle({ + cfg, accountId: "a", context, request: bound, approvalKind: "exec", + })), + ).resolves.toBe(false) + await expect( + Promise.resolve(inlineApprovalNativeRuntime.availability.shouldHandle({ + cfg, accountId: "b", context, request: bound, approvalKind: "exec", + })), + ).resolves.toBe(true) + }) + it("renders, delivers, and clears Inline approval buttons", async () => { const sendTyping = vi.fn(async () => {}) const sendMessage = vi.fn(async () => ({ messageId: 9001n })) diff --git a/plugins/openclaw/src/inline/approval-native.ts b/plugins/openclaw/src/inline/approval-native.ts index 6936c1ae7..ff3704911 100644 --- a/plugins/openclaw/src/inline/approval-native.ts +++ b/plugins/openclaw/src/inline/approval-native.ts @@ -1,13 +1,13 @@ import { - createApproverRestrictedNativeApprovalCapability, + createApproverRestrictedNativeApprovalCapabilityAsync, splitChannelApprovalCapability, } from "openclaw/plugin-sdk/approval-delivery-runtime" -import { createLazyChannelApprovalNativeRuntimeAdapter } from "openclaw/plugin-sdk/approval-handler-adapter-runtime" -import type { ChannelApprovalNativeRuntimeAdapter } from "openclaw/plugin-sdk/approval-handler-runtime" +import { createLazyChannelApprovalNativeRuntimeAdapterAsync } from "openclaw/plugin-sdk/approval-handler-adapter-runtime" import { - createChannelApproverDmTargetResolver, - createChannelNativeOriginTargetResolver, + createChannelApproverDmTargetResolverAsync, + createChannelNativeOriginTargetResolverAsync, resolveApprovalRequestSessionConversation, + type ApprovalRequestSessionDelivery, } from "openclaw/plugin-sdk/approval-native-runtime" import { buildExecApprovalPendingReplyPayload, @@ -38,6 +38,7 @@ import { } from "./exec-approvals.js" type InlineOriginTarget = { to: string; threadId?: string } +type InlineApprovalNativeRuntimeAdapter = NonNullable function accountParams(params: { cfg: OpenClawConfig @@ -64,10 +65,17 @@ function requestParams(params: { cfg: OpenClawConfig accountId?: string | null | undefined request: ApprovalRequest -}): { cfg: OpenClawConfig; accountId?: string | null; request: ApprovalRequest } { + sessionDelivery?: ApprovalRequestSessionDelivery | null | undefined +}): { + cfg: OpenClawConfig + accountId?: string | null + request: ApprovalRequest + sessionDelivery?: ApprovalRequestSessionDelivery | null +} { return { ...accountParams(params), request: params.request, + ...(params.sessionDelivery !== undefined ? { sessionDelivery: params.sessionDelivery } : {}), } } @@ -173,10 +181,10 @@ function resolveInlineFallbackOriginTarget(request: ApprovalRequest): InlineOrig } } -const resolveInlineOriginTarget = createChannelNativeOriginTargetResolver({ +const resolveInlineOriginTarget = createChannelNativeOriginTargetResolverAsync({ channel: "inline", - shouldHandleRequest: ({ cfg, accountId, request }) => - shouldHandleInlineExecApprovalRequest(requestParams({ cfg, accountId, request })), + shouldHandleRequest: ({ cfg, accountId, request, sessionDelivery }) => + shouldHandleInlineExecApprovalRequest(requestParams({ cfg, accountId, request, sessionDelivery })), resolveTurnSourceTarget: resolveTurnSourceInlineOriginTarget, resolveSessionTarget: resolveSessionInlineOriginTarget, normalizeTargetForMatch: normalizeInlineOriginTarget, @@ -184,9 +192,9 @@ const resolveInlineOriginTarget = createChannelNativeOriginTargetResolver({ resolveFallbackTarget: resolveInlineFallbackOriginTarget, }) -const resolveInlineApproverDmTargets = createChannelApproverDmTargetResolver({ - shouldHandleRequest: ({ cfg, accountId, request }) => - shouldHandleInlineExecApprovalRequest(requestParams({ cfg, accountId, request })), +const resolveInlineApproverDmTargets = createChannelApproverDmTargetResolverAsync({ + shouldHandleRequest: ({ cfg, accountId, request, sessionDelivery }) => + shouldHandleInlineExecApprovalRequest(requestParams({ cfg, accountId, request, sessionDelivery })), resolveApprovers: getInlineExecApprovalApprovers, mapApprover: (approver) => ({ to: `user:${approver}` }), }) @@ -199,7 +207,7 @@ function describeInlineExecApprovalSetup(params: { accountId?: string | null }): return `Approve it from the Web UI or terminal UI for now. Inline supports native exec approvals for this account. Configure \`${prefix}.execApprovals.approvers\` or \`commands.ownerAllowFrom\`; leave \`${prefix}.execApprovals.enabled\` unset/\`auto\` or set it to \`true\`.` } -const inlineNativeApprovalCapability = createApproverRestrictedNativeApprovalCapability({ +const inlineNativeApprovalCapability = createApproverRestrictedNativeApprovalCapabilityAsync({ channel: "inline", channelLabel: "Inline", describeExecApprovalSetup: describeInlineExecApprovalSetup, @@ -210,7 +218,7 @@ const inlineNativeApprovalCapability = createApproverRestrictedNativeApprovalCap isInlineExecApprovalAuthorizedSender(senderParams({ cfg, accountId, senderId })), isPluginAuthorizedSender: ({ cfg, accountId, senderId }) => isInlineExecApprovalApprover(senderParams({ cfg, accountId, senderId })), - isNativeDeliveryEnabled: ({ cfg, accountId }) => + isNativeDeliveryEnabled: async ({ cfg, accountId }) => isInlineExecApprovalClientEnabled(accountParams({ cfg, accountId })), resolveNativeDeliveryMode: ({ cfg, accountId }) => resolveInlineExecApprovalTarget(accountParams({ cfg, accountId })), @@ -221,14 +229,14 @@ const inlineNativeApprovalCapability = createApproverRestrictedNativeApprovalCap resolveOriginTarget: resolveInlineOriginTarget, resolveApproverDmTargets: resolveInlineApproverDmTargets, notifyOriginWhenDmOnly: true, - nativeRuntime: createLazyChannelApprovalNativeRuntimeAdapter({ + nativeRuntimeAsync: createLazyChannelApprovalNativeRuntimeAdapterAsync({ eventKinds: INLINE_APPROVAL_EVENT_KINDS, isConfigured: ({ cfg, accountId }) => isInlineExecApprovalClientEnabled(accountParams({ cfg, accountId })), shouldHandle: ({ cfg, accountId, request }) => shouldHandleInlineExecApprovalRequest(requestParams({ cfg, accountId, request })), load: async () => - loadBundledEntryExportSync(import.meta.url, { + loadBundledEntryExportSync(import.meta.url, { specifier: "./approval-handler.runtime.js", exportName: "inlineApprovalNativeRuntime", }), diff --git a/plugins/openclaw/src/inline/bot-avatar-tool.ts b/plugins/openclaw/src/inline/bot-avatar-tool.ts index 5371f9d07..d9f1b51b7 100644 --- a/plugins/openclaw/src/inline/bot-avatar-tool.ts +++ b/plugins/openclaw/src/inline/bot-avatar-tool.ts @@ -5,6 +5,7 @@ import os from "node:os" import type { AnyAgentTool, OpenClawConfig } from "openclaw/plugin-sdk/core" import { BotAvatar_Kind, InlineSdkClient, Method } from "@inline-chat/realtime-sdk" import { resolveInlineAccount, resolveInlineToken } from "./accounts.js" +import { withInlineAccountClient } from "./client-registry.js" import { sanitizeInlineVisibleText } from "./outbound-sanitize.js" import { jsonResult } from "../openclaw-compat.js" import { getInlineRuntime } from "../runtime.js" @@ -418,16 +419,10 @@ async function withInlineClient(params: { throw new Error(`Inline not configured for account "${account.accountId}" (missing token or baseUrl)`) } const token = await resolveInlineToken(account) - const client = new InlineSdkClient({ - baseUrl: account.baseUrl, - token, - }) - await client.connect() - try { - return await params.fn(client, account.accountId) - } finally { - await client.close().catch(() => {}) - } + return await withInlineAccountClient( + { accountId: account.accountId, baseUrl: account.baseUrl, token }, + (client) => params.fn(client, account.accountId), + ) } export function createInlineBotAvatarTool(ctx: { diff --git a/plugins/openclaw/src/inline/bot-commands-sync.test.ts b/plugins/openclaw/src/inline/bot-commands-sync.test.ts index 6f76f8374..797f46760 100644 --- a/plugins/openclaw/src/inline/bot-commands-sync.test.ts +++ b/plugins/openclaw/src/inline/bot-commands-sync.test.ts @@ -298,7 +298,7 @@ describe("inline/bot-commands-sync", () => { it("scopes native skill commands per account route", async () => { vi.resetModules() - const listSkillCommandsForAgents = vi.fn( + const prepareSkillCommandsForAgents = vi.fn( ({ agentIds }: { agentIds?: string[] }) => agentIds?.[0] === "ops" ? [{ name: "ops_skill", description: "Ops skill" }] @@ -318,7 +318,7 @@ describe("inline/bot-commands-sync", () => { const actual = await vi.importActual>("openclaw/plugin-sdk/skill-commands-runtime") return { ...actual, - listSkillCommandsForAgents, + prepareSkillCommandsForAgents, } }) vi.doMock("openclaw/plugin-sdk/native-command-registry", async () => { @@ -372,11 +372,11 @@ describe("inline/bot-commands-sync", () => { }, }) - expect(listSkillCommandsForAgents).toHaveBeenCalledWith({ + expect(prepareSkillCommandsForAgents).toHaveBeenCalledWith({ cfg: expect.any(Object), agentIds: ["main"], }) - expect(listSkillCommandsForAgents).toHaveBeenCalledWith({ + expect(prepareSkillCommandsForAgents).toHaveBeenCalledWith({ cfg: expect.any(Object), agentIds: ["ops"], }) @@ -432,7 +432,7 @@ describe("inline/bot-commands-sync", () => { it("uses plugin-sdk native/skill/plugin command helpers when available", async () => { vi.resetModules() - const listSkillCommandsForAgents = vi.fn(() => [ + const prepareSkillCommandsForAgents = vi.fn(() => [ { name: "weather_skill", description: "Weather skill" }, ]) const listNativeCommandSpecsForConfig = vi.fn( @@ -460,7 +460,7 @@ describe("inline/bot-commands-sync", () => { const actual = await vi.importActual>("openclaw/plugin-sdk/skill-commands-runtime") return { ...actual, - listSkillCommandsForAgents, + prepareSkillCommandsForAgents, } }) vi.doMock("openclaw/plugin-sdk/native-command-registry", async () => { @@ -510,7 +510,7 @@ describe("inline/bot-commands-sync", () => { logger, }) - expect(listSkillCommandsForAgents).toHaveBeenCalledWith({ + expect(prepareSkillCommandsForAgents).toHaveBeenCalledWith({ cfg: expect.any(Object), agentIds: ["main"], }) diff --git a/plugins/openclaw/src/inline/bot-commands-sync.ts b/plugins/openclaw/src/inline/bot-commands-sync.ts index ce8a76d79..20b0b0938 100644 --- a/plugins/openclaw/src/inline/bot-commands-sync.ts +++ b/plugins/openclaw/src/inline/bot-commands-sync.ts @@ -2,7 +2,7 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/core" import { listNativeCommandSpecsForConfig } from "openclaw/plugin-sdk/native-command-registry" import { getPluginCommandSpecs } from "openclaw/plugin-sdk/plugin-runtime" import { resolveAgentRoute } from "openclaw/plugin-sdk/routing" -import { listSkillCommandsForAgents } from "openclaw/plugin-sdk/skill-commands-runtime" +import { prepareSkillCommandsForAgents } from "openclaw/plugin-sdk/skill-commands-runtime" import { findInlineTokenOwnerAccountId, formatDuplicateInlineTokenReason, @@ -191,7 +191,7 @@ async function buildInlineNativeCommandsForConfig(params: { : null const skillCommands = route - ? listSkillCommandsForAgents({ + ? await prepareSkillCommandsForAgents({ cfg: params.cfg, agentIds: [route.agentId], }) diff --git a/plugins/openclaw/src/inline/bot-skills-sync.test.ts b/plugins/openclaw/src/inline/bot-skills-sync.test.ts index 948049256..16394b355 100644 --- a/plugins/openclaw/src/inline/bot-skills-sync.test.ts +++ b/plugins/openclaw/src/inline/bot-skills-sync.test.ts @@ -17,7 +17,7 @@ describe("inline/bot-skills-sync", () => { resolveAgentRoute: () => ({ agentId: "main" }), })) vi.doMock("openclaw/plugin-sdk/skill-commands-runtime", () => ({ - listSkillCommandsForAgents: () => [ + prepareSkillCommandsForAgents: () => [ { name: "data_analysis", displayName: "Data Analysis", @@ -73,7 +73,7 @@ describe("inline/bot-skills-sync", () => { resolveAgentRoute: () => ({ agentId: "main" }), })) vi.doMock("openclaw/plugin-sdk/skill-commands-runtime", () => ({ - listSkillCommandsForAgents: () => [], + prepareSkillCommandsForAgents: () => [], })) const fetchMock = vi.fn(async () => new Response(JSON.stringify({ ok: true, result: {} }), { diff --git a/plugins/openclaw/src/inline/bot-skills-sync.ts b/plugins/openclaw/src/inline/bot-skills-sync.ts index af79f043c..3d14fdafd 100644 --- a/plugins/openclaw/src/inline/bot-skills-sync.ts +++ b/plugins/openclaw/src/inline/bot-skills-sync.ts @@ -1,6 +1,6 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/core" import { resolveAgentRoute } from "openclaw/plugin-sdk/routing" -import { listSkillCommandsForAgents } from "openclaw/plugin-sdk/skill-commands-runtime" +import { prepareSkillCommandsForAgents } from "openclaw/plugin-sdk/skill-commands-runtime" import { findInlineTokenOwnerAccountId, formatDuplicateInlineTokenReason, @@ -36,16 +36,16 @@ function truncateUtf16(value: string, maxLength: number): string { : truncated } -export function buildInlinePublishedSkills(params: { +export async function buildInlinePublishedSkills(params: { cfg: OpenClawConfig accountId: string -}): InlinePublishedSkill[] { +}): Promise { const route = resolveAgentRoute({ cfg: params.cfg, channel: "inline", accountId: params.accountId, }) - const commands = listSkillCommandsForAgents({ + const commands = await prepareSkillCommandsForAgents({ cfg: params.cfg, agentIds: [route.agentId], }) @@ -99,7 +99,7 @@ export async function syncInlineAgentSkills(params: { try { const token = await resolveInlineToken(account) - const skills = buildInlinePublishedSkills({ cfg: params.cfg, accountId: account.accountId }) + const skills = await buildInlinePublishedSkills({ cfg: params.cfg, accountId: account.accountId }) await callInlineBotApi>({ baseUrl: account.baseUrl, token, diff --git a/plugins/openclaw/src/inline/channel.test.ts b/plugins/openclaw/src/inline/channel.test.ts index a2f00e37a..62183cdd1 100644 --- a/plugins/openclaw/src/inline/channel.test.ts +++ b/plugins/openclaw/src/inline/channel.test.ts @@ -281,8 +281,8 @@ describe("inline/channel", () => { expect(inlineChannelPlugin.threading).toBeDefined() expect(inlineChannelPlugin.setup).toBeDefined() expect(inlineChannelPlugin.setupWizard).toBeDefined() - expect(inlineChannelPlugin.approvalCapability?.nativeRuntime).toBeDefined() - expect(inlineChannelPlugin.approvalCapability?.native?.describeDeliveryCapabilities?.({ + expect(inlineChannelPlugin.approvalCapability?.nativeRuntimeAsync).toBeDefined() + await expect(inlineChannelPlugin.approvalCapability?.nativeAsync?.describeDeliveryCapabilities({ cfg: { channels: { inline: { @@ -305,7 +305,7 @@ describe("inline/channel", () => { createdAtMs: 1, expiresAtMs: 2, } as any, - })).toMatchObject({ + })).resolves.toMatchObject({ enabled: true, preferredSurface: "approver-dm", supportsOriginSurface: true, @@ -337,6 +337,8 @@ describe("inline/channel", () => { selects: true, context: true, divider: false, + charts: false, + tables: false, }) expect(inlineChannelPlugin.outbound.extractMarkdownImages).toBe(true) expect(inlineChannelPlugin.outbound.preferFinalAssistantVisibleText).toBe(true) @@ -2326,11 +2328,11 @@ describe("inline/channel", () => { retryable: true, }) - expect(connect).toHaveBeenCalledWith(abortController.signal) + expect(connect).not.toHaveBeenCalled() expect(onPlatformSendDispatch).not.toHaveBeenCalled() expect(assertDirectAdapterHandoff).not.toHaveBeenCalled() expect(sendMessage).not.toHaveBeenCalled() - expect(close).toHaveBeenCalled() + expect(close).not.toHaveBeenCalled() }) it.each(["text", "media"] as const)("message %s cancels during dispatch refresh before SDK handoff", async (kind) => { @@ -3517,12 +3519,40 @@ describe("inline/channel", () => { presentation: { blocks: [ { type: "text", text: "Approve deploy?" }, - { type: "buttons", buttons: [{ label: "Approve", value: "approve" }] }, + { + type: "buttons", + buttons: [ + { label: "Approve", action: { type: "callback", value: "approve" } }, + { label: "Status", action: { type: "command", command: "/status" } }, + { label: "Docs", action: { type: "url", url: "https://inline.chat/docs" } }, + ], + }, + { + type: "select", + placeholder: "Model", + options: [{ label: "Fast", action: { type: "callback", value: "model:fast" } }], + }, ], }, ctx: {} as any, } as any) + expect(rendered).toEqual({ + text: "Approve deploy?\n\n- Approve\n- Status: `/status`\n- Docs: https://inline.chat/docs\n\nModel:\n- Fast", + channelData: { + inline: { + buttons: [ + [ + { text: "Approve", callback_data: "approve" }, + { text: "Status", callback_data: "/status", callback_data_kind: "command" }, + ], + [{ text: "Fast", callback_data: "model:fast" }], + ], + }, + }, + }) + expect(rendered).not.toHaveProperty("interactive") + await inlineChannelPlugin.outbound.sendPayload?.({ cfg: { channels: { @@ -3547,8 +3577,54 @@ describe("inline/channel", () => { }), ) expect(firstArg?.actions?.rows?.[0]?.actions?.[0]?.text).toBe("Approve") + expect(firstArg?.actions?.rows?.[0]?.actions?.[0]?.actionId).toBe("agent:1:1") const callbackData = firstArg?.actions?.rows?.[0]?.actions?.[0]?.action.callback.data expect(Buffer.from(callbackData).toString()).toBe("approve") + expect(firstArg?.actions?.rows?.[0]?.actions?.[1]?.actionId).toBe("system:1:2") + expect(firstArg?.actions?.rows?.[0]?.actions).toHaveLength(2) + expect(firstArg?.actions?.rows?.[1]?.actions?.[0]?.text).toBe("Fast") + }) + + it("outbound renderPresentation keeps explicit Inline buttons over presentation controls", async () => { + vi.resetModules() + mockRealtimeSdk({ InlineSdkClient: class {} }) + await setInlineTestRuntime() + + const { inlineChannelPlugin } = await import("./channel") + + const explicitButtons = [[{ text: "Keep", callback_data: "keep" }]] + const rendered = await inlineChannelPlugin.outbound.renderPresentation?.({ + payload: { text: "Pick", channelData: { inline: { buttons: explicitButtons } } }, + presentation: { + blocks: [{ type: "buttons", buttons: [{ label: "Drop", action: { type: "callback", value: "drop" } }] }], + }, + ctx: {} as any, + } as any) + + expect(rendered?.channelData).toEqual({ inline: { buttons: explicitButtons } }) + expect(rendered?.text).toBe("Pick\n\n- Drop") + }) + + it("outbound renderPresentation leaves channelData alone when no control maps to an Inline button", async () => { + vi.resetModules() + mockRealtimeSdk({ InlineSdkClient: class {} }) + await setInlineTestRuntime() + + const { inlineChannelPlugin } = await import("./channel") + + const rendered = await inlineChannelPlugin.outbound.renderPresentation?.({ + payload: {}, + presentation: { + title: "Release", + blocks: [ + { type: "context", text: "Build 12" }, + { type: "buttons", buttons: [{ label: "Notes", action: { type: "url", url: "https://inline.chat" } }] }, + ], + }, + ctx: {} as any, + } as any) + + expect(rendered).toEqual({ text: "Release\n\nBuild 12\n\n- Notes: https://inline.chat" }) }) it("outbound sendPayload uses interactive labels as fallback text", async () => { diff --git a/plugins/openclaw/src/inline/channel.ts b/plugins/openclaw/src/inline/channel.ts index 38c141d4b..624b89ecc 100644 --- a/plugins/openclaw/src/inline/channel.ts +++ b/plugins/openclaw/src/inline/channel.ts @@ -15,8 +15,8 @@ import { } from "openclaw/plugin-sdk/channel-outbound" import { PlatformMessageNotDispatchedError } from "openclaw/plugin-sdk/error-runtime" import { - presentationToInteractiveReply, renderMessagePresentationFallbackText, + type MessagePresentation, } from "openclaw/plugin-sdk/interactive-runtime" import { buildDmGroupAccountAllowlistAdapter, @@ -69,6 +69,12 @@ import { resolveInlineGroupToolPolicy, } from "./policy.js" import { + acquireInlineAccountClient, + withInlineAccountClient, + type InlineClientLease, +} from "./client-registry.js" +import { + buildInlinePresentationButtons, inlineMessageActions, resolveInlineMessageActionsParam, supportsInlineMessageButtonsForConfig, @@ -725,6 +731,32 @@ function resolveInlinePayloadActions(payload: Record): MessageA return resolveInlineMessageActionsParam(payload) } +/** + * Carry rendered presentation controls to sendPayload as Inline button rows. + * Explicit Inline or Telegram-compatible buttons already on the payload win, + * matching resolveInlinePayloadActions precedence. + */ +function resolveInlinePresentationChannelData( + channelData: Record | undefined, + presentation: MessagePresentation, +): { channelData?: Record } { + const inlineData = asRecord(channelData?.inline) + const telegramData = asRecord(channelData?.telegram) + if (Object.hasOwn(inlineData ?? {}, "buttons") || Object.hasOwn(telegramData ?? {}, "buttons")) { + return {} + } + const buttons = buildInlinePresentationButtons(presentation) + if (buttons.length === 0) { + return {} + } + return { + channelData: { + ...channelData, + inline: { ...inlineData, buttons }, + }, + } +} + function formatInlineResultChatId(target: InlineParsedOutboundTarget): string { if (target.kind === "user") { return `user:${target.normalizedNumeric}` @@ -847,16 +879,10 @@ async function withInlineClient(params: { throw new Error(`Inline not configured for account "${account.accountId}" (missing token or baseUrl)`) } const token = await resolveInlineToken(account) - const client = new InlineSdkClient({ - baseUrl: account.baseUrl, - token, - }) - await client.connect() - try { - return await params.fn(client, account) - } finally { - await client.close().catch(() => {}) - } + return await withInlineAccountClient( + { accountId: account.accountId, baseUrl: account.baseUrl, token }, + (client) => params.fn(client, account), + ) } async function sendTypingInline(params: { @@ -916,22 +942,16 @@ async function notifyPairingApprovedInline(params: { } const token = await resolveInlineToken(account) - - const client = new InlineSdkClient({ - baseUrl: account.baseUrl, - token, - }) - - await client.connect() - try { - await client.sendMessage({ - userId, - text: PAIRING_APPROVED_MESSAGE, - parseMarkdown: account.config.parseMarkdown ?? true, - }) - } finally { - await client.close().catch(() => {}) - } + await withInlineAccountClient( + { accountId: account.accountId, baseUrl: account.baseUrl, token }, + async (client) => { + await client.sendMessage({ + userId, + text: PAIRING_APPROVED_MESSAGE, + parseMarkdown: account.config.parseMarkdown ?? true, + }) + }, + ) } async function sendMessageInline(params: { @@ -946,6 +966,7 @@ async function sendMessageInline(params: { } & InlinePlatformSendContext): Promise { let platformSendStarted = false let client: InlineSdkClient | null = null + let lease: InlineClientLease | null = null try { const account = resolveInlineAccount({ cfg: params.cfg, accountId: params.accountId ?? null }) if (!account.configured || !account.baseUrl) { @@ -966,11 +987,13 @@ async function sendMessageInline(params: { } const text = sanitizeInlineOutgoingText(visibleText.text) - client = new InlineSdkClient({ + lease = await acquireInlineAccountClient({ + accountId: account.accountId, baseUrl: account.baseUrl, token, + signal: params.signal, }) - await client.connect(params.signal) + client = lease.client // Inline "threads" are modeled as chats (chatId). OpenClaw's threadId is not a message id. // Only map OpenClaw replyToId -> Inline replyToMsgId. @@ -1023,7 +1046,7 @@ async function sendMessageInline(params: { if (!platformSendStarted) throw asInlinePlatformNotDispatched(error) throw error } finally { - await client?.close().catch(() => {}) + await lease?.release() } } @@ -1043,6 +1066,7 @@ async function sendMediaInline(params: { } & InlinePlatformSendContext): Promise { let platformSendStarted = false let client: InlineSdkClient | null = null + let lease: InlineClientLease | null = null try { const account = resolveInlineAccount({ cfg: params.cfg, accountId: params.accountId ?? null }) if (!account.configured || !account.baseUrl) { @@ -1057,11 +1081,13 @@ async function sendMediaInline(params: { const visibleText = sanitizeInlineVisibleText(params.text) const caption = visibleText.shouldSkip ? "" : sanitizeInlineOutgoingText(visibleText.text).trim() - client = new InlineSdkClient({ + lease = await acquireInlineAccountClient({ + accountId: account.accountId, baseUrl: account.baseUrl, token, + signal: params.signal, }) - await client.connect(params.signal) + client = lease.client const resolvedTarget = await resolveInlineOutboundTarget({ client, @@ -1122,7 +1148,7 @@ async function sendMediaInline(params: { if (!platformSendStarted) throw asInlinePlatformNotDispatched(error) throw error } finally { - await client?.close().catch(() => {}) + await lease?.release() } } @@ -1296,17 +1322,19 @@ const inlineOutbound: NonNullable["outbound selects: true, context: true, divider: false, + charts: false, + tables: false, }, renderPresentation: ({ payload, presentation }) => { const text = renderMessagePresentationFallbackText({ ...(payload.text !== undefined ? { text: payload.text } : {}), presentation, }) - const interactive = presentationToInteractiveReply(presentation) + const { presentation: _presentation, presentationTextMode: _textMode, ...rest } = payload return { - ...payload, + ...rest, text, - ...(interactive ? { interactive } : {}), + ...resolveInlinePresentationChannelData(payload.channelData, presentation), } }, sendPayload: async (context) => { @@ -1331,8 +1359,10 @@ const inlineOutbound: NonNullable["outbound const text = resolveInlineInteractiveTextFallback({ text: payload.text ?? undefined, - interactive: payload.interactive, + // Untyped legacy input from producers that still emit `interactive`. + interactive: (payload as Record).interactive, presentation: payload.presentation, + ...(payload.presentationTextMode === "fallback" ? { presentationTextMode: "fallback" as const } : {}), }) ?? payload.text ?? "" diff --git a/plugins/openclaw/src/inline/client-registry.test.ts b/plugins/openclaw/src/inline/client-registry.test.ts new file mode 100644 index 000000000..f9ff122db --- /dev/null +++ b/plugins/openclaw/src/inline/client-registry.test.ts @@ -0,0 +1,128 @@ +import { afterEach, describe, expect, it, vi } from "vitest" + +const created: Array<{ connect: ReturnType; close: ReturnType }> = [] + +let failNextConnect = false + +vi.mock("@inline-chat/realtime-sdk", () => ({ + InlineSdkClient: class { + connect = vi.fn(async () => { + if (failNextConnect) { + failNextConnect = false + throw new Error("offline") + } + }) + close = vi.fn(async () => {}) + constructor() { + created.push(this) + } + }, +})) + +const { + acquireInlineAccountClient, + resetSharedInlineClientsForTest, + shareInlineAccountClient, + withInlineAccountClient, +} = await import("./client-registry.js") + +const account = { accountId: "default", baseUrl: "https://api.inline.chat", token: "t1" } + +afterEach(() => { + resetSharedInlineClientsForTest() + created.length = 0 +}) + +describe("inline client registry", () => { + it("reuses the monitor's client and never closes it", async () => { + const monitorClient = { close: vi.fn() } as any + shareInlineAccountClient({ ...account, client: monitorClient }) + + const seen = await withInlineAccountClient(account, async (client) => client) + + expect(seen).toBe(monitorClient) + expect(created).toHaveLength(0) + expect(monitorClient.close).not.toHaveBeenCalled() + }) + + it("opens and closes a short-lived client when no monitor is running", async () => { + await withInlineAccountClient(account, async () => "ok") + + expect(created).toHaveLength(1) + expect(created[0]?.connect).toHaveBeenCalledTimes(1) + expect(created[0]?.close).toHaveBeenCalledTimes(1) + }) + + it("doesn't reuse a client shared under another token or server", async () => { + shareInlineAccountClient({ ...account, client: {} as any }) + + await withInlineAccountClient({ ...account, token: "t2" }, async () => {}) + await withInlineAccountClient({ ...account, baseUrl: "https://staging.inline.chat" }, async () => {}) + + expect(created).toHaveLength(2) + }) + + it("stops sharing once the monitor unshares, even for a lease taken before", async () => { + const unshare = shareInlineAccountClient({ ...account, client: {} as any }) + const lease = await acquireInlineAccountClient(account) + unshare() + await lease.release() + + await withInlineAccountClient(account, async () => {}) + + expect(created).toHaveLength(1) + }) + + it("keeps a newer monitor's client when an older one unshares", async () => { + const unshareOld = shareInlineAccountClient({ ...account, client: { id: "old" } as any }) + const newer = { id: "new" } as any + shareInlineAccountClient({ ...account, client: newer }) + unshareOld() + + const seen = await withInlineAccountClient(account, async (client) => client) + + expect(seen).toBe(newer) + }) + + it("rejects an aborted lease without borrowing or opening a client", async () => { + const monitorClient = { close: vi.fn() } as any + shareInlineAccountClient({ ...account, client: monitorClient }) + const controller = new AbortController() + controller.abort(new Error("cancelled")) + const operation = vi.fn() + + await expect(withInlineAccountClient({ ...account, signal: controller.signal }, operation)).rejects.toThrow("cancelled") + expect(operation).not.toHaveBeenCalled() + expect(created).toHaveLength(0) + expect(monitorClient.close).not.toHaveBeenCalled() + }) + + it("rechecks cancellation between borrowing a lease and invoking its operation", async () => { + const monitorClient = { close: vi.fn() } as any + shareInlineAccountClient({ ...account, client: monitorClient }) + const controller = new AbortController() + const operation = vi.fn() + const result = withInlineAccountClient({ ...account, signal: controller.signal }, operation) + controller.abort(new Error("cancelled after borrowing")) + + await expect(result).rejects.toThrow("cancelled after borrowing") + expect(operation).not.toHaveBeenCalled() + expect(monitorClient.close).not.toHaveBeenCalled() + }) + + it("closes a newly connected client if its caller cancels during connect", async () => { + const controller = new AbortController() + const lease = acquireInlineAccountClient({ ...account, signal: controller.signal }) + controller.abort(new Error("cancelled during connect")) + + await expect(lease).rejects.toThrow("cancelled during connect") + expect(created[0]?.close).toHaveBeenCalledTimes(1) + }) + + it("closes a client whose connect failed", async () => { + failNextConnect = true + + await expect(withInlineAccountClient(account, async () => {})).rejects.toThrow("offline") + expect(created[0]?.close).toHaveBeenCalledTimes(1) + }) +}) diff --git a/plugins/openclaw/src/inline/client-registry.ts b/plugins/openclaw/src/inline/client-registry.ts new file mode 100644 index 000000000..cc5d7661c --- /dev/null +++ b/plugins/openclaw/src/inline/client-registry.ts @@ -0,0 +1,100 @@ +import { InlineSdkClient } from "@inline-chat/realtime-sdk" + +/** + * One realtime connection per account. The monitor shares its live client so + * tools, outbound sends and commands reuse it instead of opening a connection + * per call. Callers outside a running monitor (CLI, setup, probe) still get a + * short-lived client that is closed after use. + */ +type SharedInlineClient = { + client: InlineSdkClient + baseUrl: string + token: string + active: boolean +} + +// The build bundles each plugin entry separately (no code splitting), so the +// monitor (channel entry) and the tools (runtime-register entry) each carry their +// own copy of this module. A process-wide key gives them one map. +const SHARED_CLIENTS_KEY = Symbol.for("inline-openclaw.sharedClients.v1") +const sharedClients: Map = ((globalThis as Record)[ + SHARED_CLIENTS_KEY +] ??= new Map()) as Map + +export function shareInlineAccountClient(params: { + accountId: string + baseUrl: string + token: string + client: InlineSdkClient +}): () => void { + const entry: SharedInlineClient = { + client: params.client, + baseUrl: params.baseUrl, + token: params.token, + active: true, + } + sharedClients.set(params.accountId, entry) + return () => { + entry.active = false + if (sharedClients.get(params.accountId) === entry) { + sharedClients.delete(params.accountId) + } + } +} + +export function getSharedInlineAccountClient(params: { + accountId: string + baseUrl: string + token: string +}): InlineSdkClient | undefined { + const entry = sharedClients.get(params.accountId) + if (!entry?.active) return undefined + // A token or server change means the monitor is about to restart; don't + // send with the old identity. + if (entry.baseUrl !== params.baseUrl || entry.token !== params.token) return undefined + return entry.client +} + +export type InlineClientLease = { + client: InlineSdkClient + /** Closes the client only when this lease opened it; the shared one stays up. */ + release: () => Promise +} + +export async function acquireInlineAccountClient(params: { + accountId: string + baseUrl: string + token: string + signal?: AbortSignal | undefined +}): Promise { + params.signal?.throwIfAborted() + const shared = getSharedInlineAccountClient(params) + if (shared) return { client: shared, release: async () => {} } + + const client = new InlineSdkClient({ baseUrl: params.baseUrl, token: params.token }) + try { + await client.connect(params.signal) + params.signal?.throwIfAborted() + } catch (error) { + await client.close().catch(() => {}) + throw error + } + return { client, release: async () => { await client.close().catch(() => {}) } } +} + +export async function withInlineAccountClient( + params: { accountId: string; baseUrl: string; token: string; signal?: AbortSignal | undefined }, + fn: (client: InlineSdkClient) => Promise, +): Promise { + const lease = await acquireInlineAccountClient(params) + try { + params.signal?.throwIfAborted() + return await fn(lease.client) + } finally { + await lease.release() + } +} + +export function resetSharedInlineClientsForTest(): void { + sharedClients.clear() +} diff --git a/plugins/openclaw/src/inline/compose-action.ts b/plugins/openclaw/src/inline/compose-action.ts index 2b25b1502..30a4bfed3 100644 --- a/plugins/openclaw/src/inline/compose-action.ts +++ b/plugins/openclaw/src/inline/compose-action.ts @@ -1,8 +1,4 @@ -import { Method, type InlineSdkClient, type UpdateComposeAction_ComposeAction } from "@inline-chat/realtime-sdk" - -// ComposeAction.WORKING = 6. The pinned SDK predates the named enum value. The -// server shows typing to clients that have not opted in to the Working status. -const INLINE_COMPOSE_ACTION_WORKING = 6 as UpdateComposeAction_ComposeAction +import { Method, UpdateComposeAction_ComposeAction, type InlineSdkClient } from "@inline-chat/realtime-sdk" // Agents show "Working" rather than "typing" while they run, so people don't // mistake agent work for someone typing. @@ -14,7 +10,7 @@ export async function sendInlineWorkingAction( oneofKind: "sendComposeAction", sendComposeAction: { peerId: { type: { oneofKind: "chat", chat: { chatId: params.chatId } } }, - ...(params.working ? { action: INLINE_COMPOSE_ACTION_WORKING } : {}), + ...(params.working ? { action: UpdateComposeAction_ComposeAction.WORKING } : {}), }, }) } diff --git a/plugins/openclaw/src/inline/config-schema.ts b/plugins/openclaw/src/inline/config-schema.ts index 36bdb3a38..110599c89 100644 --- a/plugins/openclaw/src/inline/config-schema.ts +++ b/plugins/openclaw/src/inline/config-schema.ts @@ -124,8 +124,8 @@ const InlineStreamingSchema = z.union([ }) .strict(), ]) -const InlineRuntimeStreamingChunkSchema = InlineStreamingChunkSchema.passthrough() -const InlineRuntimeBlockStreamingCoalesceSchema = BlockStreamingCoalesceSchema.passthrough() +const InlineRuntimeStreamingChunkSchema = InlineStreamingChunkSchema.loose() +const InlineRuntimeBlockStreamingCoalesceSchema = BlockStreamingCoalesceSchema.loose() const InlineRuntimeStreamingSchema = z.union([ z.boolean(), InlineStreamingModeSchema, @@ -139,7 +139,7 @@ const InlineRuntimeStreamingSchema = z.union([ toolProgress: z.boolean().optional(), commandText: InlineStreamingCommandTextSchema.optional(), }) - .passthrough() + .loose() .optional(), progress: z .object({ @@ -151,17 +151,17 @@ const InlineRuntimeStreamingSchema = z.union([ toolProgress: z.boolean().optional(), commandText: InlineStreamingCommandTextSchema.optional(), }) - .passthrough() + .loose() .optional(), block: z .object({ enabled: z.boolean().optional(), coalesce: InlineRuntimeBlockStreamingCoalesceSchema.optional(), }) - .passthrough() + .loose() .optional(), }) - .passthrough(), + .loose(), ]) export const InlineAccountSchemaBase = z @@ -223,7 +223,7 @@ export const InlineAccountSchema = InlineAccountSchemaBase.superRefine((value, c const InlineConfigAccountSchema = InlineAccountSchemaBase.extend({ streaming: InlineRuntimeStreamingSchema.optional(), }) - .passthrough() + .loose() .superRefine((value, ctx) => { requireOpenAllowFrom({ policy: value.dmPolicy, @@ -237,17 +237,17 @@ const InlineConfigAccountSchema = InlineAccountSchemaBase.extend({ export const InlineRuntimeAccountSchema = InlineAccountSchemaBase.extend({ streaming: InlineRuntimeStreamingSchema.optional(), -}).passthrough() +}).loose() export const InlineRuntimeConfigSchema = InlineRuntimeAccountSchema.extend({ accounts: z.record(z.string(), InlineRuntimeAccountSchema.optional()).optional(), -}).passthrough() +}).loose() export const InlineConfigSchema = InlineAccountSchemaBase.extend({ streaming: InlineRuntimeStreamingSchema.optional(), accounts: z.record(z.string(), InlineConfigAccountSchema.optional()).optional(), }) - .passthrough() + .loose() .superRefine((value, ctx) => { requireOpenAllowFrom({ policy: value.dmPolicy, diff --git a/plugins/openclaw/src/inline/delivery.test.ts b/plugins/openclaw/src/inline/delivery.test.ts new file mode 100644 index 000000000..73265b1c5 --- /dev/null +++ b/plugins/openclaw/src/inline/delivery.test.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, it, vi } from "vitest" +import { ProtocolClientError } from "@inline-chat/realtime-sdk" +import { createInlineMessageSender } from "./delivery" + +const rateLimit = (seconds: number, code = 3) => new Error("send failed", { + cause: new ProtocolClientError("rpc-error", { code, retryAfterSeconds: seconds }), +}) + +afterEach(() => vi.useRealTimers()) + +describe("monitor delivery", () => { + it("waits the server hint and reuses one send identity", async () => { + vi.useFakeTimers() + const client = { sendMessage: vi.fn().mockRejectedValueOnce(rateLimit(2)).mockResolvedValue({ messageId: 9n }) } + const sending = createInlineMessageSender(client)({ chatId: 7n, text: "Answer" }) + await vi.advanceTimersByTimeAsync(1_999) + expect(client.sendMessage).toHaveBeenCalledTimes(1) + await vi.advanceTimersByTimeAsync(1) + await expect(sending).resolves.toEqual({ messageId: 9n }) + const first = client.sendMessage.mock.calls[0]![0] + const second = client.sendMessage.mock.calls[1]![0] + expect(first).toBe(second) + expect(first.randomId).not.toBe(0n) + expect(typeof first.randomId).toBe("bigint") + }) + + it("preserves a supplied identity and stops after two retries", async () => { + vi.useFakeTimers() + const error = rateLimit(1, 429) + const client = { sendMessage: vi.fn().mockRejectedValue(error) } + const sending = createInlineMessageSender(client)({ chatId: 7n, text: "Answer", randomId: -42n }) + const result = expect(sending).rejects.toBe(error) + await vi.advanceTimersByTimeAsync(2_000) + await result + expect(client.sendMessage).toHaveBeenCalledTimes(3) + expect(client.sendMessage.mock.calls.every(([request]) => request.randomId === -42n)).toBe(true) + }) + + it("cancels a flood wait without retrying or closing the shared client", async () => { + const controller = new AbortController() + const client = { sendMessage: vi.fn().mockRejectedValue(rateLimit(2)), close: vi.fn() } + const sending = createInlineMessageSender(client, controller.signal)({ chatId: 7n, text: "Answer" }) + controller.abort() + await expect(sending).rejects.toMatchObject({ name: "AbortError" }) + expect(client.sendMessage).toHaveBeenCalledTimes(1) + expect(client.close).not.toHaveBeenCalled() + }) + + it.each([ + rateLimit(61), rateLimit(Number.POSITIVE_INFINITY), rateLimit(Number.NaN), rateLimit(0), rateLimit(-1), + new ProtocolClientError("timeout"), new ProtocolClientError("commit-outcome-unknown"), + new Error("rate limit, try again"), new ProtocolClientError("rpc-error", { code: 403, retryAfterSeconds: 1 }), + new ProtocolClientError("rpc-error", { code: 3 }), + new Error("uncertain", { cause: new ProtocolClientError("commit-outcome-unknown", { retryAfterSeconds: 1 }) }), + ])("does not retry an unqualified rejection or uncertain send (%s)", async (error) => { + const client = { sendMessage: vi.fn().mockRejectedValue(error) } + await expect(createInlineMessageSender(client)({ chatId: 7n, text: "Answer" })).rejects.toBe(error) + expect(client.sendMessage).toHaveBeenCalledTimes(1) + }) + + it("rejects a cancelled send before the SDK handoff", async () => { + const controller = new AbortController() + controller.abort(new Error("cancelled")) + const client = { sendMessage: vi.fn() } + await expect(createInlineMessageSender(client, controller.signal)({ chatId: 7n, text: "Answer" })).rejects.toThrow("cancelled") + expect(client.sendMessage).not.toHaveBeenCalled() + }) +}) diff --git a/plugins/openclaw/src/inline/delivery.ts b/plugins/openclaw/src/inline/delivery.ts new file mode 100644 index 000000000..a727fd087 --- /dev/null +++ b/plugins/openclaw/src/inline/delivery.ts @@ -0,0 +1,46 @@ +import { randomBytes } from "node:crypto" +import { setTimeout as delay } from "node:timers/promises" +import type { InlineSdkClient, InlineSdkSendMessageParams } from "@inline-chat/realtime-sdk" + +const MAX_SEND_RETRIES = 2 +const MAX_SEND_WAIT_SECONDS = 60 + +/** Only an explicit server rejection is retry-safe; transport uncertainty is not. */ +function retryDelayMs(error: unknown): number | undefined { + const seen = new Set() + let wait: number | undefined + while (error && typeof error === "object" && !seen.has(error)) { + seen.add(error) + const entry = error as { code?: unknown; rpcCode?: unknown; retryAfterSeconds?: unknown; cause?: unknown } + if (entry.code === "commit-outcome-unknown") return undefined + const seconds = entry.retryAfterSeconds + if (entry.code === "rpc-error" && (entry.rpcCode === 3 || entry.rpcCode === 429) + && typeof seconds === "number" && Number.isFinite(seconds) + && seconds > 0 && seconds <= MAX_SEND_WAIT_SECONDS) { + wait = seconds * 1_000 + } + error = entry.cause + } + return wait +} + +/** Monitor sends bypass the host outbox. Keep one identity across flood retries. */ +export function createInlineMessageSender( + client: Pick, + signal?: AbortSignal, +): InlineSdkClient["sendMessage"] { + return async (params: InlineSdkSendMessageParams) => { + const request = { ...params, randomId: params.randomId ?? (randomBytes(8).readBigInt64LE() || 1n) } + for (let retry = 0; ; retry++) { + signal?.throwIfAborted() + try { + return await client.sendMessage(request) + } catch (error) { + const wait = retryDelayMs(error) + if (retry >= MAX_SEND_RETRIES || wait === undefined) throw error + // Never shorten a server wait. Long waits stay with the caller's failure path. + await delay(wait, undefined, { signal }) + } + } + } +} diff --git a/plugins/openclaw/src/inline/exec-approvals.ts b/plugins/openclaw/src/inline/exec-approvals.ts index bc32e8015..2fbbab761 100644 --- a/plugins/openclaw/src/inline/exec-approvals.ts +++ b/plugins/openclaw/src/inline/exec-approvals.ts @@ -1,11 +1,14 @@ import { resolveApprovalApprovers } from "openclaw/plugin-sdk/approval-auth-runtime" import { - createChannelExecApprovalProfile, + createChannelExecApprovalProfileAsync, isChannelExecApprovalClientEnabledFromConfig, isChannelExecApprovalTargetRecipient, matchesApprovalRequestFilters, } from "openclaw/plugin-sdk/approval-client-runtime" -import { resolveApprovalRequestChannelAccountId } from "openclaw/plugin-sdk/approval-native-runtime" +import { + resolveApprovalRequestChannelAccountIdAsync, + type ApprovalRequestSessionDelivery, +} from "openclaw/plugin-sdk/approval-native-runtime" import type { OpenClawConfig } from "openclaw/plugin-sdk/core" import { normalizeLowercaseStringOrEmpty, @@ -154,11 +157,15 @@ function matchesExplicitInlineForwardTargetAccount(params: { ) } -function matchesInlineRequestAccount(params: { +type InlineApprovalRequestParams = { cfg: OpenClawConfig accountId?: string | null request: ApprovalRequest -}): boolean { + /** Host-prepared session routing facts; reused instead of re-reading the session store. */ + sessionDelivery?: ApprovalRequestSessionDelivery | null +} + +async function matchesInlineRequestAccount(params: InlineApprovalRequestParams): Promise { const explicitTargetMatch = matchesExplicitInlineForwardTargetAccount(params) if (explicitTargetMatch !== undefined) { return explicitTargetMatch @@ -166,10 +173,11 @@ function matchesInlineRequestAccount(params: { const turnSourceChannel = normalizeLowercaseStringOrEmpty( params.request.request.turnSourceChannel, ) - const boundAccountId = resolveApprovalRequestChannelAccountId({ + const boundAccountId = await resolveApprovalRequestChannelAccountIdAsync({ cfg: params.cfg, request: params.request, channel: "inline", + ...(params.sessionDelivery !== undefined ? { sessionDelivery: params.sessionDelivery } : {}), }) if (turnSourceChannel && turnSourceChannel !== "inline" && !boundAccountId) { return ( @@ -186,7 +194,7 @@ function matchesInlineRequestAccount(params: { ) } -const inlineExecApprovalProfile = createChannelExecApprovalProfile({ +const inlineExecApprovalProfile = createChannelExecApprovalProfileAsync({ resolveConfig: resolveInlineExecApprovalConfig, resolveApprovers: getInlineExecApprovalApprovers, normalizeSenderId: normalizeInlineDirectUserId, @@ -201,8 +209,17 @@ export const isInlineExecApprovalApprover = inlineExecApprovalProfile.isApprover export const isInlineExecApprovalAuthorizedSender = inlineExecApprovalProfile.isAuthorizedSender export const resolveInlineExecApprovalTarget = inlineExecApprovalProfile.resolveTarget -export const shouldHandleInlineExecApprovalRequest = - inlineExecApprovalProfile.shouldHandleRequest + +/** + * Whether this Inline account should handle a forwarded approval request. + * Async because request-to-account binding may read the owning session through + * the host worker; pass `sessionDelivery` when the host already supplied it. + */ +export function shouldHandleInlineExecApprovalRequest( + params: InlineApprovalRequestParams, +): Promise { + return inlineExecApprovalProfile.shouldHandleRequest(params) +} export function isInlineExecApprovalHandlerConfigured(params: { cfg: OpenClawConfig diff --git a/plugins/openclaw/src/inline/follow-command.ts b/plugins/openclaw/src/inline/follow-command.ts index b4b5f1bed..5c2bee9c1 100644 --- a/plugins/openclaw/src/inline/follow-command.ts +++ b/plugins/openclaw/src/inline/follow-command.ts @@ -11,6 +11,7 @@ import { type InputPeer, } from "@inline-chat/realtime-sdk" import { resolveInlineAccount, resolveInlineToken } from "./accounts.js" +import { withInlineAccountClient } from "./client-registry.js" import { normalizeInlineTarget } from "./normalize.js" export type InlineFollowCommand = "follow" | "unfollow" @@ -187,16 +188,10 @@ export async function withInlineFollowClient(params: { throw new Error(`Inline not configured for account "${account.accountId}" (missing token or baseUrl)`) } const token = await resolveInlineToken(account) - const client = new InlineSdkClient({ - baseUrl: account.baseUrl, - token, - }) - await client.connect() - try { - return await params.fn(client, account.accountId) - } finally { - await client.close().catch(() => {}) - } + return await withInlineAccountClient( + { accountId: account.accountId, baseUrl: account.baseUrl, token }, + (client) => params.fn(client, account.accountId), + ) } export async function handleInlineFollowCommand( diff --git a/plugins/openclaw/src/inline/interactive-fallback.test.ts b/plugins/openclaw/src/inline/interactive-fallback.test.ts new file mode 100644 index 000000000..d21c16781 --- /dev/null +++ b/plugins/openclaw/src/inline/interactive-fallback.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from "vitest" +import { + normalizeInlineLegacyInteractivePresentation, + resolveInlineInteractiveTextFallback, +} from "./interactive-fallback" + +describe("inline/interactive-fallback", () => { + it("combines authored text with canonical presentation content", () => { + expect( + resolveInlineInteractiveTextFallback({ + text: "Pick one", + interactive: { blocks: [{ type: "text", text: "Legacy" }] }, + presentation: { blocks: [{ type: "text", text: "Presentation" }] }, + }), + ).toBe("Pick one\n\nPresentation") + }) + + it("prefers canonical presentation over legacy interactive content", () => { + expect( + resolveInlineInteractiveTextFallback({ + text: " ", + interactive: { + blocks: [ + { type: "text", text: " First " }, + { type: "buttons", buttons: [{ label: "Approve", value: "approve" }] }, + { type: "text", text: "Second" }, + ], + }, + presentation: { blocks: [{ type: "text", text: "Presentation" }] }, + }), + ).toBe("Presentation") + }) + + it("renders presentation fallback text when text is omitted", () => { + expect( + resolveInlineInteractiveTextFallback({ + presentation: { + title: "Deploy", + blocks: [ + { type: "context", text: "Build 12" }, + { + type: "buttons", + buttons: [ + { label: "Approve", action: { type: "callback", value: "deploy:approve" } }, + { label: "Status", action: { type: "command", command: "/status" } }, + ], + }, + ], + }, + }), + ).toBe("Deploy\n\nBuild 12\n\n- Approve\n- Status: `/status`") + }) + + it("falls back to legacy button labels when nothing else renders", () => { + expect( + resolveInlineInteractiveTextFallback({ + interactive: { + blocks: [{ type: "buttons", buttons: [{ label: "Approve", value: "approve" }] }], + }, + }), + ).toBe("- Approve") + }) + + it("keeps unsupported URL controls visible beside authored text", () => { + const text = resolveInlineInteractiveTextFallback({ + text: "Read the docs", + presentation: { + blocks: [{ type: "buttons", buttons: [{ label: "Docs", action: { type: "url", url: "https://inline.chat/docs" } }] }], + }, + }) + expect(text).toBe("Read the docs\n\n- Docs: https://inline.chat/docs") + }) + + it("honors an explicitly authored presentation fallback", () => { + expect(resolveInlineInteractiveTextFallback({ + text: "Readable fallback", + presentationTextMode: "fallback", + presentation: { blocks: [{ type: "text", text: "Native content" }] }, + })).toBe("Readable fallback") + }) + + it("returns the original text when there is nothing to render", () => { + expect(resolveInlineInteractiveTextFallback({ text: "" })).toBe("") + expect(resolveInlineInteractiveTextFallback({ text: null })).toBeUndefined() + expect(resolveInlineInteractiveTextFallback({ interactive: { blocks: [] } })).toBeUndefined() + }) + + it("normalizes only legacy block kinds from interactive input", () => { + expect( + normalizeInlineLegacyInteractivePresentation({ + title: "Ignored", + blocks: [ + { type: "context", text: "Ignored" }, + { type: "text", text: "Kept" }, + { type: "buttons", buttons: [{ label: "Go", action: { type: "callback", value: "go" } }] }, + { type: "select", options: [{ label: "A", value: "a" }] }, + ], + }), + ).toEqual({ + blocks: [ + { type: "text", text: "Kept" }, + { + type: "buttons", + buttons: [{ label: "Go", action: { type: "callback", value: "go" }, style: undefined }], + }, + { type: "select", placeholder: undefined, options: [{ label: "A", value: "a" }] }, + ], + }) + expect(normalizeInlineLegacyInteractivePresentation({ blocks: [{ type: "divider" }] })).toBeUndefined() + expect(normalizeInlineLegacyInteractivePresentation("nope")).toBeUndefined() + }) +}) diff --git a/plugins/openclaw/src/inline/interactive-fallback.ts b/plugins/openclaw/src/inline/interactive-fallback.ts index c88a224ea..f5f5b93d6 100644 --- a/plugins/openclaw/src/inline/interactive-fallback.ts +++ b/plugins/openclaw/src/inline/interactive-fallback.ts @@ -1,45 +1,61 @@ import { - interactiveReplyToPresentation, - normalizeInteractiveReply, normalizeMessagePresentation, renderMessagePresentationFallbackText, - resolveInteractiveTextFallback, + type MessagePresentation, } from "openclaw/plugin-sdk/interactive-runtime" +/** + * Normalize an untyped legacy `interactive` value as a MessagePresentation. + * + * Some OpenClaw producers (plugin command results, older message-tool calls) + * still emit `{ blocks: [text | buttons | select] }` under `interactive`. That + * shape is a structural subset of MessagePresentation, so it normalizes through + * the presentation contract directly. Only the legacy block kinds are kept so + * the result matches what those producers could express. + */ +export function normalizeInlineLegacyInteractivePresentation( + raw: unknown, +): MessagePresentation | undefined { + const presentation = normalizeMessagePresentation(raw) + if (!presentation) { + return undefined + } + const blocks = presentation.blocks.filter( + (block) => block.type === "text" || block.type === "buttons" || block.type === "select", + ) + return blocks.length > 0 ? { blocks } : undefined +} + export function resolveInlineInteractiveTextFallback(params: { text?: string | null | undefined interactive?: unknown | undefined presentation?: unknown | undefined + presentationTextMode?: "fallback" | undefined }): string | undefined { - const interactive = normalizeInteractiveReply(params.interactive) - const text = resolveInteractiveTextFallback({ - ...(params.text != null ? { text: params.text } : {}), - ...(interactive ? { interactive } : {}), - }) - if (text?.trim()) { - return text - } - + const text = params.text ?? undefined const presentation = normalizeMessagePresentation(params.presentation) if (presentation) { - const fallback = renderMessagePresentationFallbackText({ - ...(params.text != null ? { text: params.text } : {}), - presentation, - }) - if (fallback.trim()) { - return fallback - } + if (params.presentationTextMode === "fallback" && text?.trim()) return text + const fallback = renderMessagePresentationFallbackText({ ...(text !== undefined ? { text } : {}), presentation }) + if (fallback.trim()) return fallback } - - if (!interactive) { + if (text?.trim()) { return text } - const interactivePresentation = interactiveReplyToPresentation(interactive) - if (!interactivePresentation) { + const legacy = normalizeInlineLegacyInteractivePresentation(params.interactive) + const legacyText = (legacy?.blocks ?? []) + .flatMap((block) => (block.type === "text" ? [block.text.trim()] : [])) + .filter(Boolean) + .join("\n\n") + if (legacyText) { + return legacyText + } + + if (!legacy) { return text } - const fallback = renderMessagePresentationFallbackText({ presentation: interactivePresentation }) + const fallback = renderMessagePresentationFallbackText({ presentation: legacy }) return fallback.trim() ? fallback : text } diff --git a/plugins/openclaw/src/inline/members-tool.ts b/plugins/openclaw/src/inline/members-tool.ts index 8ac5525b0..3287e83a6 100644 --- a/plugins/openclaw/src/inline/members-tool.ts +++ b/plugins/openclaw/src/inline/members-tool.ts @@ -1,6 +1,7 @@ import type { AnyAgentTool, OpenClawConfig } from "openclaw/plugin-sdk/core" import { InlineSdkClient, Method } from "@inline-chat/realtime-sdk" import { resolveInlineAccount, resolveInlineToken } from "./accounts.js" +import { withInlineAccountClient } from "./client-registry.js" import { getSpaceMembersWithUsers, type InlineSpaceMemberRecord } from "./space-members.js" import { parseCurrentInlineSession } from "./tool-targets.js" @@ -125,16 +126,10 @@ async function withInlineClient(params: { throw new Error(`Inline not configured for account "${account.accountId}" (missing token or baseUrl)`) } const token = await resolveInlineToken(account) - const client = new InlineSdkClient({ - baseUrl: account.baseUrl, - token, - }) - await client.connect() - try { - return await params.fn(client, account.accountId) - } finally { - await client.close().catch(() => {}) - } + return await withInlineAccountClient( + { accountId: account.accountId, baseUrl: account.baseUrl, token }, + (client) => params.fn(client, account.accountId), + ) } function buildChatPeer(chatId: bigint) { diff --git a/plugins/openclaw/src/inline/message-tools.ts b/plugins/openclaw/src/inline/message-tools.ts index d92488913..60bc9cc6c 100644 --- a/plugins/openclaw/src/inline/message-tools.ts +++ b/plugins/openclaw/src/inline/message-tools.ts @@ -1,6 +1,7 @@ import type { AnyAgentTool, OpenClawConfig } from "openclaw/plugin-sdk/core" import { BotPresenceState_Kind, InlineSdkClient, Method } from "@inline-chat/realtime-sdk" import { resolveInlineAccount, resolveInlineToken } from "./accounts.js" +import { withInlineAccountClient } from "./client-registry.js" import { sanitizeInlineVisibleText } from "./outbound-sanitize.js" import { parseCurrentInlineTarget, @@ -394,16 +395,10 @@ async function withInlineClient(params: { throw new Error(`Inline not configured for account "${account.accountId}" (missing token or baseUrl)`) } const token = await resolveInlineToken(account) - const client = new InlineSdkClient({ - baseUrl: account.baseUrl, - token, - }) - await client.connect() - try { - return await params.fn(client, account.accountId) - } finally { - await client.close().catch(() => {}) - } + return await withInlineAccountClient( + { accountId: account.accountId, baseUrl: account.baseUrl, token }, + (client) => params.fn(client, account.accountId), + ) } function createInlineNudgeTool(ctx: InlineMessageToolContext): AnyAgentTool { diff --git a/plugins/openclaw/src/inline/monitor-addressing.ts b/plugins/openclaw/src/inline/monitor-addressing.ts new file mode 100644 index 000000000..0242e1754 --- /dev/null +++ b/plugins/openclaw/src/inline/monitor-addressing.ts @@ -0,0 +1,299 @@ +import { + findCommandByNativeName, + listNativeCommandSpecsForConfig, +} from "openclaw/plugin-sdk/native-command-registry" +import { isAbortRequestText } from "openclaw/plugin-sdk/command-primitives-runtime" +import { prepareSkillCommandsForAgents } from "openclaw/plugin-sdk/skill-commands-runtime" +import { getPluginCommandSpecs } from "openclaw/plugin-sdk/plugin-runtime" +import type { OpenClawConfig } from "openclaw/plugin-sdk/core" +import { DialogFollowMode, type Message } from "@inline-chat/realtime-sdk" +import type { ResolvedInlineAccount } from "./accounts.js" +import { summarizeInlineMessageContent, type InlineMessageEntitySummary } from "./message-content.js" +import { + shouldSyncInlineNativeCommandsForAccount, + shouldSyncInlineNativeSkillsForAccount, +} from "./bot-commands-sync.js" +import { listInlineBuiltinCommandSpecs } from "./threadreply-command.js" +import { DIALOG_FOLLOW_MODE_UNFOLLOWED, listInlineFollowCommandSpecs } from "./follow-command.js" +import { listInlineMaintenanceCommandSpecs } from "./maintenance-command-specs.js" +import { CHANNEL_ID, isRecord } from "./monitor-shared.js" + +const INLINE_NATIVE_COMMAND_PROVIDER = CHANNEL_ID +const INLINE_NATIVE_COMMAND_CALLBACK_PREFIX = "icmd:" + +type InlineMentionSource = + | "explicit_bot" + | "subteam" + | "implicit_thread" + | "command_bypass" + | "none" + +export function normalizeInlineUsername(raw: string | undefined): string | undefined { + const trimmed = raw?.trim() + if (!trimmed) return undefined + return trimmed.startsWith("@") ? trimmed.slice(1) : trimmed +} + +export function resolveInlineCommandTarget(raw: string, botUsername: string | undefined): { + body: string + targeted: boolean + addressedToMe: boolean +} { + const normalized = raw.trim() + const match = normalized.match(/^\/([^\s@]+)@([a-z0-9_]+)(?=\s|$)([\s\S]*)$/i) + if (!match) return { body: normalized, targeted: false, addressedToMe: false } + const [, command, targetUsername, suffix] = match + const normalizedBotUsername = normalizeInlineUsername(botUsername)?.toLowerCase() + const addressedToMe = Boolean( + normalizedBotUsername && targetUsername?.toLowerCase() === normalizedBotUsername, + ) + return { + body: addressedToMe ? `/${command}${suffix ?? ""}` : normalized, + targeted: true, + addressedToMe, + } +} + +export function normalizeInlineCommandBody(raw: string, botUsername: string | undefined): string { + return resolveInlineCommandTarget(raw, botUsername).body +} + +// The app attaches the bot a command was picked for to a leading command +// entity. Returns that bot's user id, or null when the command names none. +export function leadingInlineBotCommandTargetUserId(msg: Message): string | null { + const text = typeof msg.message === "string" ? msg.message : "" + for (const entity of msg.entities?.entities ?? []) { + if (entity.entity.oneofKind !== "botCommand") continue + const offset = Number(entity.offset) + if (!Number.isInteger(offset) || offset < 0) continue + if (text.slice(0, offset).trim()) continue + const botUserId = entity.entity.botCommand.botUserId + return botUserId > 0n ? String(botUserId) : null + } + return null +} + +// The server stamps the bot a bare command settles on; no stamp means no bot won. +export function startsWithUnclaimedInlineBotCommand(msg: Message): boolean { + const text = typeof msg.message === "string" ? msg.message : "" + for (const entity of msg.entities?.entities ?? []) { + if (entity.entity.oneofKind !== "botCommand") continue + const offset = Number(entity.offset) + if (!Number.isInteger(offset) || offset < 0) continue + if (text.slice(0, offset).trim()) continue + return entity.entity.botCommand.botUserId <= 0n + } + return false +} + +function isInlineBotCommandForOtherBot(msg: Message, meId: bigint): boolean { + const target = leadingInlineBotCommandTargetUserId(msg) + return target != null && target !== String(meId) +} + +// Service notices (thread backlinks, member changes, ...) are system records, not +// requests. They never wake the agent; Hermes and the bridge skip them the same way. +export function isInlineServiceMessage(msg: Message): boolean { + const record = msg as unknown as Record + const raw = isRecord(record.raw) ? record.raw : undefined + return isRecord(record.serviceMessage) || isRecord(raw?.serviceMessage) +} + +export function isInlineAbortRequestMessage(msg: Message, botUsername: string | undefined, meId: bigint): boolean { + const text = typeof msg.message === "string" ? msg.message : "" + if (isInlineBotCommandForOtherBot(msg, meId)) return false + return isAbortRequestText(text, botUsername ? { botUsername } : undefined) +} + +export function findInlineNativeCommandFromBody(commandBody: string): NonNullable> | null { + const match = commandBody.trim().match(/^\/([^\s]+)(?:\s+[\s\S]*)?$/) + if (!match?.[1]) return null + return findCommandByNativeName(match[1], INLINE_NATIVE_COMMAND_PROVIDER) ?? null +} + +export function resolveInlineCommandNameFromBody(commandBody: string): string | null { + const match = commandBody.trim().match(/^\/([^\s]+)(?:\s+[\s\S]*)?$/) + const raw = match?.[1]?.trim().toLowerCase() + return raw || null +} + +export function buildInlineNativeCommandCallbackData(commandText: string): string { + return `${INLINE_NATIVE_COMMAND_CALLBACK_PREFIX}${commandText}` +} + +export function parseInlineNativeCommandCallbackData(raw: string | undefined): string | null { + if (!raw) return null + const trimmed = raw.trim() + if (!trimmed.startsWith(INLINE_NATIVE_COMMAND_CALLBACK_PREFIX)) return null + const commandText = trimmed.slice(INLINE_NATIVE_COMMAND_CALLBACK_PREFIX.length).trim() + return commandText.startsWith("/") ? commandText : null +} + +export function listHostInlinePluginCommandSpecs(cfg: OpenClawConfig): Array<{ name: string; description: string }> { + return getPluginCommandSpecs("inline", { config: cfg }) +} + +function listInlinePluginCommandSpecs(cfg: OpenClawConfig): Array<{ name: string; description: string }> { + return [ + ...listHostInlinePluginCommandSpecs(cfg), + ...listInlineFollowCommandSpecs(), + ...listInlineBuiltinCommandSpecs(), + ...listInlineMaintenanceCommandSpecs(), + ] +} + +export function hasInlineCommandSpec( + specs: Array<{ name: string }>, + commandName: string, +): boolean { + const normalized = commandName.trim().toLowerCase() + if (!normalized) return false + return specs.some((spec) => spec.name.trim().toLowerCase() === normalized) +} + +export function isInlineThreadReplyCommandBody(commandBody: string): boolean { + return resolveInlineCommandNameFromBody(commandBody) === "threads" +} + +export function parseInlineCommandArgs(commandBody: string): string { + const match = commandBody.trim().match(/^\/[^\s]+(?:\s+([\s\S]*))?$/) + return match?.[1]?.trim() ?? "" +} + +export async function isInlineNativeCommandBody(params: { + cfg: OpenClawConfig + account: ResolvedInlineAccount + commandBody: string + agentId: string +}): Promise { + if (!shouldSyncInlineNativeCommandsForAccount({ cfg: params.cfg, account: params.account })) { + return false + } + const commandName = resolveInlineCommandNameFromBody(params.commandBody) + if (!commandName) return false + + const skillCommands = shouldSyncInlineNativeSkillsForAccount({ + cfg: params.cfg, + account: params.account, + }) + ? await prepareSkillCommandsForAgents({ + cfg: params.cfg, + agentIds: [params.agentId], + }) + : [] + const commandSpecs = listNativeCommandSpecsForConfig(params.cfg, { + skillCommands, + provider: INLINE_NATIVE_COMMAND_PROVIDER, + }) + for (const spec of commandSpecs) { + if (spec.name.trim().toLowerCase() === commandName) return true + } + for (const spec of listInlinePluginCommandSpecs(params.cfg)) { + if (spec.name.trim().toLowerCase() === commandName) return true + } + return false +} + +function escapeInlineRegExp(raw: string): string { + return raw.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") +} + +export function stripInlineBotMention(raw: string, botUsername: string | undefined): string { + const username = normalizeInlineUsername(botUsername) + if (!username) return raw + const mention = new RegExp(`(^|\\s)@${escapeInlineRegExp(username)}(?=$|[\\s,.:;!?])(?:[\\s,.:;!?]+)?`, "gi") + return raw.replace(mention, "$1").replace(/[ \t]{2,}/g, " ").trim() +} + +export function stripInlineBotMentionEntityText(text: string | null, botUsername: string | undefined): string | null { + const username = normalizeInlineUsername(botUsername)?.toLowerCase() + if (!text || !username) return text + const token = `@${username}` + const parts = text + .split(" | ") + .filter((part) => { + const normalized = part.trim().toLowerCase() + return normalized !== `username mention "${token}"` && !normalized.startsWith(`mention "${token}"`) + }) + const next = parts.join(" | ").trim() + return next || null +} + +export function collectInlineMentionedUserIds( + content: ReturnType | null, +): string[] { + const ids = new Set() + for (const entity of content?.entities ?? []) { + if (entity.type !== "mention" || !entity.userId) continue + ids.add(entity.userId) + } + return [...ids] +} + +// User ids in the run of mentions a message starts with, in order. That run +// names who the message is for ("@alice @bot compare"). +export function leadingInlineMentionUserIds(params: { + content: ReturnType | null + messageText: string | undefined +}): string[] { + const mentions = (params.content?.entities ?? []) + .filter((entity): entity is InlineMessageEntitySummary & { userId: string } => + entity.type === "mention" && + Boolean(entity.userId) && + Number.isInteger(entity.offset) && + entity.offset >= 0 && + Number.isInteger(entity.length) && + entity.length > 0) + .sort((a, b) => a.offset - b.offset) + // Inline entity offsets use UTF-16 code units, which matches JavaScript slice. + const text = params.messageText ?? "" + const leading: string[] = [] + let cursor = 0 + for (const mention of mentions) { + if (mention.offset < cursor) break + if (text.slice(cursor, mention.offset).replace(/,/g, "").trim()) break + leading.push(mention.userId) + cursor = mention.offset + mention.length + } + return leading +} + +export function isInlineDialogFollowing(followMode: unknown): boolean { + if (followMode == null || typeof followMode === "boolean") return false + if (typeof followMode === "number") return followMode === DialogFollowMode.FOLLOWING + if (typeof followMode === "bigint") return followMode === BigInt(DialogFollowMode.FOLLOWING) + const text = String(followMode).trim().toLowerCase() + return ( + text === String(DialogFollowMode.FOLLOWING) || + text === "following" || + text === "follow_mode_following" || + text === "dialog_following" + ) +} + +export function isInlineDialogExplicitlyUnfollowed(followMode: unknown): boolean { + if (followMode == null || typeof followMode === "boolean") return false + if (typeof followMode === "number") return followMode === DIALOG_FOLLOW_MODE_UNFOLLOWED + if (typeof followMode === "bigint") { + return followMode === BigInt(DIALOG_FOLLOW_MODE_UNFOLLOWED) + } + const text = String(followMode).trim().toLowerCase() + return ( + text === String(DIALOG_FOLLOW_MODE_UNFOLLOWED) || + text === "unfollowed" || + text === "follow_mode_unfollowed" || + text === "dialog_unfollowed" + ) +} + +// Only structured Inline addressing wakes the agent; words such as its name never do. +export function resolveInlineMentionSource(params: { + nativeMentioned: boolean + implicitMention: boolean + shouldBypassMention: boolean +}): InlineMentionSource { + if (params.nativeMentioned) return "explicit_bot" + if (params.shouldBypassMention) return "command_bypass" + if (params.implicitMention) return "implicit_thread" + return "none" +} diff --git a/plugins/openclaw/src/inline/monitor-history.ts b/plugins/openclaw/src/inline/monitor-history.ts new file mode 100644 index 000000000..bb6ad1d1e --- /dev/null +++ b/plugins/openclaw/src/inline/monitor-history.ts @@ -0,0 +1,850 @@ +import { DEFAULT_GROUP_HISTORY_LIMIT } from "../sdk-runtime-compat.js" +import type { OpenClawConfig } from "openclaw/plugin-sdk/core" +import { DialogFollowMode, InlineSdkClient, Method, type Message } from "@inline-chat/realtime-sdk" +import { summarizeInlineMessageContent } from "./message-content.js" +import { InlineSenderProfileCache } from "./sender-profile-cache.js" +import { loadInlineReplyThreadAnchorMessage, loadInlineReplyThreadMetadata } from "./reply-threads.js" +import { rememberInlineReplyThreadRoute, type InlineReplyThreadRouteRecord } from "./thread-routes.js" +import { INLINE_JOIN_MENTION_LOOKBACK_SECONDS } from "./thread-freshness.js" +import { CHANNEL_ID, buildChatPeer } from "./monitor-shared.js" +import { buildInlineInboundBodyText } from "./monitor-media.js" + +export type CachedChatInfo = { + kind: "direct" | "group" + title: string | null + peerUserId?: bigint | null + parentChatId?: bigint | null + parentMessageId?: bigint | null + lastMsgId?: bigint | null + dialogFollowMode?: DialogFollowMode | null +} + +type SenderProfileLookup = Pick + +type HistoryContext = { + historyText: string | null + attachmentText: string | null + entityText: string | null + inboundHistory: InlinePendingHistoryEntry[] + repliedToBot: boolean + replyToSenderId: string | null +} + +export type InlinePendingHistoryEntry = { + sender: string + body: string + timestamp?: number + messageId?: string +} + +type InlineChannelStructuredContextEntry = { + label: string + source: typeof CHANNEL_ID + type: string + payload: Record +} + +export type InlineReplyThreadContext = { + childChatId: bigint + parentChatId: bigint + parentChatKind: CachedChatInfo["kind"] + parentPeerUserId?: bigint | null + parentChatTitle: string | null + parentMessageId: bigint | null + threadLabel: string | null + anchorMessage: Message | null +} + +type InlineHistoryEntryPayload = { + line: string | null + attachmentLine: string | null + entityLine: string | null + inboundEntry: InlinePendingHistoryEntry | null +} + +const DEFAULT_DM_HISTORY_LIMIT = 6 +const HISTORY_LINE_MAX_CHARS = 280 +const REPLY_THREAD_LABEL_MAX_CHARS = 72 +const URL_LIKE_PATTERN = /https?:\/\/\S+/i +const BOT_MESSAGE_CACHE_LIMIT = 500 +const REACTION_TARGET_LOOKUP_LIMIT = 8 +export const REPLY_TARGET_LOOKUP_LIMIT = 8 +const ATTACHMENT_CONTEXT_LIMIT = 6 +const GET_MESSAGES_METHOD = + typeof (Method as Record).GET_MESSAGES === "number" && + Number.isInteger((Method as Record).GET_MESSAGES) && + ((Method as Record).GET_MESSAGES as number) > 0 + ? ((Method as Record).GET_MESSAGES as Method) + : null + +export async function resolveChatInfo( + client: InlineSdkClient, + cache: Map, + chatId: bigint, +): Promise { + const existing = cache.get(chatId) + if (existing) return existing + + const result = await client.getChat({ chatId }) + const peerType = result.peer?.type + const kind: CachedChatInfo["kind"] = peerType?.oneofKind === "user" ? "direct" : "group" + const peerUserId = peerType?.oneofKind === "user" ? peerType.user.userId : null + const title = result.title?.trim() || null + const info: CachedChatInfo = { + kind, + title, + ...(peerUserId != null ? { peerUserId } : {}), + ...(result.parentChatId != null ? { parentChatId: result.parentChatId } : {}), + ...(result.parentMessageId != null ? { parentMessageId: result.parentMessageId } : {}), + ...(result.lastMsgId != null ? { lastMsgId: result.lastMsgId } : {}), + ...(result.dialogFollowMode != null ? { dialogFollowMode: result.dialogFollowMode } : {}), + } + cache.set(chatId, info) + return info +} + +export function rewriteNumericMentionsToUsernames(text: string, senderProfilesById: SenderProfileLookup): string { + if (!text.includes("@")) return text + return text.replace(/(^|[^\w])@([0-9]+)\b/g, (full, prefix: string, userId: string) => { + const username = senderProfilesById.get(userId)?.username + if (!username) return full + return `${prefix}@${username}` + }) +} + +function rememberBotMessageId(cache: Map, chatId: bigint, messageId: bigint): void { + const key = String(chatId) + const list = cache.get(key) ?? [] + const nextId = String(messageId) + if (!list.includes(nextId)) list.push(nextId) + if (list.length > BOT_MESSAGE_CACHE_LIMIT) { + list.splice(0, list.length - BOT_MESSAGE_CACHE_LIMIT) + } + cache.set(key, list) +} + +export function rememberInlineBotDelivery(params: { + botMessageIdsByChat: Map + chatId: bigint + messageId: bigint + accountId: string + agentId: string + inlineAgentId?: string + replyThreadContext?: InlineReplyThreadContext | null +}): void { + rememberBotMessageId(params.botMessageIdsByChat, params.chatId, params.messageId) + if (!params.replyThreadContext || params.chatId !== params.replyThreadContext.childChatId) return + rememberInlineReplyThreadRoute({ + accountId: params.accountId, + parentChatId: params.replyThreadContext.parentChatId, + threadId: params.replyThreadContext.childChatId, + parentMessageId: params.replyThreadContext.parentMessageId, + threadLabel: params.replyThreadContext.threadLabel, + agentId: params.agentId, + ...(params.inlineAgentId ? { inlineAgentId: params.inlineAgentId } : {}), + }) +} + +function hasBotMessageId(cache: Map, chatId: bigint, messageId: bigint): boolean { + const key = String(chatId) + return (cache.get(key) ?? []).includes(String(messageId)) +} + +function rememberBotMessagesFromList(params: { + messages: Message[] + meId: bigint + chatId: bigint + botMessageIdsByChat: Map +}): void { + for (const item of params.messages) { + if (item.fromId === params.meId) { + rememberBotMessageId(params.botMessageIdsByChat, params.chatId, item.id) + } + } +} + +async function loadChatHistoryMessages(params: { + client: InlineSdkClient + chatId: bigint + limit: number + offsetId?: bigint +}): Promise { + const result = await params.client.invokeRaw(Method.GET_CHAT_HISTORY, { + oneofKind: "getChatHistory", + getChatHistory: { + peerId: buildChatPeer(params.chatId), + ...(params.offsetId != null ? { offsetId: params.offsetId } : {}), + limit: params.limit, + }, + }) + if (result.oneofKind !== "getChatHistory") { + return null + } + return result.getChatHistory.messages ?? [] +} + +export async function loadInlineJoinWindowMessages(params: { + client: InlineSdkClient + chatId: bigint + participantDate: bigint +}): Promise { + const cutoff = params.participantDate - INLINE_JOIN_MENTION_LOOKBACK_SECONDS + const messages = new Map() + let offsetId: bigint | undefined + + for (;;) { + const page = await loadChatHistoryMessages({ + client: params.client, + chatId: params.chatId, + limit: 100, + ...(offsetId != null ? { offsetId } : {}), + }) + if (!page) return null + if (page.length === 0) break + + let oldest: Message | undefined + let added = 0 + for (const message of page) { + const key = `${String(message.id)}:${String(message.date ?? 0n)}` + if (!messages.has(key)) { + messages.set(key, message) + added += 1 + } + if (!oldest || message.date < oldest.date || (message.date === oldest.date && message.id < oldest.id)) { + oldest = message + } + } + if (!oldest || oldest.date < cutoff || page.length < 100 || added === 0) break + offsetId = oldest.id + } + + return [...messages.values()] +} + +export async function findChatMessageById(params: { + client: InlineSdkClient + chatId: bigint + messageId: bigint + limit: number + meId: bigint + botMessageIdsByChat: Map +}): Promise { + const directResult = + GET_MESSAGES_METHOD == null + ? null + : await params.client + .invokeRaw(GET_MESSAGES_METHOD, { + oneofKind: "getMessages", + getMessages: { + peerId: buildChatPeer(params.chatId), + messageIds: [params.messageId], + }, + }) + .catch(() => null) + if (directResult?.oneofKind === "getMessages") { + const directMessages = directResult.getMessages.messages ?? [] + rememberBotMessagesFromList({ + messages: directMessages, + meId: params.meId, + chatId: params.chatId, + botMessageIdsByChat: params.botMessageIdsByChat, + }) + const directTarget = directMessages.find((item) => item.id === params.messageId) ?? null + if (directTarget) { + return directTarget + } + } + + // Compatibility fallback for older servers without GET_MESSAGES. + const historyMessages = await loadChatHistoryMessages({ + client: params.client, + chatId: params.chatId, + offsetId: params.messageId + 1n, + limit: params.limit, + }) + if (!historyMessages) { + return null + } + + rememberBotMessagesFromList({ + messages: historyMessages, + meId: params.meId, + chatId: params.chatId, + botMessageIdsByChat: params.botMessageIdsByChat, + }) + + return historyMessages.find((item) => item.id === params.messageId) ?? null +} + +export async function isReactionTargetBotMessage(params: { + client: InlineSdkClient + chatId: bigint + messageId: bigint + meId: bigint + botMessageIdsByChat: Map +}): Promise { + const target = await findChatMessageById({ + client: params.client, + chatId: params.chatId, + messageId: params.messageId, + limit: REACTION_TARGET_LOOKUP_LIMIT, + meId: params.meId, + botMessageIdsByChat: params.botMessageIdsByChat, + }) + if (!target) { + return hasBotMessageId(params.botMessageIdsByChat, params.chatId, params.messageId) + } + return target.fromId === params.meId +} + +export function normalizeHistoryText(raw: string | undefined): string { + const compact = (raw ?? "").replace(/\s+/g, " ").trim() + if (!compact) return "" + if (compact.length <= HISTORY_LINE_MAX_CHARS) return compact + // Keep full URLs for media/file discoverability in history context. + if (URL_LIKE_PATTERN.test(compact)) return compact + return `${compact.slice(0, HISTORY_LINE_MAX_CHARS - 1)}…` +} + +export function buildInlineReplyThreadLabel(params: { + title: string | null | undefined + anchorMessage: Message | null | undefined +}): string | null { + const title = params.title?.trim() + if (title) return title + + if (!params.anchorMessage) return "Re: Message" + const anchorText = buildInlineInboundBodyText(summarizeInlineMessageContent(params.anchorMessage)) + .replace(/\s+/g, " ") + .trim() + if (!anchorText) return "Re: Message" + + return `Re: ${anchorText.slice(0, REPLY_THREAD_LABEL_MAX_CHARS)}` +} + +function resolveHistorySenderLabel(params: { + senderId: bigint + meId: bigint + senderProfilesById: SenderProfileLookup +}): string { + if (params.senderId === params.meId) return "assistant" + const senderId = String(params.senderId) + const profile = params.senderProfilesById.get(senderId) + return resolveInlineSenderLabel({ + senderId, + senderName: profile?.name, + senderUsername: profile?.username, + }) +} + +export function resolveInlineSenderLabel(params: { + senderId: string + senderName?: string | null | undefined + senderUsername?: string | null | undefined +}): string { + const senderId = params.senderId.trim() + const name = params.senderName?.trim() + const usernameRaw = params.senderUsername?.trim().replace(/^@+/, "") + const username = usernameRaw ? `@${usernameRaw}` : undefined + + let label = name + if (name && username) { + label = `${name} (${username})` + } else if (!name && username) { + label = username + } + + const id = senderId ? `id:${senderId}` : undefined + const fallback = senderId ? `user:${senderId}` : undefined + if (label && id) return `${label} ${id}` + if (label) return label + return fallback ?? "id:unknown" +} + +export function resolveInlineDirectSenderName(params: { + profileName: string | null | undefined + chatTitle: string | null | undefined + senderId: string +}): string | undefined { + const profileName = params.profileName?.trim() + if (profileName) return profileName + const chatTitle = params.chatTitle?.trim() + if (!chatTitle || chatTitle === `user:${params.senderId}` || chatTitle === params.senderId) return undefined + return chatTitle +} + +export function resolveInlineConversationLabel(params: { + isGroup: boolean + groupTitle?: string | null + groupId: string + senderLabel: string +}): string { + if (!params.isGroup) return params.senderLabel + const title = params.groupTitle?.trim() + return title ? `${title} id:${params.groupId}` : `group:${params.groupId}` +} + +export function resolveHistoryLimit(params: { + cfg: OpenClawConfig + isGroup: boolean + historyLimit: number | undefined + dmHistoryLimit: number | undefined +}): number { + if (params.isGroup) { + return Math.max(0, params.historyLimit ?? params.cfg.messages?.groupChat?.historyLimit ?? DEFAULT_GROUP_HISTORY_LIMIT) + } + return Math.max(0, params.dmHistoryLimit ?? params.historyLimit ?? DEFAULT_DM_HISTORY_LIMIT) +} + +function historyEntryDedupeKey(entry: InlinePendingHistoryEntry): string { + if (entry.messageId) return `id:${entry.messageId}:ts:${entry.timestamp ?? "unknown"}` + return `ts:${entry.timestamp ?? "unknown"}:${entry.sender}:${entry.body}` +} + +export function mergeInboundHistoryEntries(params: { + historyContextEntries: InlinePendingHistoryEntry[] + pendingEntries: InlinePendingHistoryEntry[] + limit: number +}): Array<{ sender: string; body: string; timestamp?: number }> { + if (params.limit <= 0) return [] + + const deduped: InlinePendingHistoryEntry[] = [] + const seen = new Set() + for (const entry of [...params.historyContextEntries, ...params.pendingEntries]) { + const key = historyEntryDedupeKey(entry) + if (seen.has(key)) continue + seen.add(key) + deduped.push(entry) + } + + return deduped.slice(-params.limit).map((entry) => ({ + sender: entry.sender, + body: entry.body, + ...(entry.timestamp != null ? { timestamp: entry.timestamp } : {}), + })) +} + +function buildInlineHistoryEntryPayload(params: { + message: Message + senderProfilesById: SenderProfileLookup + meId: bigint + syntheticMessageId?: string +}): InlineHistoryEntryPayload { + const content = summarizeInlineMessageContent(params.message) + const text = normalizeHistoryText(content.text) + if (!text) { + return { + line: null, + attachmentLine: null, + entityLine: null, + inboundEntry: null, + } + } + + const label = resolveHistorySenderLabel({ + senderId: params.message.fromId, + meId: params.meId, + senderProfilesById: params.senderProfilesById, + }) + const replySuffix = params.message.replyToMsgId != null ? ` ->${String(params.message.replyToMsgId)}` : "" + const messageId = params.syntheticMessageId ?? String(params.message.id) + const attachmentText = normalizeHistoryText(content.attachmentText) + const entityText = normalizeHistoryText(content.entityText) + + return { + line: `#${String(params.message.id)}${replySuffix} ${label}: ${text}`, + attachmentLine: attachmentText ? `#${String(params.message.id)}${replySuffix} ${label}: ${attachmentText}` : null, + entityLine: entityText ? `#${String(params.message.id)}${replySuffix} ${label}: ${entityText}` : null, + inboundEntry: { + sender: label, + body: text, + ...(params.message.date != null ? { timestamp: Number(params.message.date) * 1000 } : {}), + messageId, + }, + } +} + +function appendInlineHistoryEntry( + target: { + lines: string[] + attachmentLines: string[] + entityLines: string[] + inboundHistory: InlinePendingHistoryEntry[] + }, + entry: InlineHistoryEntryPayload, +): void { + if (!entry.inboundEntry || !entry.line) return + target.inboundHistory.push(entry.inboundEntry) + target.lines.push(entry.line) + if (entry.attachmentLine) { + target.attachmentLines.push(entry.attachmentLine) + } + if (entry.entityLine) { + target.entityLines.push(entry.entityLine) + } +} + +function prependLabeledHistoryLine(params: { + existing: string | null + heading: string + line: string | null +}): string | null { + if (!params.line) return params.existing + const prefix = `${params.heading}\n` + const existingBody = params.existing?.startsWith(prefix) ? params.existing.slice(prefix.length) : params.existing + return existingBody ? `${prefix}${params.line}\n${existingBody}` : `${prefix}${params.line}` +} + +function stripLabeledHistoryHeading(text: string | null, heading: string): string | null { + if (!text) return null + const prefix = `${heading}\n` + const body = text.startsWith(prefix) ? text.slice(prefix.length) : text + return body.trim() || null +} + +function prependLabeledHistoryBlock(params: { + existing: string | null + heading: string + block: string | null +}): string | null { + const block = stripLabeledHistoryHeading(params.block, params.heading) + if (!block) return params.existing + const prefix = `${params.heading}\n` + const existingBody = stripLabeledHistoryHeading(params.existing, params.heading) + return existingBody ? `${prefix}${block}\n${existingBody}` : `${prefix}${block}` +} + +export function prependInlineReplyThreadAnchor(params: { + historyContext: HistoryContext + anchorMessage: Message + parentChatId: bigint + senderProfilesById: SenderProfileLookup + meId: bigint +}): HistoryContext { + const entry = buildInlineHistoryEntryPayload({ + message: params.anchorMessage, + senderProfilesById: params.senderProfilesById, + meId: params.meId, + syntheticMessageId: `anchor:${String(params.parentChatId)}:${String(params.anchorMessage.id)}`, + }) + if (!entry.inboundEntry || !entry.line) { + return { + ...params.historyContext, + } + } + + return { + ...params.historyContext, + inboundHistory: [entry.inboundEntry, ...params.historyContext.inboundHistory], + historyText: prependLabeledHistoryLine({ + existing: params.historyContext.historyText, + heading: "Recent thread messages (oldest -> newest):", + line: entry.line, + }), + attachmentText: prependLabeledHistoryLine({ + existing: params.historyContext.attachmentText, + heading: "Recent media/attachments:", + line: entry.attachmentLine, + }), + entityText: prependLabeledHistoryLine({ + existing: params.historyContext.entityText, + heading: "Recent message entities:", + line: entry.entityLine, + }), + } +} + +export function prependInlineParentHistoryContext(params: { + historyContext: HistoryContext + parentHistoryContext: HistoryContext +}): HistoryContext { + return { + ...params.historyContext, + inboundHistory: [ + ...params.parentHistoryContext.inboundHistory, + ...params.historyContext.inboundHistory, + ], + historyText: prependLabeledHistoryBlock({ + existing: params.historyContext.historyText, + heading: "Recent thread messages (oldest -> newest):", + block: params.parentHistoryContext.historyText, + }), + attachmentText: prependLabeledHistoryBlock({ + existing: params.historyContext.attachmentText, + heading: "Recent media/attachments:", + block: params.parentHistoryContext.attachmentText, + }), + entityText: prependLabeledHistoryBlock({ + existing: params.historyContext.entityText, + heading: "Recent message entities:", + block: params.parentHistoryContext.entityText, + }), + } +} + +export function buildInlineChannelStructuredContext(params: { + currentAttachmentText: string | null + currentEntityText: string | null + currentBody: string + historyAttachmentText: string | null + historyEntityText: string | null + actionContext?: InlineChannelStructuredContextEntry +}): InlineChannelStructuredContextEntry[] { + const entries: InlineChannelStructuredContextEntry[] = [] + const append = (label: string, type: string, summary: string | null) => { + if (!summary) return + const normalized = normalizeHistoryText(summary) + if (!normalized || normalized === params.currentBody) return + entries.push({ + label, + source: CHANNEL_ID, + type, + payload: { summary: normalized }, + }) + } + + append("Current Inline media/attachments", "current_media_attachments", params.currentAttachmentText) + append("Current Inline message entities", "current_message_entities", params.currentEntityText) + append("Recent Inline media/attachments", "recent_media_attachments", params.historyAttachmentText) + append("Recent Inline message entities", "recent_message_entities", params.historyEntityText) + if (params.actionContext) entries.push(params.actionContext) + return entries +} + +export async function resolveInlineInboundReplyThreadContext(params: { + replyThreadsEnabled: boolean + client: InlineSdkClient + chatId: bigint + chatInfo: CachedChatInfo + chatCache: Map + useCachedChatMetadata?: boolean +}): Promise { + if (params.chatInfo.kind === "direct" || !params.replyThreadsEnabled) { + return null + } + + const metadata = params.useCachedChatMetadata + ? params.chatInfo.parentChatId != null + ? { + childChatId: params.chatId, + parentChatId: params.chatInfo.parentChatId, + parentMessageId: params.chatInfo.parentMessageId ?? null, + title: params.chatInfo.title ?? null, + } + : null + : await loadInlineReplyThreadMetadata({ + client: params.client, + chatId: params.chatId, + }) + if (!metadata) { + return null + } + + const parentChatInfo = + metadata.parentChatId === params.chatId + ? params.chatInfo + : await resolveChatInfo(params.client, params.chatCache, metadata.parentChatId) + const anchorMessage = + metadata.parentMessageId != null + ? await loadInlineReplyThreadAnchorMessage({ + client: params.client, + parentChatId: metadata.parentChatId, + parentMessageId: metadata.parentMessageId, + }).catch(() => null) + : null + + return { + childChatId: metadata.childChatId, + parentChatId: metadata.parentChatId, + parentChatKind: parentChatInfo.kind, + ...(parentChatInfo.peerUserId != null ? { parentPeerUserId: parentChatInfo.peerUserId } : {}), + parentChatTitle: parentChatInfo.title ?? null, + parentMessageId: metadata.parentMessageId, + threadLabel: buildInlineReplyThreadLabel({ + title: metadata.title ?? params.chatInfo.title ?? null, + anchorMessage, + }), + anchorMessage, + } +} + +function parseCachedInlineId(raw: string | undefined): bigint | null { + if (!raw || !/^\d+$/.test(raw)) return null + try { + return BigInt(raw) + } catch { + return null + } +} + +export async function resolveInlineReplyThreadContextFromRoute(params: { + route: InlineReplyThreadRouteRecord + client: InlineSdkClient + chatCache: Map + fallbackParentChatInfo: CachedChatInfo +}): Promise { + const parentChatId = parseCachedInlineId(params.route.parentChatId) + const childChatId = parseCachedInlineId(params.route.threadId) + if (parentChatId == null || childChatId == null) { + return null + } + const parentMessageId = parseCachedInlineId(params.route.parentMessageId) + const parentChatInfo = await resolveChatInfo(params.client, params.chatCache, parentChatId) + .catch(() => params.fallbackParentChatInfo) + const anchorMessage = + parentMessageId != null + ? await loadInlineReplyThreadAnchorMessage({ + client: params.client, + parentChatId, + parentMessageId, + }).catch(() => null) + : null + const labelSource = params.route.threadLabel ?? params.route.title ?? null + + return { + childChatId, + parentChatId, + parentChatKind: parentChatInfo.kind, + ...(parentChatInfo.peerUserId != null ? { parentPeerUserId: parentChatInfo.peerUserId } : {}), + parentChatTitle: parentChatInfo.title ?? params.fallbackParentChatInfo.title, + parentMessageId, + threadLabel: buildInlineReplyThreadLabel({ + title: labelSource, + anchorMessage, + }), + anchorMessage, + } +} + +export async function buildReplyThreadParentHistoryContext(params: { + client: InlineSdkClient + replyThreadContext: InlineReplyThreadContext + parentHistoryLimit: number + senderProfilesById: SenderProfileLookup + meId: bigint + botMessageIdsByChat: Map +}): Promise { + if (params.parentHistoryLimit <= 0 || params.replyThreadContext.parentMessageId == null) { + return null + } + + return buildHistoryContext({ + client: params.client, + chatId: params.replyThreadContext.parentChatId, + currentMessageId: params.replyThreadContext.parentMessageId, + replyToMsgId: params.replyThreadContext.anchorMessage?.replyToMsgId, + senderProfilesById: params.senderProfilesById, + meId: params.meId, + historyLimit: params.parentHistoryLimit, + botMessageIdsByChat: params.botMessageIdsByChat, + }) +} + +export async function buildHistoryContext(params: { + client: InlineSdkClient + chatId: bigint + currentMessageId: bigint + replyToMsgId: bigint | undefined + senderProfilesById: SenderProfileLookup + meId: bigint + historyLimit: number + botMessageIdsByChat: Map +}): Promise { + const cachedReplyToBot = + params.replyToMsgId != null && + hasBotMessageId(params.botMessageIdsByChat, params.chatId, params.replyToMsgId) + let repliedToBot = cachedReplyToBot + let replyToSenderId: string | null = null + let foundReplyTargetInHistory = false + const lines: string[] = [] + const attachmentLines: string[] = [] + const entityLines: string[] = [] + const inboundHistory: InlinePendingHistoryEntry[] = [] + + if (params.historyLimit > 0) { + const messages = await loadChatHistoryMessages({ + client: params.client, + chatId: params.chatId, + offsetId: params.currentMessageId, + limit: params.historyLimit, + }) + + if (messages) { + for (const item of messages) { + if (item.fromId === params.meId) { + rememberBotMessageId(params.botMessageIdsByChat, params.chatId, item.id) + } + } + + const sortedMessages = messages + .filter((item) => item.id !== params.currentMessageId) + .sort((a, b) => { + const byDate = Number(a.date - b.date) + if (byDate !== 0) return byDate + if (a.id === b.id) return 0 + return a.id < b.id ? -1 : 1 + }) + + for (const item of sortedMessages) { + if (params.replyToMsgId != null && item.id === params.replyToMsgId) { + foundReplyTargetInHistory = true + replyToSenderId = String(item.fromId) + repliedToBot = item.fromId === params.meId + } + appendInlineHistoryEntry( + { + lines, + attachmentLines, + entityLines, + inboundHistory, + }, + buildInlineHistoryEntryPayload({ + message: item, + senderProfilesById: params.senderProfilesById, + meId: params.meId, + }), + ) + } + } + } + + if (params.replyToMsgId != null && !foundReplyTargetInHistory) { + const replyTarget = await findChatMessageById({ + client: params.client, + chatId: params.chatId, + messageId: params.replyToMsgId, + limit: REPLY_TARGET_LOOKUP_LIMIT, + meId: params.meId, + botMessageIdsByChat: params.botMessageIdsByChat, + }) + if (replyTarget) { + replyToSenderId = String(replyTarget.fromId) + repliedToBot = replyTarget.fromId === params.meId + } else if (!cachedReplyToBot) { + repliedToBot = false + } + } + + if (!lines.length) { + return { + historyText: null, + attachmentText: attachmentLines.length ? attachmentLines.slice(-ATTACHMENT_CONTEXT_LIMIT).join("\n") : null, + entityText: entityLines.length ? entityLines.slice(-ATTACHMENT_CONTEXT_LIMIT).join("\n") : null, + inboundHistory, + repliedToBot, + replyToSenderId, + } + } + return { + historyText: `Recent thread messages (oldest -> newest):\n${lines.join("\n")}`, + attachmentText: attachmentLines.length + ? `Recent media/attachments:\n${attachmentLines.slice(-ATTACHMENT_CONTEXT_LIMIT).join("\n")}` + : null, + entityText: entityLines.length + ? `Recent message entities:\n${entityLines.slice(-ATTACHMENT_CONTEXT_LIMIT).join("\n")}` + : null, + inboundHistory, + repliedToBot, + replyToSenderId, + } +} diff --git a/plugins/openclaw/src/inline/monitor-media.ts b/plugins/openclaw/src/inline/monitor-media.ts new file mode 100644 index 000000000..c5d1afda3 --- /dev/null +++ b/plugins/openclaw/src/inline/monitor-media.ts @@ -0,0 +1,158 @@ +import path from "node:path" +import { toInboundMediaFacts } from "openclaw/plugin-sdk/channel-inbound" +import type { OpenClawConfig } from "openclaw/plugin-sdk/core" +import type { Message } from "@inline-chat/realtime-sdk" +import type { ResolvedInlineAccount } from "./accounts.js" +import { getInlineRuntime } from "../runtime.js" +import { summarizeInlineMessageContent } from "./message-content.js" +import { resolveChannelMediaMaxBytes } from "../openclaw-compat.js" + +type InlineInboundMediaInfo = { + path?: string + contentType?: string | undefined + fileName?: string | undefined + kind?: "image" | "video" | "audio" | "document" | undefined + messageId?: string | undefined +} + +const DEFAULT_INLINE_MEDIA_MAX_BYTES = 300 * 1024 * 1024 + +export function resolveInlineMediaMaxBytes(params: { + cfg: OpenClawConfig + account: ResolvedInlineAccount +}): number { + return ( + resolveChannelMediaMaxBytes({ + cfg: params.cfg, + accountId: params.account.accountId, + resolveChannelLimitMb: ({ accountId }) => { + if (accountId != null && accountId !== params.account.accountId) return undefined + return params.account.config.mediaMaxMb + }, + }) ?? DEFAULT_INLINE_MEDIA_MAX_BYTES + ) +} + +export function buildInlineInboundMediaFacts(media: InlineInboundMediaInfo[]) { + return toInboundMediaFacts(media.map((item) => ({ + ...(item.path ? { path: item.path } : {}), + ...(item.contentType?.trim() ? { contentType: item.contentType.trim() } : {}), + ...(item.fileName?.trim() ? { fileName: item.fileName.trim() } : {}), + ...(item.kind ? { kind: item.kind } : {}), + ...(item.messageId ? { messageId: item.messageId } : {}), + }))) +} + +export function buildInlineInboundBodyText(content: ReturnType): string { + if (content.rawText) return content.rawText + if (content.media?.kind === "nudge" || (!content.media && content.attachmentText)) { + return content.text + } + return "" +} + +function resolveFilePathHint(params: { sourceUrl: string; preferredName?: string | null | undefined }): string | undefined { + const preferred = params.preferredName?.trim() + if (preferred) return preferred + + try { + const pathname = new URL(params.sourceUrl).pathname + const base = path.basename(pathname).trim() + if (base) return base + } catch { + // ignore malformed urls and let the media pipeline choose a filename + } + + return undefined +} + +export async function resolveInlineInboundMedia(params: { + core: ReturnType + message: Message + maxBytes: number + log?: { warn?: (msg: string) => void; debug?: (msg: string) => void } | undefined +}): Promise { + const content = summarizeInlineMessageContent(params.message) + const candidates = new Map< + string, + { + fileName?: string | null + mimeType?: string | null + kind: "image" | "video" | "audio" | "document" + } + >() + + const out: InlineInboundMediaInfo[] = [] + const messageId = String(params.message.id) + const inlineMediaKind = content.media?.kind === "photo" + ? "image" + : content.media?.kind === "video" + ? "video" + : content.media?.kind === "voice" + ? "audio" + : content.media?.kind === "document" + ? "document" + : null + + if (content.media?.url) { + candidates.set(content.media.url, { + fileName: content.media.fileName ?? null, + mimeType: content.media.mimeType ?? null, + kind: inlineMediaKind ?? "document", + }) + } else if (inlineMediaKind) { + out.push({ + kind: inlineMediaKind, + ...(content.media?.fileName ? { fileName: content.media.fileName } : {}), + ...(content.media?.mimeType ? { contentType: content.media.mimeType } : {}), + messageId, + }) + } + + for (const attachment of content.attachments) { + if (attachment.kind !== "urlPreview" || !attachment.previewImageUrl) continue + candidates.set(attachment.previewImageUrl, { + mimeType: null, + kind: "image", + }) + } + + for (const [url, candidate] of candidates.entries()) { + try { + const filePathHint = resolveFilePathHint({ sourceUrl: url, preferredName: candidate.fileName }) + const fetched = await params.core.channel.media.readRemoteMediaBuffer({ + url, + maxBytes: params.maxBytes, + ...(filePathHint ? { filePathHint } : {}), + }) + const fetchedContentType = fetched.contentType ?? candidate.mimeType ?? undefined + const saved = await params.core.channel.media.saveMediaBuffer( + fetched.buffer, + fetchedContentType, + "inbound", + params.maxBytes, + fetched.fileName ?? candidate.fileName ?? undefined, + ) + const contentType = saved.contentType ?? fetchedContentType + const fileName = fetched.fileName ?? candidate.fileName + out.push({ + path: saved.path, + ...(contentType ? { contentType } : {}), + ...(fileName ? { fileName } : {}), + // Documents may contain image/audio bytes; keep the host's MIME inference. + ...(candidate.kind !== "document" ? { kind: candidate.kind } : {}), + messageId, + }) + } catch (err) { + params.log?.warn?.(`inline: failed to download inbound media ${url}: ${String(err)}`) + out.push({ + kind: candidate.kind, + ...(candidate.fileName ? { fileName: candidate.fileName } : {}), + ...(candidate.mimeType ? { contentType: candidate.mimeType } : {}), + messageId, + }) + } + } + + return out +} diff --git a/plugins/openclaw/src/inline/monitor-model-menu.ts b/plugins/openclaw/src/inline/monitor-model-menu.ts new file mode 100644 index 000000000..de6ffd79e --- /dev/null +++ b/plugins/openclaw/src/inline/monitor-model-menu.ts @@ -0,0 +1,390 @@ +import { + buildCommandTextFromArgs, + findCommandByNativeName, + formatCommandArgMenuTitle, + parseCommandArgs, + resolveCommandArgMenuAsync, +} from "openclaw/plugin-sdk/native-command-registry" +import { resolveStoredModelOverrideAsync } from "openclaw/plugin-sdk/command-auth-native" +import { + buildConfiguredModelCatalog, + resolveAgentConfig, + resolveDefaultModelForAgent, + resolveThinkingDefault, +} from "openclaw/plugin-sdk/agent-runtime" +import { + getSessionEntryAsync, + type SessionEntry, + resolveStorePath, +} from "openclaw/plugin-sdk/session-store-runtime" +import type { OpenClawConfig } from "openclaw/plugin-sdk/core" +import type { InlineReplyMarkupButton } from "./command-ui.js" +import { normalizeOptionalString } from "./monitor-shared.js" +import { + findInlineNativeCommandFromBody, + buildInlineNativeCommandCallbackData, +} from "./monitor-addressing.js" + +type InlineModelPickerCallback = + | { type: "providers" | "back" } + | { type: "list"; provider: string; page: number } + | { type: "select"; provider?: string; model: string } + +type InlineModelPickerSelection = + | { kind: "resolved"; provider: string; model: string } + | { kind: "ambiguous"; model: string } + +export function resolveInlineParentSessionKeyCandidate( + sessionKey: string, + entry: { parentSessionKey?: string } | undefined, +): string | undefined { + const explicit = normalizeOptionalString(entry?.parentSessionKey) + if (explicit && explicit !== sessionKey) return explicit + + const inlineThreadMarker = ":thread:" + const markerIndex = sessionKey.lastIndexOf(inlineThreadMarker) + if (markerIndex > 0) return sessionKey.slice(0, markerIndex) + + return undefined +} + +type InlineSessionModelEntry = SessionEntry & { + fallbackNoticeSelectedModel?: string + fallbackNoticeActiveModel?: string +} + +type InlineProviderModelRef = { + provider: string + model: string +} + +function resolveInlineProviderModelRef( + providerValue: unknown, + modelValue: unknown, +): InlineProviderModelRef | undefined { + let provider = normalizeOptionalString(providerValue) + let model = normalizeOptionalString(modelValue) + if (!model) return undefined + + const slash = model.indexOf("/") + if (!provider && slash > 0 && slash < model.length - 1) { + provider = model.slice(0, slash) + model = model.slice(slash + 1) + } else if (provider && model.startsWith(`${provider}/`)) { + model = model.slice(provider.length + 1) + } + + return provider && model ? { provider, model } : undefined +} + +function resolveInlineFallbackModelContext(params: { + entry?: InlineSessionModelEntry + parentEntry?: InlineSessionModelEntry + defaultModel: InlineProviderModelRef + storedOverride?: InlineProviderModelRef | null +}): { + selected: InlineProviderModelRef + activeFallback?: InlineProviderModelRef +} { + const entryHasOverride = Boolean( + normalizeOptionalString(params.entry?.providerOverride) || + normalizeOptionalString(params.entry?.modelOverride), + ) + const parentHasOverride = Boolean( + normalizeOptionalString(params.parentEntry?.providerOverride) || + normalizeOptionalString(params.parentEntry?.modelOverride), + ) + const overrideEntry = entryHasOverride + ? params.entry + : parentHasOverride + ? params.parentEntry + : undefined + const fallbackEntry = overrideEntry?.modelOverrideSource === "auto" || + (normalizeOptionalString(overrideEntry?.modelOverrideFallbackOriginProvider) && + normalizeOptionalString(overrideEntry?.modelOverrideFallbackOriginModel)) + ? overrideEntry + : undefined + + const selected = fallbackEntry + ? resolveInlineProviderModelRef( + fallbackEntry.modelOverrideFallbackOriginProvider, + fallbackEntry.modelOverrideFallbackOriginModel, + ) ?? resolveInlineProviderModelRef(undefined, fallbackEntry.fallbackNoticeSelectedModel) + ?? params.defaultModel + : params.storedOverride ?? params.defaultModel + if (!fallbackEntry) return { selected } + + const activeFallback = resolveInlineProviderModelRef( + params.entry?.modelProvider, + params.entry?.model, + ) ?? resolveInlineProviderModelRef( + fallbackEntry.providerOverride, + fallbackEntry.modelOverride, + ) ?? resolveInlineProviderModelRef(undefined, fallbackEntry.fallbackNoticeActiveModel) + if ( + !activeFallback || + (activeFallback.provider === selected.provider && activeFallback.model === selected.model) + ) { + return { selected } + } + return { selected, activeFallback } +} + +export async function resolveInlineCommandMenuModelContext(params: { + cfg: OpenClawConfig + agentId: string + sessionKey: string +}): Promise<{ + provider?: string + model?: string + activeProvider?: string + activeModel?: string + thinkingLevel?: string +}> { + if (!params.sessionKey.trim()) { + return {} + } + try { + const storePath = resolveStorePath(params.cfg.session?.store, { agentId: params.agentId }) + const defaultModel = resolveDefaultModelForAgent({ + cfg: params.cfg, + agentId: params.agentId, + }) + const entry = await getSessionEntryAsync({ + agentId: params.agentId, + sessionKey: params.sessionKey, + storePath, + readConsistency: "latest", + }) as InlineSessionModelEntry | undefined + const thinkingLevel = normalizeOptionalString(entry?.thinkingLevel) + const parentSessionKey = resolveInlineParentSessionKeyCandidate(params.sessionKey, entry) + const parentEntry = parentSessionKey + ? await getSessionEntryAsync({ + agentId: params.agentId, + sessionKey: parentSessionKey, + storePath, + readConsistency: "latest", + }) as InlineSessionModelEntry | undefined + : undefined + const override = await resolveStoredModelOverrideAsync({ + ...(entry ? { sessionEntry: entry } : {}), + ...(parentSessionKey ? { parentSessionKey } : {}), + ...(parentSessionKey && parentEntry ? { sessionStore: { [parentSessionKey]: parentEntry } } : {}), + sessionKey: params.sessionKey, + defaultProvider: defaultModel.provider, + }) + const modelContext = resolveInlineFallbackModelContext({ + ...(entry ? { entry } : {}), + ...(parentEntry ? { parentEntry } : {}), + defaultModel, + ...(override?.model + ? { + storedOverride: { + provider: override.provider || defaultModel.provider, + model: override.model, + }, + } + : {}), + }) + return { + provider: modelContext.selected.provider, + model: modelContext.selected.model, + ...(modelContext.activeFallback + ? { + activeProvider: modelContext.activeFallback.provider, + activeModel: modelContext.activeFallback.model, + } + : {}), + ...(thinkingLevel ? { thinkingLevel } : {}), + } + } catch { + return {} + } +} + +export function resolveInlineThinkMenuCurrentLevel(params: { + cfg: OpenClawConfig + agentId: string + provider?: string + model?: string + thinkingLevel?: string +}): string { + const explicit = normalizeOptionalString(params.thinkingLevel) + if (explicit) return explicit + + const agentThinkingDefault = normalizeOptionalString( + resolveAgentConfig(params.cfg, params.agentId)?.thinkingDefault, + ) + if (agentThinkingDefault) return agentThinkingDefault + + const defaultModel = resolveDefaultModelForAgent({ + cfg: params.cfg, + agentId: params.agentId, + }) + return resolveThinkingDefault({ + cfg: params.cfg, + provider: params.provider ?? defaultModel.provider, + model: params.model ?? defaultModel.model, + catalog: buildConfiguredModelCatalog({ cfg: params.cfg }), + }) +} + +function formatInlineCommandArgMenuTitle(params: { + command: NonNullable> + menu: NonNullable>> + currentThinkingLevel?: string +}): string { + const title = formatCommandArgMenuTitle({ command: params.command, menu: params.menu }) + if (params.command.key !== "think" || !params.currentThinkingLevel) { + return title + } + return `Current thinking level: ${params.currentThinkingLevel}.\n${title}` +} + +export async function resolveInlineNativeCommandMenu(params: { + commandBody: string + cfg: OpenClawConfig + agentId: string + sessionKey: string +}): Promise<{ + title: string + buttons: InlineReplyMarkupButton[][] +} | null> { + const normalized = params.commandBody.trim() + const match = normalized.match(/^\/([^\s]+)(?:\s+([\s\S]+))?$/) + if (!match?.[1]) return null + + const command = findInlineNativeCommandFromBody(normalized) + if (!command) return null + + const args = parseCommandArgs(command, match[2]) + const menuNeedsModelContext = + command.argsMenu && + !(args?.raw && !args.values) && + command.args?.some( + (arg) => typeof arg.choices === "function" && args?.values?.[arg.name] == null, + ) + const menuModelContext = menuNeedsModelContext + ? await resolveInlineCommandMenuModelContext({ + cfg: params.cfg, + agentId: params.agentId, + sessionKey: params.sessionKey, + }) + : {} + const menu = await resolveCommandArgMenuAsync({ + command, + ...(args ? { args } : {}), + cfg: params.cfg, + ...menuModelContext, + }) + if (!menu) return null + + const currentThinkingLevel = + command.key === "think" + ? resolveInlineThinkMenuCurrentLevel({ + cfg: params.cfg, + agentId: params.agentId, + ...menuModelContext, + }) + : undefined + const title = formatInlineCommandArgMenuTitle({ + command, + menu, + ...(currentThinkingLevel ? { currentThinkingLevel } : {}), + }) + const rows: InlineReplyMarkupButton[][] = [] + for (let index = 0; index < menu.choices.length; index += 2) { + const slice = menu.choices.slice(index, index + 2) + rows.push( + slice.map((choice) => ({ + text: choice.label, + callback_data: buildInlineNativeCommandCallbackData( + buildCommandTextFromArgs(command, { + values: { [menu.arg.name]: choice.value }, + }), + ), + })), + ) + } + + return { title, buttons: rows } +} + +export function mapInlineModelPickerCallbackToCommand(raw: string): string | undefined { + const callback = parseInlineModelPickerCallback(raw) + if (!callback) return undefined + switch (callback.type) { + case "providers": + case "back": + return "/models" + case "list": + return `/models ${callback.provider} ${String(callback.page)}` + case "select": + return callback.provider ? `/model ${callback.provider}/${callback.model}` : `/model ${callback.model}` + } +} + +export function parseInlineModelPickerCallback(raw: string): InlineModelPickerCallback | null { + const trimmed = raw.trim() + if (!trimmed) return null + + if (trimmed === "mdl_prov") return { type: "providers" } + if (trimmed === "mdl_back") return { type: "back" } + + const listMatch = trimmed.match(/^mdl_list_([a-z0-9_.-]+)_(\d+)$/i) + if (listMatch?.[1] && listMatch[2]) { + const provider = listMatch[1].trim() + const page = Number.parseInt(listMatch[2], 10) + if (provider && Number.isFinite(page) && page > 0) { + return { type: "list", provider, page } + } + } + + const standardSelectionMatch = trimmed.match(/^mdl_sel_(.+)$/) + if (standardSelectionMatch?.[1]?.trim()) { + const modelRef = standardSelectionMatch[1].trim() + const slashIndex = modelRef.indexOf("/") + if (slashIndex > 0 && slashIndex < modelRef.length - 1) { + return { + type: "select", + provider: modelRef.slice(0, slashIndex), + model: modelRef.slice(slashIndex + 1), + } + } + } + + const compactSelectionMatch = trimmed.match(/^mdl_sel\/(.+)$/) + if (compactSelectionMatch?.[1]?.trim()) { + return { type: "select", model: compactSelectionMatch[1].trim() } + } + + return null +} + +export function resolveInlineModelPickerSelection(params: { + callback: Extract + providers: readonly string[] + byProvider: ReadonlyMap> +}): InlineModelPickerSelection { + if (params.callback.provider) { + return { + kind: "resolved", + provider: params.callback.provider, + model: params.callback.model, + } + } + + const matchingProviders = params.providers.filter((id) => params.byProvider.get(id)?.has(params.callback.model)) + if (matchingProviders.length === 1 && matchingProviders[0]) { + return { + kind: "resolved", + provider: matchingProviders[0], + model: params.callback.model, + } + } + + return { + kind: "ambiguous", + model: params.callback.model, + } +} diff --git a/plugins/openclaw/src/inline/monitor-presence.ts b/plugins/openclaw/src/inline/monitor-presence.ts new file mode 100644 index 000000000..13ea40f72 --- /dev/null +++ b/plugins/openclaw/src/inline/monitor-presence.ts @@ -0,0 +1,394 @@ +import type { InlineTypingCallbacks } from "../sdk-runtime-compat.js" +import { BotPresenceState_Kind, InlineSdkClient, Method } from "@inline-chat/realtime-sdk" +import { sendInlineWorkingAction } from "./compose-action.js" +import { type InlineReplyPayload, isRecord } from "./monitor-shared.js" + +const INLINE_BOT_PRESENCE_IDLE_DELAY_MS = 8_000 +const INLINE_BOT_PRESENCE_FINISH_IDLE_DELAY_MS = 8_000 +const INLINE_BOT_PRESENCE_GESTURE_MS = 1_400 +const INLINE_BOT_PRESENCE_FAILED_MS = 8_000 +const INLINE_BOT_PRESENCE_COMMENT_MAX_LENGTH = 30 + +type InlineBotPresenceKind = + | "idle" + | "happy" + | "waving" + | "jumping" + | "failed" + | "waiting" + | "running" + | "review" + +type InlineBotPresenceSignal = { + kind: InlineBotPresenceKind + comment?: string +} + +export function buildInlineTypingDispatcherOptions(typingCallbacks?: InlineTypingCallbacks) { + if (!typingCallbacks) { + return {} + } + return { + typingCallbacks, + onReplyStart: typingCallbacks.onReplyStart, + ...(typingCallbacks.onIdle ? { onIdle: typingCallbacks.onIdle } : {}), + ...(typingCallbacks.onCleanup ? { onCleanup: typingCallbacks.onCleanup } : {}), + } +} + +export async function sendInlineTypingToChats(params: { + client: Pick + chatIds: bigint[] + typing: boolean + onPartialError?: (chatId: bigint, error: unknown) => void +}): Promise { + if (params.chatIds.length === 0) return + + const failures: Array<{ chatId: bigint; error: unknown }> = [] + await Promise.all( + params.chatIds.map(async (chatId) => { + try { + await sendInlineWorkingAction(params.client, { chatId, working: params.typing }) + } catch (error) { + failures.push({ chatId, error }) + } + }), + ) + if (failures.length === 0) return + if (failures.length === params.chatIds.length) { + throw failures[0]?.error ?? new Error("inline typing failed") + } + for (const failure of failures) { + params.onPartialError?.(failure.chatId, failure.error) + } +} + +function inlineBotPresenceStateKind(kind: InlineBotPresenceKind): BotPresenceState_Kind { + switch (kind) { + case "idle": + return BotPresenceState_Kind.IDLE + case "happy": + return BotPresenceState_Kind.HAPPY + case "waving": + return BotPresenceState_Kind.WAVING + case "jumping": + return BotPresenceState_Kind.JUMPING + case "failed": + return BotPresenceState_Kind.FAILED + case "waiting": + return BotPresenceState_Kind.WAITING + case "running": + return BotPresenceState_Kind.RUNNING + case "review": + return BotPresenceState_Kind.REVIEW + } +} + +function normalizeInlineBotPresenceKind(value: unknown): InlineBotPresenceKind | undefined { + if (typeof value !== "string") return undefined + switch (value.trim().toLowerCase()) { + case "idle": + case "happy": + case "waving": + case "jumping": + case "failed": + case "waiting": + case "running": + case "review": + return value.trim().toLowerCase() as InlineBotPresenceKind + default: + return undefined + } +} + +function normalizeInlineBotPresenceComment(value: unknown): string | undefined { + if (typeof value !== "string") return undefined + const text = value.replace(/\s+/g, " ").trim() + if (!text) return undefined + return Array.from(text).slice(0, INLINE_BOT_PRESENCE_COMMENT_MAX_LENGTH).join("") +} + +export function resolveInlineBotPresenceSignal(payload: InlineReplyPayload): InlineBotPresenceSignal | null { + const channelData = isRecord(payload.channelData) ? payload.channelData : undefined + const inlineData = channelData && isRecord(channelData.inline) ? channelData.inline : undefined + const rawPresence = inlineData && isRecord(inlineData.botPresence) ? inlineData.botPresence : undefined + if (!rawPresence) return null + + const kind = normalizeInlineBotPresenceKind(rawPresence.kind) ?? "happy" + const comment = normalizeInlineBotPresenceComment(rawPresence.comment) + return comment ? { kind, comment } : { kind } +} + +async function sendInlineBotPresenceToChats(params: { + client: Pick + chatIds: bigint[] + kind: InlineBotPresenceKind + comment?: string + onPartialError?: (chatId: bigint, error: unknown) => void +}): Promise { + if (params.chatIds.length === 0) return + + const failures: Array<{ chatId: bigint; error: unknown }> = [] + await Promise.all( + params.chatIds.map(async (chatId) => { + try { + await params.client.invokeRaw(Method.SET_BOT_PRESENCE_STATE, { + oneofKind: "setBotPresenceState", + setBotPresenceState: { + peerId: { + type: { + oneofKind: "chat", + chat: { chatId }, + }, + }, + state: { + kind: inlineBotPresenceStateKind(params.kind), + ...(params.comment ? { comment: params.comment } : {}), + }, + }, + }) + } catch (error) { + failures.push({ chatId, error }) + } + }), + ) + if (failures.length === 0) return + if (failures.length === params.chatIds.length) { + throw failures[0]?.error ?? new Error("inline bot presence failed") + } + for (const failure of failures) { + params.onPartialError?.(failure.chatId, failure.error) + } +} + +export async function sendInlineTypingAndBotPresenceToChats(params: { + client: Pick + chatIds: bigint[] + typing: boolean + presenceKind: InlineBotPresenceKind + onTypingPartialError?: (chatId: bigint, error: unknown) => void + onPresencePartialError?: (chatId: bigint, error: unknown) => void + onPresenceError?: (error: unknown) => void +}): Promise { + let typingError: unknown = null + await Promise.all([ + sendInlineTypingToChats({ + client: params.client, + chatIds: params.chatIds, + typing: params.typing, + ...(params.onTypingPartialError ? { onPartialError: params.onTypingPartialError } : {}), + }).catch((error) => { + typingError = error + }), + sendInlineBotPresenceToChats({ + client: params.client, + chatIds: params.chatIds, + kind: params.presenceKind, + ...(params.onPresencePartialError ? { onPartialError: params.onPresencePartialError } : {}), + }).catch((error) => { + params.onPresenceError?.(error) + }), + ]) + + if (typingError != null) { + throw typingError + } +} + +export function createInlineBotPresenceLifecycle(params: { + client: Pick + chatIds: bigint[] + onPartialError: (chatId: bigint, error: unknown) => void + onError: (error: unknown) => void +}) { + let active = false + let finishing = false + let idleTimer: ReturnType | undefined + let gestureTimer: ReturnType | undefined + let currentKind: InlineBotPresenceKind | undefined + let currentComment: string | undefined + let busyKind: InlineBotPresenceKind = "running" + let busyComment: string | undefined + let sendChain: Promise = Promise.resolve() + let generation = 0 + + const clearIdleTimer = () => { + if (!idleTimer) return + clearTimeout(idleTimer) + idleTimer = undefined + } + + const clearGestureTimer = () => { + if (!gestureTimer) return + clearTimeout(gestureTimer) + gestureTimer = undefined + } + + const send = (kind: InlineBotPresenceKind, comment?: string): Promise => { + if (currentKind === kind && currentComment === comment) return Promise.resolve() + const chatIds = params.chatIds + currentKind = kind + currentComment = comment + sendChain = sendChain + .catch(() => {}) + .then(() => + sendInlineBotPresenceToChats({ + client: params.client, + chatIds, + kind, + ...(comment ? { comment } : {}), + onPartialError: params.onPartialError, + }).catch(params.onError), + ) + return sendChain + } + + const scheduleIdle = ( + delayMs = INLINE_BOT_PRESENCE_IDLE_DELAY_MS, + expectedGeneration = generation, + ) => { + clearIdleTimer() + idleTimer = setTimeout(() => { + idleTimer = undefined + if (generation !== expectedGeneration) return + active = false + finishing = false + busyKind = "running" + busyComment = undefined + void send("idle") + }, delayMs) + idleTimer.unref?.() + } + + const showBusy = async (kind: InlineBotPresenceKind, comment?: string): Promise => { + if (!active || finishing) return + clearIdleTimer() + clearGestureTimer() + busyKind = kind + busyComment = comment + await send(kind, comment) + } + + const gesture = ( + kind: InlineBotPresenceKind, + resumeKind = busyKind, + comment?: string, + resumeComment = busyComment, + ) => { + if (params.chatIds.length === 0) return + clearIdleTimer() + clearGestureTimer() + active = true + const expectedGeneration = generation + void send(kind, comment) + gestureTimer = setTimeout(() => { + gestureTimer = undefined + if (generation !== expectedGeneration) return + if (!active || finishing) return + void send(resumeKind, resumeComment) + }, INLINE_BOT_PRESENCE_GESTURE_MS) + gestureTimer.unref?.() + } + + return { + retarget: async (chatId: bigint) => { + generation += 1 + clearIdleTimer() + clearGestureTimer() + const oldChatIds = params.chatIds + params.chatIds = [chatId] + // Adoption means work continues in the child even if a parent gesture was + // finishing. Start a fresh lifecycle so the child cannot remain busy. + active = true + finishing = false + currentKind = undefined + currentComment = undefined + sendChain = sendChain.catch(() => {}).then(() => sendInlineBotPresenceToChats({ + client: params.client, + chatIds: oldChatIds, + kind: "idle", + onPartialError: params.onPartialError, + }).catch(params.onError)) + await send(busyKind, busyComment) + }, + start: async (cue?: InlineBotPresenceSignal | null) => { + if (params.chatIds.length === 0) return + clearIdleTimer() + if (active && !finishing) return + active = true + finishing = false + const cueKind = cue?.kind + busyKind = cueKind === "review" ? "review" : "running" + busyComment = undefined + if (cueKind === "waving") { + gesture("waving", busyKind) + return + } + await send(busyKind, busyComment) + }, + busy: (kind: InlineBotPresenceKind, comment?: string) => { + void showBusy(kind, comment) + }, + gesture: (kind: InlineBotPresenceKind, comment?: string) => { + gesture(kind, busyKind, comment) + }, + express: (signal: InlineBotPresenceSignal) => { + if (signal.kind === "failed") { + if (params.chatIds.length === 0) return + clearIdleTimer() + clearGestureTimer() + active = true + finishing = true + const expectedGeneration = generation + void send("failed", signal.comment).finally(() => { + if (generation === expectedGeneration) { + scheduleIdle(INLINE_BOT_PRESENCE_FAILED_MS, expectedGeneration) + } + }) + return + } + if (params.chatIds.length === 0) return + clearIdleTimer() + clearGestureTimer() + active = true + finishing = true + const expectedGeneration = generation + void send(signal.kind, signal.comment).finally(() => { + if (generation === expectedGeneration) scheduleIdle(undefined, expectedGeneration) + }) + }, + finish: () => { + if (!active || finishing) return + finishing = true + clearGestureTimer() + scheduleIdle(INLINE_BOT_PRESENCE_FINISH_IDLE_DELAY_MS) + }, + fail: (comment?: string) => { + if (params.chatIds.length === 0) return + clearIdleTimer() + clearGestureTimer() + active = true + finishing = true + const expectedGeneration = generation + void send("failed", comment).finally(() => { + if (generation === expectedGeneration) { + scheduleIdle(INLINE_BOT_PRESENCE_FAILED_MS, expectedGeneration) + } + }) + }, + cleanup: () => { + if (!active) return + if (finishing) return + clearIdleTimer() + clearGestureTimer() + active = false + const expectedGeneration = generation + void send("idle").finally(() => { + if (generation !== expectedGeneration) return + finishing = false + busyKind = "running" + busyComment = undefined + }) + }, + } +} diff --git a/plugins/openclaw/src/inline/monitor-reasoning.ts b/plugins/openclaw/src/inline/monitor-reasoning.ts new file mode 100644 index 000000000..938a9e42c --- /dev/null +++ b/plugins/openclaw/src/inline/monitor-reasoning.ts @@ -0,0 +1,145 @@ +import { isReasoningReplyPayload } from "openclaw/plugin-sdk/reply-payload" +import { findCodeRegions, isInsideCode, stripReasoningTagsFromText } from "openclaw/plugin-sdk/text-chunking" +import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime" +import { sanitizeInlineVisibleText } from "./outbound-sanitize.js" +import type { InlineReplyPayload } from "./monitor-shared.js" + +const REASONING_MESSAGE_PREFIX = "Reasoning:\n" +const REASONING_TAG_PREFIXES = [ + "]*>/gi + +export function resolveInlinePayloadMediaUrls(payload: InlineReplyPayload): string[] { + const urls = payload.mediaUrls?.length + ? payload.mediaUrls + : payload.mediaUrl + ? [payload.mediaUrl] + : [] + return Array.from(new Set(urls.map((url) => url.trim()).filter(Boolean))) +} + +function isPartialReasoningTagPrefix(text: string): boolean { + const trimmed = normalizeLowercaseStringOrEmpty(text.trimStart()) + if (!trimmed.startsWith("<")) return false + if (trimmed.includes(">")) return false + return REASONING_TAG_PREFIXES.some((prefix) => prefix.startsWith(trimmed)) +} + +type InlineReasoningSplit = { + reasoningText?: string + answerText?: string +} + +function formatInlineReasoningMessage(text: string): string { + const trimmed = text.trim() + if (!trimmed) return "" + const italicLines = trimmed + .split("\n") + .map((line) => (line ? `_${line}_` : line)) + .join("\n") + return `${REASONING_MESSAGE_PREFIX}${italicLines}` +} + +function extractInlineThinkingFromTaggedStreamOutsideCode(text: string): string { + if (!text) return "" + const codeRegions = findCodeRegions(text) + let result = "" + let lastIndex = 0 + let inThinking = false + THINKING_TAG_RE.lastIndex = 0 + for (const match of text.matchAll(THINKING_TAG_RE)) { + const idx = match.index ?? 0 + if (isInsideCode(idx, codeRegions)) continue + if (inThinking) { + result += text.slice(lastIndex, idx) + } + const isClose = match[1] === "/" + inThinking = !isClose + lastIndex = idx + match[0].length + } + if (inThinking) { + result += text.slice(lastIndex) + } + return result.trim() +} + +function splitInlineReasoningText(text?: string, isReasoning?: boolean): InlineReasoningSplit { + if (typeof text !== "string") return {} + + const trimmed = text.trim() + if (isPartialReasoningTagPrefix(trimmed)) { + return {} + } + if ( + trimmed.startsWith(REASONING_MESSAGE_PREFIX) && + trimmed.length > REASONING_MESSAGE_PREFIX.length + ) { + return { reasoningText: trimmed } + } + + const taggedReasoning = extractInlineThinkingFromTaggedStreamOutsideCode(text) + const strippedAnswer = stripReasoningTagsFromText(text, { mode: "strict", trim: "both" }) + + if (isReasoning === true) { + return { + reasoningText: formatInlineReasoningMessage( + taggedReasoning || strippedAnswer || text, + ), + } + } + if (!taggedReasoning && strippedAnswer === text) { + return { answerText: text } + } + const reasoningText = taggedReasoning + ? formatInlineReasoningMessage(taggedReasoning) + : undefined + return { + ...(reasoningText ? { reasoningText } : {}), + ...(strippedAnswer ? { answerText: strippedAnswer } : {}), + } +} + +export function resolveInlineChatVisibleReplyPayload(payload: T): T | null { + const mediaUrls = resolveInlinePayloadMediaUrls(payload) + if (isReasoningReplyPayload({ + ...(typeof payload.text === "string" ? { text: payload.text } : {}), + ...(payload.isReasoning !== undefined ? { isReasoning: payload.isReasoning } : {}), + })) { + return mediaUrls.length > 0 ? { ...payload, text: undefined } : null + } + + if (typeof payload.text !== "string") { + return payload + } + + const internal = sanitizeInlineVisibleText(payload.text) + if (internal.shouldSkip) { + return mediaUrls.length > 0 ? { ...payload, text: undefined } : null + } + if (isPartialReasoningTagPrefix(internal.text.trim())) { + return mediaUrls.length > 0 ? { ...payload, text: undefined } : null + } + + const split = splitInlineReasoningText(internal.text, payload.isReasoning) + if (split.answerText !== undefined) { + if (split.answerText === payload.text) { + return payload + } + return { + ...payload, + text: split.answerText || undefined, + } + } + if (!split.reasoningText) { + return payload + } + return mediaUrls.length > 0 ? { ...payload, text: undefined } : null +} diff --git a/plugins/openclaw/src/inline/monitor-reply-actions.ts b/plugins/openclaw/src/inline/monitor-reply-actions.ts new file mode 100644 index 000000000..41e2eba2a --- /dev/null +++ b/plugins/openclaw/src/inline/monitor-reply-actions.ts @@ -0,0 +1,173 @@ +import { + InlineSdkClient, + Method, + type MessageActions, + type MessageActionResponseUi, +} from "@inline-chat/realtime-sdk" +import { buildInlinePresentationButtons, resolveInlineMessageActionsParam } from "./actions.js" +import { normalizeMessagePresentation } from "openclaw/plugin-sdk/interactive-runtime" +import type { InlineMessageActionOwner } from "./message-actions.js" +import { sanitizeInlineVisibleText } from "./outbound-sanitize.js" +import { isRecord } from "./monitor-shared.js" +import { normalizeInlineCommandBody, parseInlineNativeCommandCallbackData } from "./monitor-addressing.js" +import { + mapInlineModelPickerCallbackToCommand, + parseInlineModelPickerCallback, +} from "./monitor-model-menu.js" + +export function callbackDataToBase64(data: Uint8Array): string { + return Buffer.from(data).toString("base64") +} + +export function callbackDataToUtf8(data: Uint8Array): string | undefined { + try { + const decoded = new TextDecoder("utf-8", { fatal: true }).decode(data).trim() + return decoded || undefined + } catch { + return undefined + } +} + +const INLINE_CALLBACK_TOAST_MAX_LENGTH = 256 + +function normalizeInlineCallbackToastText(raw: unknown): string | undefined { + if (typeof raw !== "string") return undefined + + const visible = sanitizeInlineVisibleText(raw) + if (visible.shouldSkip) return undefined + + const text = visible.text.replace(/\s+/g, " ").trim() + if (!text) return undefined + + if (text.length <= INLINE_CALLBACK_TOAST_MAX_LENGTH) { + return text + } + + return text.slice(0, INLINE_CALLBACK_TOAST_MAX_LENGTH).trimEnd() +} + +export function resolveInlineCallbackResponseUi(data: Uint8Array): MessageActionResponseUi | undefined { + const decoded = callbackDataToUtf8(data) + if (!decoded) return undefined + + let parsed: unknown + try { + parsed = JSON.parse(decoded) + } catch { + return undefined + } + + if (!isRecord(parsed)) return undefined + + const text = + normalizeInlineCallbackToastText(parsed.callbackToast) ?? + normalizeInlineCallbackToastText(parsed.callback_toast) ?? + normalizeInlineCallbackToastText(parsed.toast) + + if (!text) return undefined + + return { + kind: { + oneofKind: "toast", + toast: { text }, + }, + } +} + +function normalizeInlineActionCallbackData(raw: string): string { + const trimmed = raw.trim() + if (!trimmed) return "" + const nativeCommandText = parseInlineNativeCommandCallbackData(trimmed) + if (nativeCommandText) return nativeCommandText + return mapInlineModelPickerCallbackToCommand(trimmed) ?? trimmed +} + +function normalizeCompatibleButtonCallbackData(raw: string): string { + const trimmed = raw.trim() + if (!trimmed) return "" + if (parseInlineModelPickerCallback(trimmed)) return trimmed + return normalizeInlineActionCallbackData(trimmed) +} + +export function resolveInlineReplyActions( + payload: Record, + options?: { owner?: InlineMessageActionOwner }, +): MessageActions | undefined { + const owner = options?.owner ?? "agent" + const channelData = isRecord(payload.channelData) ? payload.channelData : undefined + const inlineData = channelData && isRecord(channelData.inline) ? channelData.inline : undefined + const telegramData = channelData && isRecord(channelData.telegram) ? channelData.telegram : undefined + + let rawButtons: unknown = undefined + let hasExplicitButtons = false + let mapCallbackData: ((value: string) => string) | undefined + + if (inlineData && Object.prototype.hasOwnProperty.call(inlineData, "buttons")) { + rawButtons = inlineData.buttons + hasExplicitButtons = true + } else if (telegramData && Object.prototype.hasOwnProperty.call(telegramData, "buttons")) { + // Backwards compatibility for replies emitted before Inline had its own command surface. + rawButtons = telegramData.buttons + hasExplicitButtons = true + mapCallbackData = normalizeCompatibleButtonCallbackData + } else if (Object.prototype.hasOwnProperty.call(payload, "buttons")) { + rawButtons = payload.buttons + hasExplicitButtons = true + } + + if (!hasExplicitButtons) { + const legacyActions = resolveInlineMessageActionsParam(payload, { owner }) + if (legacyActions) return legacyActions + // Host replies can carry buttons only as a MessagePresentation. + const presentation = normalizeMessagePresentation(payload.presentation) + if (!presentation) return undefined + rawButtons = buildInlinePresentationButtons(presentation).filter((row) => row.length > 0) + if ((rawButtons as unknown[]).length === 0) return undefined + } + + return resolveInlineMessageActionsParam( + { buttons: rawButtons }, + { owner, ...(mapCallbackData ? { mapCallbackData } : {}) }, + ) +} + +export async function answerInlineMessageAction( + client: InlineSdkClient, + interactionId: bigint, + ui?: MessageActionResponseUi, +): Promise { + const withAnswer = client as InlineSdkClient & { + answerMessageAction?: (params: { interactionId: bigint; ui?: MessageActionResponseUi }) => Promise + invokeUncheckedRaw?: ( + method: Method, + input?: { + oneofKind?: string + answerMessageAction?: { interactionId: bigint; ui?: MessageActionResponseUi } + }, + ) => Promise + } + + if (typeof withAnswer.answerMessageAction === "function") { + await withAnswer.answerMessageAction({ interactionId, ...(ui ? { ui } : {}) }) + return + } + + const answerMethod = (Method as Record)["ANSWER_MESSAGE_ACTION"] + if (typeof answerMethod === "number" && typeof withAnswer.invokeUncheckedRaw === "function") { + await withAnswer.invokeUncheckedRaw(answerMethod as Method, { + oneofKind: "answerMessageAction", + answerMessageAction: { interactionId, ...(ui ? { ui } : {}) }, + }) + } +} + +export function resolveCallbackCommandBodyFromActionData(params: { + data: Uint8Array + botUsername?: string +}): string | undefined { + const decoded = callbackDataToUtf8(params.data) + if (!decoded) return undefined + const normalized = normalizeInlineActionCallbackData(decoded) + if (!normalized.startsWith("/")) return undefined + return normalizeInlineCommandBody(normalized, params.botUsername) +} diff --git a/plugins/openclaw/src/inline/monitor-shared.ts b/plugins/openclaw/src/inline/monitor-shared.ts new file mode 100644 index 000000000..6cd7a052e --- /dev/null +++ b/plugins/openclaw/src/inline/monitor-shared.ts @@ -0,0 +1,33 @@ +export const CHANNEL_ID = "inline" as const + +export type InlineReplyPayload = { + isError?: boolean | undefined + text?: string | undefined + mediaUrl?: string | undefined + mediaUrls?: string[] | undefined + replyToId?: string | undefined + channelData?: Record | undefined + isReasoning?: boolean | undefined +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +export function normalizeOptionalString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined +} + +export function buildChatPeer(chatId: bigint): { + type: { + oneofKind: "chat" + chat: { chatId: bigint } + } +} { + return { + type: { + oneofKind: "chat", + chat: { chatId }, + }, + } +} diff --git a/plugins/openclaw/src/inline/monitor.test.ts b/plugins/openclaw/src/inline/monitor.test.ts index 81e72485d..dda54d611 100644 --- a/plugins/openclaw/src/inline/monitor.test.ts +++ b/plugins/openclaw/src/inline/monitor.test.ts @@ -9,8 +9,8 @@ import { const modelRuntimeMocks = vi.hoisted(() => ({ buildModelsProviderData: vi.fn(), resolveDefaultModelForAgent: vi.fn(), - getSessionEntry: vi.fn(), - patchSessionEntry: vi.fn(), + getSessionEntryAsync: vi.fn(), + applySessionEntryPatch: vi.fn(), applyModelOverrideToSessionEntry: vi.fn(), })) @@ -51,8 +51,8 @@ vi.mock("openclaw/plugin-sdk/session-store-runtime", async () => { ) return { ...actual, - getSessionEntry: modelRuntimeMocks.getSessionEntry, - patchSessionEntry: modelRuntimeMocks.patchSessionEntry, + getSessionEntryAsync: modelRuntimeMocks.getSessionEntryAsync, + applySessionEntryPatch: modelRuntimeMocks.applySessionEntryPatch, } }) @@ -73,7 +73,7 @@ type MonitorHarness = { sendMessage: ReturnType invokeRaw: ReturnType uploadFile: ReturnType - fetchRemoteMedia: ReturnType + readRemoteMediaBuffer: ReturnType saveMediaBuffer: ReturnType resolveAgentRoute: ReturnType recordInboundSession: ReturnType @@ -495,11 +495,11 @@ async function setupMonitorHarness(setup: MonitorSetup): Promise } }) modelRuntimeMocks.resolveDefaultModelForAgent.mockReset().mockImplementation(({ cfg }: any) => resolveMockModelRef(cfg)) - modelRuntimeMocks.getSessionEntry.mockReset().mockReturnValue(undefined) - modelRuntimeMocks.patchSessionEntry.mockReset().mockImplementation(async ({ fallbackEntry, update }: any) => { - const entry = structuredClone(fallbackEntry ?? { sessionId: "test-session", updatedAt: 1 }) - return await update(entry, { existingEntry: fallbackEntry ? undefined : entry }) - }) + modelRuntimeMocks.getSessionEntryAsync.mockReset().mockReturnValue(undefined) + modelRuntimeMocks.applySessionEntryPatch.mockReset().mockImplementation(async ({ fallbackEntry, patch }: any) => ({ + ...structuredClone(fallbackEntry ?? { sessionId: "test-session", updatedAt: 1 }), + ...patch, + })) modelRuntimeMocks.applyModelOverrideToSessionEntry.mockReset().mockImplementation(({ entry, selection }: any) => { entry.provider = selection.provider entry.model = selection.model @@ -537,7 +537,7 @@ async function setupMonitorHarness(setup: MonitorSetup): Promise error.name = "InlineSdkAuthenticationError" onAuthenticationError?.(error) }) - const fetchRemoteMedia = vi.fn(async ({ url }: { url: string }) => { + const readRemoteMediaBuffer = vi.fn(async ({ url }: { url: string }) => { const media = setup.mediaByUrl?.[url] return { buffer: Buffer.from(media?.buffer ?? [1, 2, 3]), @@ -994,6 +994,7 @@ async function setupMonitorHarness(setup: MonitorSetup): Promise SET_BOT_PRESENCE_STATE, UPDATE_DIALOG_FOLLOW_MODE, }, + UpdateComposeAction_ComposeAction: { WORKING: 6 }, DialogFollowMode: { DIALOG_FOLLOW_MODE_UNSPECIFIED: 0, FOLLOWING: 1, @@ -1114,7 +1115,7 @@ async function setupMonitorHarness(setup: MonitorSetup): Promise ) return { ...actual, - listSkillCommandsForAgents: vi.fn(() => setup.skillCommands ?? []), + prepareSkillCommandsForAgents: vi.fn(() => setup.skillCommands ?? []), } }) vi.doMock("openclaw/plugin-sdk/plugin-runtime", async () => { @@ -1199,7 +1200,7 @@ async function setupMonitorHarness(setup: MonitorSetup): Promise shouldHandleTextCommands: setup.shouldHandleTextCommands ?? (() => true), }, media: { - fetchRemoteMedia, + readRemoteMediaBuffer, saveMediaBuffer, }, text: { @@ -1243,7 +1244,7 @@ async function setupMonitorHarness(setup: MonitorSetup): Promise sendTyping, invokeRaw, uploadFile, - fetchRemoteMedia, + readRemoteMediaBuffer, saveMediaBuffer, resolveAgentRoute, recordInboundSession, @@ -1409,7 +1410,7 @@ describe("inline/monitor", () => { agents: { defaults: { model: "openai/gpt-5.6-sol" } }, }, }) - modelRuntimeMocks.getSessionEntry.mockReturnValue({ + modelRuntimeMocks.getSessionEntryAsync.mockReturnValue({ sessionId: "fallback-session", updatedAt: Date.now(), providerOverride: "minimax", @@ -1685,7 +1686,7 @@ describe("inline/monitor", () => { }, }) configureCatalog() - modelRuntimeMocks.getSessionEntry.mockImplementation(() => selectedEntry as any) + modelRuntimeMocks.getSessionEntryAsync.mockImplementation(() => selectedEntry as any) const mutationHandle = await mutationHarness.monitorInlineProvider({ cfg: {} as any, account: buildAccount({ groupPolicy: "open" }), @@ -1710,7 +1711,7 @@ describe("inline/monitor", () => { oneofKind: "select", select: { value: "minimax/MiniMax-M3" }, }) - expect(modelRuntimeMocks.patchSessionEntry).not.toHaveBeenCalled() + expect(modelRuntimeMocks.applySessionEntryPatch).not.toHaveBeenCalled() await mutationHandle.stop() }) @@ -3259,7 +3260,7 @@ describe("inline/monitor", () => { MessageSid: "1001", }), ) - expect(harness.calls.fetchRemoteMedia).not.toHaveBeenCalled() + expect(harness.calls.readRemoteMediaBuffer).not.toHaveBeenCalled() await handle.stop() }) @@ -3352,7 +3353,7 @@ describe("inline/monitor", () => { }), ) }) - expect(harness.calls.fetchRemoteMedia).not.toHaveBeenCalled() + expect(harness.calls.readRemoteMediaBuffer).not.toHaveBeenCalled() expect(harness.calls.finalizeInboundContext).not.toHaveBeenCalledWith( expect.objectContaining({ Body: "/stop", @@ -3448,7 +3449,7 @@ describe("inline/monitor", () => { await new Promise((resolve) => setTimeout(resolve, 80)) expect(harness.calls.dispatchReply).toHaveBeenCalledTimes(1) - expect(harness.calls.fetchRemoteMedia).not.toHaveBeenCalled() + expect(harness.calls.readRemoteMediaBuffer).not.toHaveBeenCalled() expect(harness.calls.enqueueSystemEvent).not.toHaveBeenCalled() expect(harness.calls.finalizeInboundContext).not.toHaveBeenCalledWith( expect.objectContaining({ @@ -4776,7 +4777,7 @@ describe("inline/monitor", () => { }) await waitFor(() => { - expect(harness.calls.fetchRemoteMedia).not.toHaveBeenCalled() + expect(harness.calls.readRemoteMediaBuffer).not.toHaveBeenCalled() expect(harness.calls.finalizeInboundContext).toHaveBeenCalledWith( expect.objectContaining({ Body: "what about this image?", @@ -6936,6 +6937,42 @@ describe("inline/monitor", () => { await handle.stop() }) + it("routes a rendered typed command button through native command handling", async () => { + const { resolveInlineReplyActions } = await import("./monitor-reply-actions") + const actions = resolveInlineReplyActions({ + presentation: { blocks: [{ type: "buttons", buttons: [ + { label: "Verbose", action: { type: "command", command: "/verbose on" } }, + ] }] }, + }) + const action = actions!.rows[0]!.actions[0]! + expect(action.action.oneofKind).toBe("callback") + const harness = await setupMonitorHarness({ + events: [{ + kind: "message.action.invoke", chatId: 7n, interactionId: 23n, + messageId: 1005n, actorUserId: 42n, actionId: action.actionId, + data: action.action.oneofKind === "callback" ? action.action.callback.data : new Uint8Array(), + }], + chats: { "7": { kind: "direct", title: "Alice" } }, + historyByChat: { "7": [{ id: 1005n, date: 1_700_000_000n, fromId: 777n, message: "Verbose?" }] }, + shouldHandleTextCommands: vi.fn(() => true), + dispatchReplyPayload: { text: "verbose enabled" }, + }) + const handle = await harness.monitorInlineProvider({ + cfg: { commands: { text: false } } as any, + account: buildAccount({ dmPolicy: "open" }), + runtime: { log: vi.fn(), error: vi.fn() } as any, + abortSignal: new AbortController().signal, + log: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }) + try { + await waitFor(() => expect(harness.calls.finalizeInboundContext).toHaveBeenCalledWith( + expect.objectContaining({ CommandBody: "/verbose on", CommandSource: "native" }), + )) + } finally { + await handle.stop() + } + }) + it("does not interpret agent-owned callback data as a native command", async () => { const harness = await setupMonitorHarness({ events: [ @@ -7174,7 +7211,7 @@ describe("inline/monitor", () => { }) expect(harness.calls.sendMessage).not.toHaveBeenCalled() - expect(modelRuntimeMocks.patchSessionEntry).not.toHaveBeenCalled() + expect(modelRuntimeMocks.applySessionEntryPatch).not.toHaveBeenCalled() await handle.stop() }) @@ -7498,6 +7535,93 @@ describe("inline/monitor", () => { await handle.stop() }) + it("retries a flood-limited final reply with the same send identity", async () => { + const flood = new Error("SDK send failed", { cause: Object.assign(new Error("paced"), { + code: "rpc-error", rpcCode: 3, retryAfterSeconds: 1, + }) }) + const harness = await setupMonitorHarness({ + events: [{ kind: "message.new", chatId: 72n, message: { + id: 10022n, date: 1_700_000_001n, fromId: 42n, message: "hello", + } }], + chats: { "72": { kind: "direct", title: "Alice" } }, + sendMessageErrors: [flood], + dispatchReplyPayload: { text: "final answer" }, + }) + const handle = await harness.monitorInlineProvider({ + cfg: {} as any, account: buildAccount({ dmPolicy: "open" }), + runtime: { log: vi.fn(), error: vi.fn() } as any, + abortSignal: new AbortController().signal, + log: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }) + try { + await waitFor(() => expect(harness.calls.sendMessage).toHaveBeenCalledTimes(2)) + const [first, second] = harness.calls.sendMessage.mock.calls.map(([request]) => request) + expect(first.text).toBe("final answer") + expect(second).toBe(first) + expect(first.randomId).not.toBe(0n) + } finally { + await handle.stop() + } + }) + + it("maps presentation-only buttons to inline message actions on text replies", async () => { + const harness = await setupMonitorHarness({ + events: [ + { + kind: "message.new", + chatId: 72n, + message: { + id: 10022n, + date: 1_700_000_001n, + fromId: 42n, + message: "show options", + }, + }, + ], + chats: { + "72": { kind: "direct", title: "Alice" }, + }, + dispatchReplyPayload: { + text: "choose one", + presentation: { + blocks: [ + { + type: "buttons", + buttons: [ + { label: "Approve", style: "primary", action: { type: "callback", value: "approve" } }, + { label: "Disabled", disabled: true, action: { type: "command", command: "/stop" } }, + { label: "Docs", action: { type: "url", url: "https://inline.chat/docs" } }, + ], + }, + ], + }, + }, + }) + + const handle = await harness.monitorInlineProvider({ + cfg: {} as any, + account: buildAccount({ dmPolicy: "open" }), + runtime: { log: vi.fn(), error: vi.fn() } as any, + abortSignal: new AbortController().signal, + log: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }) + + await waitFor(() => { + expect(harness.calls.sendMessage).toHaveBeenCalledWith( + expect.objectContaining({ chatId: 72n, actions: expect.any(Object) }), + ) + }) + + const sent = harness.calls.sendMessage.mock.calls[0]?.[0] + const buttons = sent?.actions?.rows?.flatMap((row: any) => row.actions) ?? [] + expect(buttons.map((button: any) => button.text)).toEqual(["Approve"]) + expect(buttons[0]?.style).toBe(1) + expect(sent?.text).toContain("https://inline.chat/docs") + expect(new TextDecoder().decode(buttons[0]?.action?.callback?.data)).toBe("approve") + + await handle.stop() + }) + it("uses shared interactive labels as fallback text when reply text is omitted", async () => { const harness = await setupMonitorHarness({ events: [ @@ -11678,6 +11802,66 @@ describe("inline/monitor", () => { await handle.stop() }) + it("never wakes the agent on service notices, new or edited", async () => { + const harness = await setupMonitorHarness({ + events: [ + { + kind: "message.new", + chatId: 71n, + message: { + id: 60101n, + date: 1_700_000_055n, + fromId: 42n, + message: "Alice linked a thread", + serviceMessage: { event: { oneofKind: "threadBacklink", threadBacklink: {} } }, + }, + }, + { + kind: "message.edit", + chatId: 71n, + message: { + id: 60101n, + date: 1_700_000_056n, + fromId: 42n, + message: "Alice linked a thread", + raw: { serviceMessage: { event: {} } }, + }, + }, + { + kind: "message.new", + chatId: 71n, + message: { + id: 60102n, + date: 1_700_000_057n, + fromId: 42n, + message: "real question", + }, + }, + ], + chats: { + "71": { kind: "direct", title: "Alice" }, + }, + dispatchReplyPayload: { text: "answer" }, + }) + + const handle = await harness.monitorInlineProvider({ + cfg: {} as any, + account: buildAccount({ dmPolicy: "open" }), + runtime: { log: vi.fn(), error: vi.fn() } as any, + abortSignal: new AbortController().signal, + log: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + }) + + await waitFor(() => { + expect(harness.calls.dispatchReply).toHaveBeenCalledTimes(1) + }) + const ctx = harness.calls.dispatchReply.mock.calls[0]?.[0]?.ctx + expect(ctx?.MessageSid).toBe("60102") + expect(harness.calls.enqueueSystemEvent).not.toHaveBeenCalled() + + await handle.stop() + }) + it("includes prior skipped group messages from chat history when a later message mentions the bot", async () => { const harness = await setupMonitorHarness({ events: [ @@ -13307,7 +13491,7 @@ describe("inline/monitor", () => { await waitFor(() => { expect(harness.calls.dispatchReply).toHaveBeenCalled() - expect(harness.calls.fetchRemoteMedia).toHaveBeenCalledWith( + expect(harness.calls.readRemoteMediaBuffer).toHaveBeenCalledWith( expect.objectContaining({ url: "https://cdn.inline.chat/current-photo.jpg", }), @@ -13507,7 +13691,7 @@ describe("inline/monitor", () => { await waitFor(() => { expect(harness.calls.dispatchReply).toHaveBeenCalled() - expect(harness.calls.fetchRemoteMedia).toHaveBeenCalledWith( + expect(harness.calls.readRemoteMediaBuffer).toHaveBeenCalledWith( expect.objectContaining({ url: "https://cdn.inline.chat/spec.pdf", filePathHint: "spec.pdf", @@ -13602,7 +13786,7 @@ describe("inline/monitor", () => { await waitFor(() => { expect(harness.calls.dispatchReply).toHaveBeenCalled() - expect(harness.calls.fetchRemoteMedia).toHaveBeenCalledWith( + expect(harness.calls.readRemoteMediaBuffer).toHaveBeenCalledWith( expect.objectContaining({ url: "https://cdn.inline.chat/image-document", filePathHint: "image-document.png", @@ -13684,7 +13868,7 @@ describe("inline/monitor", () => { await waitFor(() => { expect(harness.calls.dispatchReply).toHaveBeenCalled() - expect(harness.calls.fetchRemoteMedia).toHaveBeenCalledWith( + expect(harness.calls.readRemoteMediaBuffer).toHaveBeenCalledWith( expect.objectContaining({ url: "https://cdn.inline.chat/flattened-photo.jpg", filePathHint: "flattened-photo.jpg", @@ -13787,7 +13971,7 @@ describe("inline/monitor", () => { }), ) }) - expect(harness.calls.fetchRemoteMedia).not.toHaveBeenCalled() + expect(harness.calls.readRemoteMediaBuffer).not.toHaveBeenCalled() expect(harness.calls.enqueueSystemEvent).not.toHaveBeenCalled() await handle.stop() @@ -13848,7 +14032,7 @@ describe("inline/monitor", () => { await waitFor(() => { expect(harness.calls.dispatchReply).toHaveBeenCalledTimes(1) - expect(harness.calls.fetchRemoteMedia).toHaveBeenCalledWith( + expect(harness.calls.readRemoteMediaBuffer).toHaveBeenCalledWith( expect.objectContaining({ url: "https://cdn.inline.chat/fallback-voice.ogg", filePathHint: "fallback-voice.ogg", @@ -13945,7 +14129,7 @@ describe("inline/monitor", () => { }, }) - harness.calls.fetchRemoteMedia.mockRejectedValueOnce(new Error("network failed")) + harness.calls.readRemoteMediaBuffer.mockRejectedValueOnce(new Error("network failed")) const handle = await harness.monitorInlineProvider({ cfg: {} as any, @@ -14023,7 +14207,7 @@ describe("inline/monitor", () => { }, }) - harness.calls.fetchRemoteMedia.mockRejectedValueOnce( + harness.calls.readRemoteMediaBuffer.mockRejectedValueOnce( new Error( "Failed to fetch media from https://cdn.inline.chat/huge-spec.pdf: content length 500000000 exceeds maxBytes 1048576", ), @@ -14043,7 +14227,7 @@ describe("inline/monitor", () => { await waitFor(() => { expect(harness.calls.dispatchReply).toHaveBeenCalled() - expect(harness.calls.fetchRemoteMedia).toHaveBeenCalledWith( + expect(harness.calls.readRemoteMediaBuffer).toHaveBeenCalledWith( expect.objectContaining({ url: "https://cdn.inline.chat/huge-spec.pdf", maxBytes: 1 * 1024 * 1024, diff --git a/plugins/openclaw/src/inline/monitor.ts b/plugins/openclaw/src/inline/monitor.ts index 76182095d..d8cc0f2c3 100644 --- a/plugins/openclaw/src/inline/monitor.ts +++ b/plugins/openclaw/src/inline/monitor.ts @@ -4,22 +4,15 @@ import { mkdir, stat } from "node:fs/promises" import path from "node:path" import { beginInlineActiveThreadRoute, type ActiveInlineThreadRoute } from "./active-thread-route.js" import { - buildCommandTextFromArgs, - findCommandByNativeName, - formatCommandArgMenuTitle, - listNativeCommandSpecsForConfig, - parseCommandArgs, - resolveCommandArgMenu, + resolveCommandArgMenuAsync, } from "openclaw/plugin-sdk/native-command-registry" import { CHANNEL_APPROVAL_NATIVE_RUNTIME_CONTEXT_CAPABILITY } from "openclaw/plugin-sdk/approval-handler-adapter-runtime" import { registerChannelRuntimeContext } from "openclaw/plugin-sdk/channel-runtime-context" import { - resolveCommandAuthorization, - resolveStoredModelOverride, + resolveCommandAuthorizationAsync, } from "openclaw/plugin-sdk/command-auth-native" import { expandAllowFromWithAccessGroups } from "openclaw/plugin-sdk/security-runtime" import { buildCommandsMessagePaginated } from "openclaw/plugin-sdk/command-status" -import { isAbortRequestText } from "openclaw/plugin-sdk/command-primitives-runtime" import { createConnectedChannelStatusPatch, createTransportActivityStatusPatch, @@ -31,23 +24,16 @@ import { createChannelInboundDebouncer, resolveUnmentionedGroupInboundPolicy, shouldDebounceTextInbound, - toInboundMediaFacts, } from "openclaw/plugin-sdk/channel-inbound" import { - buildConfiguredModelCatalog, - resolveAgentConfig, resolveDefaultModelForAgent, - resolveThinkingDefault, } from "openclaw/plugin-sdk/agent-runtime" import { - getSessionEntry, - patchSessionEntry, + applySessionEntryPatch, resolveStorePath, } from "openclaw/plugin-sdk/session-store-runtime" import { buildModelsProviderData } from "openclaw/plugin-sdk/models-provider-runtime" -import { listSkillCommandsForAgents } from "openclaw/plugin-sdk/skill-commands-runtime" -import { getPluginCommandSpecs } from "openclaw/plugin-sdk/plugin-runtime" -import { isReasoningReplyPayload } from "openclaw/plugin-sdk/reply-payload" +import { prepareSkillCommandsForAgents } from "openclaw/plugin-sdk/skill-commands-runtime" import { buildChannelProgressDraftLine, buildChannelProgressDraftLineForEntry, @@ -61,14 +47,8 @@ import { type AgentPlanStep, type ChannelProgressDraftLine, } from "openclaw/plugin-sdk/channel-outbound" -import { - findCodeRegions, - isInsideCode, - stripReasoningTagsFromText, -} from "openclaw/plugin-sdk/text-chunking" import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime" import { - DEFAULT_GROUP_HISTORY_LIMIT, clearHistoryEntriesIfEnabled, createChannelReplyPipelineCompat, recordPendingHistoryEntryIfEnabled, @@ -81,7 +61,6 @@ import { resolveInboundLastRouteSessionKey } from "openclaw/plugin-sdk/routing" import { BotCapability_Kind, BotChatSettingsProblem_Code, - BotPresenceState_Kind, DialogFollowMode, InlineSdkAuthenticationError, InlineSdkClient, @@ -90,12 +69,11 @@ import { registerBotCapabilitiesWithRetry, type Message, type MessageActions, - type MessageActionResponseUi, type User, } from "@inline-chat/realtime-sdk" import { resolveInlineToken, type ResolvedInlineAccount } from "./accounts.js" -import { resolveInlineMessageActionsParam } from "./actions.js" -import { sendInlineWorkingAction } from "./compose-action.js" +import { shareInlineAccountClient } from "./client-registry.js" +import { createInlineMessageSender } from "./delivery.js" import { INLINE_DEFAULT_GROUP_POLICY, INLINE_DEFAULT_REQUIRE_MENTION, @@ -110,13 +88,11 @@ import { resolveInlineGroupReplyThreadParentHistoryLimit, resolveInlineGroupReplyThreadRequireExplicitMention, resolveInlineGroupSystemPrompt, - type InlineReplyThreadMode, } from "./policy.js" import { getInlineRuntime } from "../runtime.js" import { uploadInlineMediaFromUrl } from "./media.js" import { summarizeInlineMessageContent, - type InlineMessageEntitySummary, } from "./message-content.js" import { callInlineBotApi } from "./bot-commands-api.js" import { @@ -127,16 +103,8 @@ import { import { buildInlineAgentActionBody, buildInlineAgentActionStructuredContext, - buildInlineMessageActionId, resolveInlineMessageActionOwnership, - type InlineMessageActionOwner, } from "./message-actions.js" -import { - sanitizeInlineActionCallbackData, - sanitizeInlineActionCopyText, - sanitizeInlineActionLabel, - sanitizeInlineVisibleText, -} from "./outbound-sanitize.js" import { resolveInlineInteractiveTextFallback } from "./interactive-fallback.js" import { InlineSenderProfileCache, @@ -150,51 +118,38 @@ import { import { createInlineReplyThreadForMessage, isInlineReplyThreadsEnabled, - loadInlineReplyThreadAnchorMessage, - loadInlineReplyThreadMetadata, } from "./reply-threads.js" import { lookupInlineReplyThreadRoute, lookupInlineReplyThreadRouteByThreadId, rememberInlineReplyThreadRoute, - type InlineReplyThreadRouteRecord, } from "./thread-routes.js" import { - INLINE_JOIN_MENTION_LOOKBACK_SECONDS, resolveInlineThreadFreshness, } from "./thread-freshness.js" import { logInboundDrop, - resolveChannelMediaMaxBytes, resolveControlCommandGate, resolveMentionGatingWithBypass, setOpenClawAgentDefaultModel, resolveUseAccessGroups, } from "../openclaw-compat.js" -import { - shouldSyncInlineNativeCommandsForAccount, - shouldSyncInlineNativeSkillsForAccount, -} from "./bot-commands-sync.js" import { handleInlineThreadReplyCommandWithConfigRuntime, INLINE_LIVE_CONFIG_WRITE_AFTER_WRITE, - listInlineBuiltinCommandSpecs, resolveCurrentMode as resolveInlineThreadReplyMode, setGroupMode as setInlineThreadReplyMode, } from "./threadreply-command.js" import { - DIALOG_FOLLOW_MODE_UNFOLLOWED, inlineFollowCommandFailureText, inlineFollowCommandSuccessText, inlineFollowCommandUsageText, inlineFollowModeForCommand, - listInlineFollowCommandSpecs, parseInlineFollowCommandBody, summarizeInlineFollowCommandError, takeInlineFollowModeChanged, updateInlineFollowMode, } from "./follow-command.js" -import { listInlineMaintenanceCommandSpecs } from "./maintenance-command-specs.js" import { OPENCLAW_BOT_CHAT_SETTINGS_VERSION, buildOpenClawBotChatSettingsDocument, @@ -203,10 +158,86 @@ import { type OpenClawBotChatSettingsContext, type OpenClawBotChatSettingsOption, } from "./bot-chat-settings.js" +import { CHANNEL_ID, type InlineReplyPayload, isRecord, buildChatPeer } from "./monitor-shared.js" +import { + buildInlineTypingDispatcherOptions, + sendInlineTypingToChats, + resolveInlineBotPresenceSignal, + sendInlineTypingAndBotPresenceToChats, + createInlineBotPresenceLifecycle, +} from "./monitor-presence.js" +import { resolveInlinePayloadMediaUrls, resolveInlineChatVisibleReplyPayload } from "./monitor-reasoning.js" +import { + normalizeInlineUsername, + resolveInlineCommandTarget, + leadingInlineBotCommandTargetUserId, + startsWithUnclaimedInlineBotCommand, + isInlineServiceMessage, + isInlineAbortRequestMessage, + findInlineNativeCommandFromBody, + resolveInlineCommandNameFromBody, + listHostInlinePluginCommandSpecs, + hasInlineCommandSpec, + isInlineThreadReplyCommandBody, + parseInlineCommandArgs, + isInlineNativeCommandBody, + stripInlineBotMention, + stripInlineBotMentionEntityText, + collectInlineMentionedUserIds, + leadingInlineMentionUserIds, + isInlineDialogFollowing, + isInlineDialogExplicitlyUnfollowed, + resolveInlineMentionSource, +} from "./monitor-addressing.js" +import { + resolveInlineParentSessionKeyCandidate, + resolveInlineCommandMenuModelContext, + resolveInlineThinkMenuCurrentLevel, + resolveInlineNativeCommandMenu, + parseInlineModelPickerCallback, + resolveInlineModelPickerSelection, +} from "./monitor-model-menu.js" +import { + callbackDataToBase64, + callbackDataToUtf8, + resolveInlineCallbackResponseUi, + resolveInlineReplyActions, + answerInlineMessageAction, + resolveCallbackCommandBodyFromActionData, +} from "./monitor-reply-actions.js" +import { + type CachedChatInfo, + type InlinePendingHistoryEntry, + type InlineReplyThreadContext, + REPLY_TARGET_LOOKUP_LIMIT, + resolveChatInfo, + rewriteNumericMentionsToUsernames, + rememberInlineBotDelivery, + loadInlineJoinWindowMessages, + findChatMessageById, + isReactionTargetBotMessage, + normalizeHistoryText, + buildInlineReplyThreadLabel, + resolveInlineSenderLabel, + resolveInlineDirectSenderName, + resolveInlineConversationLabel, + resolveHistoryLimit, + mergeInboundHistoryEntries, + prependInlineReplyThreadAnchor, + prependInlineParentHistoryContext, + buildInlineChannelStructuredContext, + resolveInlineInboundReplyThreadContext, + resolveInlineReplyThreadContextFromRoute, + buildReplyThreadParentHistoryContext, + buildHistoryContext, +} from "./monitor-history.js" +import { + resolveInlineMediaMaxBytes, + buildInlineInboundMediaFacts, + buildInlineInboundBodyText, + resolveInlineInboundMedia, +} from "./monitor-media.js" -const CHANNEL_ID = "inline" as const -const INLINE_NATIVE_COMMAND_PROVIDER = CHANNEL_ID -const INLINE_NATIVE_COMMAND_CALLBACK_PREFIX = "icmd:" const INLINE_REQUEST_ERROR_FALLBACK = "OpenClaw could not process that request. Please try again." const INLINE_DEBOUNCE_ERROR_FALLBACK = @@ -216,11 +247,6 @@ const INLINE_REPLY_SESSION_INIT_RETRY_DELAYS_MS = [50, 150, 300, 600] as const const DEFAULT_REPLY_THREAD_PARENT_HISTORY_LIMIT = 10 const DEFAULT_INLINE_VOICE_TRANSCRIPT_WAIT_MS = 8_000 const MAX_INLINE_VOICE_TRANSCRIPT_WAIT_MS = 60_000 -const INLINE_BOT_PRESENCE_IDLE_DELAY_MS = 8_000 -const INLINE_BOT_PRESENCE_FINISH_IDLE_DELAY_MS = 8_000 -const INLINE_BOT_PRESENCE_GESTURE_MS = 1_400 -const INLINE_BOT_PRESENCE_FAILED_MS = 8_000 -const INLINE_BOT_PRESENCE_COMMENT_MAX_LENGTH = 30 const INLINE_REPLY_THREAD_SYSTEM_PROMPT = "Inline reply threads are scoped conversations: reply to the current reply-thread message using this thread's own history, with parent-chat context as background. Don't pick up questions asked in the parent chat or other reply threads as if they were addressed here. Earlier conversation lives in the parent chat; read it with inline_parent_context. When someone here asks about another chat or thread you're in, you may read it with the message tool's read action." // Telegram's drop_pending_on_cold_boot, for Inline: updates the server recorded @@ -239,21 +265,6 @@ const INLINE_COLD_BOOT_GATED_EVENTS = new Set([ const INLINE_BOT_HAND_BACK_SYSTEM_PROMPT = "This message is another bot's reply to you. It needs no answer: your response will not reach that bot. To ask it for more, mention it." -type InlineBotPresenceKind = - | "idle" - | "happy" - | "waving" - | "jumping" - | "failed" - | "waiting" - | "running" - | "review" - -type InlineBotPresenceSignal = { - kind: InlineBotPresenceKind - comment?: string -} - type InlineMonitorHandle = { stop: () => Promise done: Promise @@ -285,13 +296,6 @@ async function dispatchInlineReplyWithSessionInitRetry(params: { } } -type InlineMentionSource = - | "explicit_bot" - | "subteam" - | "implicit_thread" - | "command_bypass" - | "none" - type StatusSink = (patch: { connected?: boolean lastConnectedAt?: number | null @@ -316,35 +320,6 @@ type InlineDiagnosticsSnapshot = { } | null } -type CachedChatInfo = { - kind: "direct" | "group" - title: string | null - peerUserId?: bigint | null - parentChatId?: bigint | null - parentMessageId?: bigint | null - lastMsgId?: bigint | null - dialogFollowMode?: DialogFollowMode | null -} - -type SenderProfileLookup = Pick - -type HistoryContext = { - historyText: string | null - attachmentText: string | null - entityText: string | null - inboundHistory: InlinePendingHistoryEntry[] - repliedToBot: boolean - replyToSenderId: string | null -} - -type InlineInboundMediaInfo = { - path?: string - contentType?: string | undefined - fileName?: string | undefined - kind?: "image" | "video" | "audio" | "document" | undefined - messageId?: string | undefined -} - type InlineEditStreamState = { messageId: bigint | null initialSendMayHaveLanded: boolean @@ -395,31 +370,6 @@ type InlineProgressPlaceholderState = { pendingClosures: Set } -type InlinePendingHistoryEntry = { - sender: string - body: string - timestamp?: number - messageId?: string -} - -type InlineChannelStructuredContextEntry = { - label: string - source: typeof CHANNEL_ID - type: string - payload: Record -} - -type InlineReplyThreadContext = { - childChatId: bigint - parentChatId: bigint - parentChatKind: CachedChatInfo["kind"] - parentPeerUserId?: bigint | null - parentChatTitle: string | null - parentMessageId: bigint | null - threadLabel: string | null - anchorMessage: Message | null -} - type InlineDispatchReplyInfo = { kind?: string reason?: string @@ -440,16 +390,6 @@ type InlineReplyDeliveryResult = { suppression?: { reason: "no_visible_result" } } -type InlineReplyPayload = { - isError?: boolean | undefined - text?: string | undefined - mediaUrl?: string | undefined - mediaUrls?: string[] | undefined - replyToId?: string | undefined - channelData?: Record | undefined - isReasoning?: boolean | undefined -} - function normalizeInlineDispatchResult(value: unknown): InlineDispatchResult { return isRecord(value) ? (value as InlineDispatchResult) : {} } @@ -506,375 +446,6 @@ function summarizeOpenClawCommandFailure(reply: string): string { : "OpenClaw did not apply the requested session model." } -function buildInlineTypingDispatcherOptions(typingCallbacks?: InlineTypingCallbacks) { - if (!typingCallbacks) { - return {} - } - return { - typingCallbacks, - onReplyStart: typingCallbacks.onReplyStart, - ...(typingCallbacks.onIdle ? { onIdle: typingCallbacks.onIdle } : {}), - ...(typingCallbacks.onCleanup ? { onCleanup: typingCallbacks.onCleanup } : {}), - } -} - -async function sendInlineTypingToChats(params: { - client: Pick - chatIds: bigint[] - typing: boolean - onPartialError?: (chatId: bigint, error: unknown) => void -}): Promise { - if (params.chatIds.length === 0) return - - const failures: Array<{ chatId: bigint; error: unknown }> = [] - await Promise.all( - params.chatIds.map(async (chatId) => { - try { - await sendInlineWorkingAction(params.client, { chatId, working: params.typing }) - } catch (error) { - failures.push({ chatId, error }) - } - }), - ) - if (failures.length === 0) return - if (failures.length === params.chatIds.length) { - throw failures[0]?.error ?? new Error("inline typing failed") - } - for (const failure of failures) { - params.onPartialError?.(failure.chatId, failure.error) - } -} - -function inlineBotPresenceStateKind(kind: InlineBotPresenceKind): BotPresenceState_Kind { - switch (kind) { - case "idle": - return BotPresenceState_Kind.IDLE - case "happy": - return BotPresenceState_Kind.HAPPY - case "waving": - return BotPresenceState_Kind.WAVING - case "jumping": - return BotPresenceState_Kind.JUMPING - case "failed": - return BotPresenceState_Kind.FAILED - case "waiting": - return BotPresenceState_Kind.WAITING - case "running": - return BotPresenceState_Kind.RUNNING - case "review": - return BotPresenceState_Kind.REVIEW - } -} - -function normalizeInlineBotPresenceKind(value: unknown): InlineBotPresenceKind | undefined { - if (typeof value !== "string") return undefined - switch (value.trim().toLowerCase()) { - case "idle": - case "happy": - case "waving": - case "jumping": - case "failed": - case "waiting": - case "running": - case "review": - return value.trim().toLowerCase() as InlineBotPresenceKind - default: - return undefined - } -} - -function normalizeInlineBotPresenceComment(value: unknown): string | undefined { - if (typeof value !== "string") return undefined - const text = value.replace(/\s+/g, " ").trim() - if (!text) return undefined - return Array.from(text).slice(0, INLINE_BOT_PRESENCE_COMMENT_MAX_LENGTH).join("") -} - -function resolveInlineBotPresenceSignal(payload: InlineReplyPayload): InlineBotPresenceSignal | null { - const channelData = isRecord(payload.channelData) ? payload.channelData : undefined - const inlineData = channelData && isRecord(channelData.inline) ? channelData.inline : undefined - const rawPresence = inlineData && isRecord(inlineData.botPresence) ? inlineData.botPresence : undefined - if (!rawPresence) return null - - const kind = normalizeInlineBotPresenceKind(rawPresence.kind) ?? "happy" - const comment = normalizeInlineBotPresenceComment(rawPresence.comment) - return comment ? { kind, comment } : { kind } -} - -async function sendInlineBotPresenceToChats(params: { - client: Pick - chatIds: bigint[] - kind: InlineBotPresenceKind - comment?: string - onPartialError?: (chatId: bigint, error: unknown) => void -}): Promise { - if (params.chatIds.length === 0) return - - const failures: Array<{ chatId: bigint; error: unknown }> = [] - await Promise.all( - params.chatIds.map(async (chatId) => { - try { - await params.client.invokeRaw(Method.SET_BOT_PRESENCE_STATE, { - oneofKind: "setBotPresenceState", - setBotPresenceState: { - peerId: { - type: { - oneofKind: "chat", - chat: { chatId }, - }, - }, - state: { - kind: inlineBotPresenceStateKind(params.kind), - ...(params.comment ? { comment: params.comment } : {}), - }, - }, - }) - } catch (error) { - failures.push({ chatId, error }) - } - }), - ) - if (failures.length === 0) return - if (failures.length === params.chatIds.length) { - throw failures[0]?.error ?? new Error("inline bot presence failed") - } - for (const failure of failures) { - params.onPartialError?.(failure.chatId, failure.error) - } -} - -async function sendInlineTypingAndBotPresenceToChats(params: { - client: Pick - chatIds: bigint[] - typing: boolean - presenceKind: InlineBotPresenceKind - onTypingPartialError?: (chatId: bigint, error: unknown) => void - onPresencePartialError?: (chatId: bigint, error: unknown) => void - onPresenceError?: (error: unknown) => void -}): Promise { - let typingError: unknown = null - await Promise.all([ - sendInlineTypingToChats({ - client: params.client, - chatIds: params.chatIds, - typing: params.typing, - ...(params.onTypingPartialError ? { onPartialError: params.onTypingPartialError } : {}), - }).catch((error) => { - typingError = error - }), - sendInlineBotPresenceToChats({ - client: params.client, - chatIds: params.chatIds, - kind: params.presenceKind, - ...(params.onPresencePartialError ? { onPartialError: params.onPresencePartialError } : {}), - }).catch((error) => { - params.onPresenceError?.(error) - }), - ]) - - if (typingError != null) { - throw typingError - } -} - -function createInlineBotPresenceLifecycle(params: { - client: Pick - chatIds: bigint[] - onPartialError: (chatId: bigint, error: unknown) => void - onError: (error: unknown) => void -}) { - let active = false - let finishing = false - let idleTimer: ReturnType | undefined - let gestureTimer: ReturnType | undefined - let currentKind: InlineBotPresenceKind | undefined - let currentComment: string | undefined - let busyKind: InlineBotPresenceKind = "running" - let busyComment: string | undefined - let sendChain: Promise = Promise.resolve() - let generation = 0 - - const clearIdleTimer = () => { - if (!idleTimer) return - clearTimeout(idleTimer) - idleTimer = undefined - } - - const clearGestureTimer = () => { - if (!gestureTimer) return - clearTimeout(gestureTimer) - gestureTimer = undefined - } - - const send = (kind: InlineBotPresenceKind, comment?: string): Promise => { - if (currentKind === kind && currentComment === comment) return Promise.resolve() - const chatIds = params.chatIds - currentKind = kind - currentComment = comment - sendChain = sendChain - .catch(() => {}) - .then(() => - sendInlineBotPresenceToChats({ - client: params.client, - chatIds, - kind, - ...(comment ? { comment } : {}), - onPartialError: params.onPartialError, - }).catch(params.onError), - ) - return sendChain - } - - const scheduleIdle = ( - delayMs = INLINE_BOT_PRESENCE_IDLE_DELAY_MS, - expectedGeneration = generation, - ) => { - clearIdleTimer() - idleTimer = setTimeout(() => { - idleTimer = undefined - if (generation !== expectedGeneration) return - active = false - finishing = false - busyKind = "running" - busyComment = undefined - void send("idle") - }, delayMs) - idleTimer.unref?.() - } - - const showBusy = async (kind: InlineBotPresenceKind, comment?: string): Promise => { - if (!active || finishing) return - clearIdleTimer() - clearGestureTimer() - busyKind = kind - busyComment = comment - await send(kind, comment) - } - - const gesture = ( - kind: InlineBotPresenceKind, - resumeKind = busyKind, - comment?: string, - resumeComment = busyComment, - ) => { - if (params.chatIds.length === 0) return - clearIdleTimer() - clearGestureTimer() - active = true - const expectedGeneration = generation - void send(kind, comment) - gestureTimer = setTimeout(() => { - gestureTimer = undefined - if (generation !== expectedGeneration) return - if (!active || finishing) return - void send(resumeKind, resumeComment) - }, INLINE_BOT_PRESENCE_GESTURE_MS) - gestureTimer.unref?.() - } - - return { - retarget: async (chatId: bigint) => { - generation += 1 - clearIdleTimer() - clearGestureTimer() - const oldChatIds = params.chatIds - params.chatIds = [chatId] - // Adoption means work continues in the child even if a parent gesture was - // finishing. Start a fresh lifecycle so the child cannot remain busy. - active = true - finishing = false - currentKind = undefined - currentComment = undefined - sendChain = sendChain.catch(() => {}).then(() => sendInlineBotPresenceToChats({ - client: params.client, - chatIds: oldChatIds, - kind: "idle", - onPartialError: params.onPartialError, - }).catch(params.onError)) - await send(busyKind, busyComment) - }, - start: async (cue?: InlineBotPresenceSignal | null) => { - if (params.chatIds.length === 0) return - clearIdleTimer() - if (active && !finishing) return - active = true - finishing = false - const cueKind = cue?.kind - busyKind = cueKind === "review" ? "review" : "running" - busyComment = undefined - if (cueKind === "waving") { - gesture("waving", busyKind) - return - } - await send(busyKind, busyComment) - }, - busy: (kind: InlineBotPresenceKind, comment?: string) => { - void showBusy(kind, comment) - }, - gesture: (kind: InlineBotPresenceKind, comment?: string) => { - gesture(kind, busyKind, comment) - }, - express: (signal: InlineBotPresenceSignal) => { - if (signal.kind === "failed") { - if (params.chatIds.length === 0) return - clearIdleTimer() - clearGestureTimer() - active = true - finishing = true - const expectedGeneration = generation - void send("failed", signal.comment).finally(() => { - if (generation === expectedGeneration) { - scheduleIdle(INLINE_BOT_PRESENCE_FAILED_MS, expectedGeneration) - } - }) - return - } - if (params.chatIds.length === 0) return - clearIdleTimer() - clearGestureTimer() - active = true - finishing = true - const expectedGeneration = generation - void send(signal.kind, signal.comment).finally(() => { - if (generation === expectedGeneration) scheduleIdle(undefined, expectedGeneration) - }) - }, - finish: () => { - if (!active || finishing) return - finishing = true - clearGestureTimer() - scheduleIdle(INLINE_BOT_PRESENCE_FINISH_IDLE_DELAY_MS) - }, - fail: (comment?: string) => { - if (params.chatIds.length === 0) return - clearIdleTimer() - clearGestureTimer() - active = true - finishing = true - const expectedGeneration = generation - void send("failed", comment).finally(() => { - if (generation === expectedGeneration) { - scheduleIdle(INLINE_BOT_PRESENCE_FAILED_MS, expectedGeneration) - } - }) - }, - cleanup: () => { - if (!active) return - if (finishing) return - clearIdleTimer() - clearGestureTimer() - active = false - const expectedGeneration = generation - void send("idle").finally(() => { - if (generation !== expectedGeneration) return - finishing = false - busyKind = "running" - busyComment = undefined - }) - }, - } -} - function buildInlineReplyThreadSessionKey(parentSessionKey: string, threadChatId: bigint): string { const suffix = `:thread:${String(threadChatId)}` return parentSessionKey.endsWith(suffix) ? parentSessionKey : `${parentSessionKey}${suffix}` @@ -983,43 +554,9 @@ function isInlineSendOutcomeUnknown(error: unknown): boolean { return isInlineCommitOutcomeUnknown(error) || inlineErrorChainHasCode(error, "timeout") } -type InlineHistoryEntryPayload = { - line: string | null - attachmentLine: string | null - entityLine: string | null - inboundEntry: InlinePendingHistoryEntry | null -} - -const DEFAULT_DM_HISTORY_LIMIT = 6 -const HISTORY_LINE_MAX_CHARS = 280 -const REPLY_THREAD_LABEL_MAX_CHARS = 72 -const URL_LIKE_PATTERN = /https?:\/\/\S+/i -const BOT_MESSAGE_CACHE_LIMIT = 500 -const REACTION_TARGET_LOOKUP_LIMIT = 8 -const REPLY_TARGET_LOOKUP_LIMIT = 8 -const ATTACHMENT_CONTEXT_LIMIT = 6 -const DEFAULT_INLINE_MEDIA_MAX_BYTES = 300 * 1024 * 1024 const INLINE_EDIT_STREAM_MIN_INTERVAL_MS = 250 const INLINE_EDIT_STREAM_MAX_INTERVAL_MS = 750 const EMPTY_RESPONSE_FALLBACK = "No response generated. Please try again." -const GET_MESSAGES_METHOD = - typeof (Method as Record).GET_MESSAGES === "number" && - Number.isInteger((Method as Record).GET_MESSAGES) && - ((Method as Record).GET_MESSAGES as number) > 0 - ? ((Method as Record).GET_MESSAGES as Method) - : null -const REASONING_MESSAGE_PREFIX = "Reasoning:\n" -const REASONING_TAG_PREFIXES = [ - "]*>/gi function normalizeAllowEntry(raw: string): string { const withoutChannel = raw.trim().replace(/^inline:/i, "").trim() @@ -1029,139 +566,12 @@ function normalizeAllowEntry(raw: string): string { return withoutChannel.replace(/^user:/i, "").trim() } -function resolveInlinePayloadMediaUrls(payload: InlineReplyPayload): string[] { - const urls = payload.mediaUrls?.length - ? payload.mediaUrls - : payload.mediaUrl - ? [payload.mediaUrl] - : [] - return Array.from(new Set(urls.map((url) => url.trim()).filter(Boolean))) -} - function fingerprintInlineMessageActions(actions: MessageActions | undefined): string | null { if (actions === undefined) return null return JSON.stringify(actions, (_key, value) => value instanceof Uint8Array ? Array.from(value) : value) } -function isPartialReasoningTagPrefix(text: string): boolean { - const trimmed = normalizeLowercaseStringOrEmpty(text.trimStart()) - if (!trimmed.startsWith("<")) return false - if (trimmed.includes(">")) return false - return REASONING_TAG_PREFIXES.some((prefix) => prefix.startsWith(trimmed)) -} - -type InlineReasoningSplit = { - reasoningText?: string - answerText?: string -} - -function formatInlineReasoningMessage(text: string): string { - const trimmed = text.trim() - if (!trimmed) return "" - const italicLines = trimmed - .split("\n") - .map((line) => (line ? `_${line}_` : line)) - .join("\n") - return `${REASONING_MESSAGE_PREFIX}${italicLines}` -} - -function extractInlineThinkingFromTaggedStreamOutsideCode(text: string): string { - if (!text) return "" - const codeRegions = findCodeRegions(text) - let result = "" - let lastIndex = 0 - let inThinking = false - THINKING_TAG_RE.lastIndex = 0 - for (const match of text.matchAll(THINKING_TAG_RE)) { - const idx = match.index ?? 0 - if (isInsideCode(idx, codeRegions)) continue - if (inThinking) { - result += text.slice(lastIndex, idx) - } - const isClose = match[1] === "/" - inThinking = !isClose - lastIndex = idx + match[0].length - } - if (inThinking) { - result += text.slice(lastIndex) - } - return result.trim() -} - -function splitInlineReasoningText(text?: string, isReasoning?: boolean): InlineReasoningSplit { - if (typeof text !== "string") return {} - - const trimmed = text.trim() - if (isPartialReasoningTagPrefix(trimmed)) { - return {} - } - if ( - trimmed.startsWith(REASONING_MESSAGE_PREFIX) && - trimmed.length > REASONING_MESSAGE_PREFIX.length - ) { - return { reasoningText: trimmed } - } - - const taggedReasoning = extractInlineThinkingFromTaggedStreamOutsideCode(text) - const strippedAnswer = stripReasoningTagsFromText(text, { mode: "strict", trim: "both" }) - - if (isReasoning === true) { - return { - reasoningText: formatInlineReasoningMessage( - taggedReasoning || strippedAnswer || text, - ), - } - } - if (!taggedReasoning && strippedAnswer === text) { - return { answerText: text } - } - const reasoningText = taggedReasoning - ? formatInlineReasoningMessage(taggedReasoning) - : undefined - return { - ...(reasoningText ? { reasoningText } : {}), - ...(strippedAnswer ? { answerText: strippedAnswer } : {}), - } -} - -function resolveInlineChatVisibleReplyPayload(payload: T): T | null { - const mediaUrls = resolveInlinePayloadMediaUrls(payload) - if (isReasoningReplyPayload({ - ...(typeof payload.text === "string" ? { text: payload.text } : {}), - ...(payload.isReasoning !== undefined ? { isReasoning: payload.isReasoning } : {}), - })) { - return mediaUrls.length > 0 ? { ...payload, text: undefined } : null - } - - if (typeof payload.text !== "string") { - return payload - } - - const internal = sanitizeInlineVisibleText(payload.text) - if (internal.shouldSkip) { - return mediaUrls.length > 0 ? { ...payload, text: undefined } : null - } - if (isPartialReasoningTagPrefix(internal.text.trim())) { - return mediaUrls.length > 0 ? { ...payload, text: undefined } : null - } - - const split = splitInlineReasoningText(internal.text, payload.isReasoning) - if (split.answerText !== undefined) { - if (split.answerText === payload.text) { - return payload - } - return { - ...payload, - text: split.answerText || undefined, - } - } - if (!split.reasoningText) { - return payload - } - return mediaUrls.length > 0 ? { ...payload, text: undefined } : null -} - function normalizeAllowlist(entries: Array | undefined): string[] { return (entries ?? []) .map((entry) => normalizeAllowEntry(String(entry))) @@ -1233,15 +643,15 @@ async function resolveInlineGroupSenderAllowlist(params: { } } -function resolveInlineCommandAuthorized(params: { +async function resolveInlineCommandAuthorized(params: { cfg: OpenClawConfig accountId: string isGroup: boolean chatId: string senderId: string commandAuthorized: boolean -}): boolean { - return resolveCommandAuthorization({ +}): Promise { + const authorization = await resolveCommandAuthorizationAsync({ ctx: { Provider: CHANNEL_ID, Surface: CHANNEL_ID, @@ -1254,39 +664,14 @@ function resolveInlineCommandAuthorized(params: { }, cfg: params.cfg, commandAuthorized: params.commandAuthorized, - }).isAuthorizedSender + }) + return authorization.isAuthorizedSender } function hasExplicitInlineCommandAuthorizationPolicy(cfg: OpenClawConfig): boolean { return cfg.commands?.ownerAllowFrom !== undefined || cfg.commands?.allowFrom !== undefined } -async function resolveChatInfo( - client: InlineSdkClient, - cache: Map, - chatId: bigint, -): Promise { - const existing = cache.get(chatId) - if (existing) return existing - - const result = await client.getChat({ chatId }) - const peerType = result.peer?.type - const kind: CachedChatInfo["kind"] = peerType?.oneofKind === "user" ? "direct" : "group" - const peerUserId = peerType?.oneofKind === "user" ? peerType.user.userId : null - const title = result.title?.trim() || null - const info: CachedChatInfo = { - kind, - title, - ...(peerUserId != null ? { peerUserId } : {}), - ...(result.parentChatId != null ? { parentChatId: result.parentChatId } : {}), - ...(result.parentMessageId != null ? { parentMessageId: result.parentMessageId } : {}), - ...(result.lastMsgId != null ? { lastMsgId: result.lastMsgId } : {}), - ...(result.dialogFollowMode != null ? { dialogFollowMode: result.dialogFollowMode } : {}), - } - cache.set(chatId, info) - return info -} - function formatInlineSystemEventChannelLabel(params: { chatInfo: CachedChatInfo effectiveChatId: bigint @@ -1339,290 +724,6 @@ function buildInlineReactionContextKey(params: { return `inline:reaction:${params.action}:${String(params.chatId)}:${String(params.messageId)}:${String(params.senderId)}:${emoji}` } -function normalizeInlineUsername(raw: string | undefined): string | undefined { - const trimmed = raw?.trim() - if (!trimmed) return undefined - return trimmed.startsWith("@") ? trimmed.slice(1) : trimmed -} - -function resolveInlineCommandTarget(raw: string, botUsername: string | undefined): { - body: string - targeted: boolean - addressedToMe: boolean -} { - const normalized = raw.trim() - const match = normalized.match(/^\/([^\s@]+)@([a-z0-9_]+)(?=\s|$)([\s\S]*)$/i) - if (!match) return { body: normalized, targeted: false, addressedToMe: false } - const [, command, targetUsername, suffix] = match - const normalizedBotUsername = normalizeInlineUsername(botUsername)?.toLowerCase() - const addressedToMe = Boolean( - normalizedBotUsername && targetUsername?.toLowerCase() === normalizedBotUsername, - ) - return { - body: addressedToMe ? `/${command}${suffix ?? ""}` : normalized, - targeted: true, - addressedToMe, - } -} - -function normalizeInlineCommandBody(raw: string, botUsername: string | undefined): string { - return resolveInlineCommandTarget(raw, botUsername).body -} - -// The app attaches the bot a command was picked for to a leading command -// entity. Returns that bot's user id, or null when the command names none. -function leadingInlineBotCommandTargetUserId(msg: Message): string | null { - const text = typeof msg.message === "string" ? msg.message : "" - for (const entity of msg.entities?.entities ?? []) { - if (entity.entity.oneofKind !== "botCommand") continue - const offset = Number(entity.offset) - if (!Number.isInteger(offset) || offset < 0) continue - if (text.slice(0, offset).trim()) continue - const botUserId = entity.entity.botCommand.botUserId - return botUserId > 0n ? String(botUserId) : null - } - return null -} - -// The server stamps the bot a bare command settles on; no stamp means no bot won. -function startsWithUnclaimedInlineBotCommand(msg: Message): boolean { - const text = typeof msg.message === "string" ? msg.message : "" - for (const entity of msg.entities?.entities ?? []) { - if (entity.entity.oneofKind !== "botCommand") continue - const offset = Number(entity.offset) - if (!Number.isInteger(offset) || offset < 0) continue - if (text.slice(0, offset).trim()) continue - return entity.entity.botCommand.botUserId <= 0n - } - return false -} - -function isInlineBotCommandForOtherBot(msg: Message, meId: bigint): boolean { - const target = leadingInlineBotCommandTargetUserId(msg) - return target != null && target !== String(meId) -} - -function isInlineAbortRequestMessage(msg: Message, botUsername: string | undefined, meId: bigint): boolean { - const text = typeof msg.message === "string" ? msg.message : "" - if (isInlineBotCommandForOtherBot(msg, meId)) return false - return isAbortRequestText(text, botUsername ? { botUsername } : undefined) -} - -function findInlineNativeCommandFromBody(commandBody: string): NonNullable> | null { - const match = commandBody.trim().match(/^\/([^\s]+)(?:\s+[\s\S]*)?$/) - if (!match?.[1]) return null - return findCommandByNativeName(match[1], INLINE_NATIVE_COMMAND_PROVIDER) ?? null -} - -function resolveInlineCommandNameFromBody(commandBody: string): string | null { - const match = commandBody.trim().match(/^\/([^\s]+)(?:\s+[\s\S]*)?$/) - const raw = match?.[1]?.trim().toLowerCase() - return raw || null -} - -function buildInlineNativeCommandCallbackData(commandText: string): string { - return `${INLINE_NATIVE_COMMAND_CALLBACK_PREFIX}${commandText}` -} - -function parseInlineNativeCommandCallbackData(raw: string | undefined): string | null { - if (!raw) return null - const trimmed = raw.trim() - if (!trimmed.startsWith(INLINE_NATIVE_COMMAND_CALLBACK_PREFIX)) return null - const commandText = trimmed.slice(INLINE_NATIVE_COMMAND_CALLBACK_PREFIX.length).trim() - return commandText.startsWith("/") ? commandText : null -} - -function listHostInlinePluginCommandSpecs(cfg: OpenClawConfig): Array<{ name: string; description: string }> { - return getPluginCommandSpecs("inline", { config: cfg }) -} - -function listInlinePluginCommandSpecs(cfg: OpenClawConfig): Array<{ name: string; description: string }> { - return [ - ...listHostInlinePluginCommandSpecs(cfg), - ...listInlineFollowCommandSpecs(), - ...listInlineBuiltinCommandSpecs(), - ...listInlineMaintenanceCommandSpecs(), - ] -} - -function hasInlineCommandSpec( - specs: Array<{ name: string }>, - commandName: string, -): boolean { - const normalized = commandName.trim().toLowerCase() - if (!normalized) return false - return specs.some((spec) => spec.name.trim().toLowerCase() === normalized) -} - -function isInlineThreadReplyCommandBody(commandBody: string): boolean { - return resolveInlineCommandNameFromBody(commandBody) === "threads" -} - -function parseInlineCommandArgs(commandBody: string): string { - const match = commandBody.trim().match(/^\/[^\s]+(?:\s+([\s\S]*))?$/) - return match?.[1]?.trim() ?? "" -} - -function isInlineNativeCommandBody(params: { - cfg: OpenClawConfig - account: ResolvedInlineAccount - commandBody: string - agentId: string -}): boolean { - if (!shouldSyncInlineNativeCommandsForAccount({ cfg: params.cfg, account: params.account })) { - return false - } - const commandName = resolveInlineCommandNameFromBody(params.commandBody) - if (!commandName) return false - - const skillCommands = shouldSyncInlineNativeSkillsForAccount({ - cfg: params.cfg, - account: params.account, - }) - ? listSkillCommandsForAgents({ - cfg: params.cfg, - agentIds: [params.agentId], - }) - : [] - const commandSpecs = listNativeCommandSpecsForConfig(params.cfg, { - skillCommands, - provider: INLINE_NATIVE_COMMAND_PROVIDER, - }) - for (const spec of commandSpecs) { - if (spec.name.trim().toLowerCase() === commandName) return true - } - for (const spec of listInlinePluginCommandSpecs(params.cfg)) { - if (spec.name.trim().toLowerCase() === commandName) return true - } - return false -} - -function escapeInlineRegExp(raw: string): string { - return raw.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") -} - -function stripInlineBotMention(raw: string, botUsername: string | undefined): string { - const username = normalizeInlineUsername(botUsername) - if (!username) return raw - const mention = new RegExp(`(^|\\s)@${escapeInlineRegExp(username)}(?=$|[\\s,.:;!?])(?:[\\s,.:;!?]+)?`, "gi") - return raw.replace(mention, "$1").replace(/[ \t]{2,}/g, " ").trim() -} - -function stripInlineBotMentionEntityText(text: string | null, botUsername: string | undefined): string | null { - const username = normalizeInlineUsername(botUsername)?.toLowerCase() - if (!text || !username) return text - const token = `@${username}` - const parts = text - .split(" | ") - .filter((part) => { - const normalized = part.trim().toLowerCase() - return normalized !== `username mention "${token}"` && !normalized.startsWith(`mention "${token}"`) - }) - const next = parts.join(" | ").trim() - return next || null -} - -function collectInlineMentionedUserIds( - content: ReturnType | null, -): string[] { - const ids = new Set() - for (const entity of content?.entities ?? []) { - if (entity.type !== "mention" || !entity.userId) continue - ids.add(entity.userId) - } - return [...ids] -} - -// User ids in the run of mentions a message starts with, in order. That run -// names who the message is for ("@alice @bot compare"). -function leadingInlineMentionUserIds(params: { - content: ReturnType | null - messageText: string | undefined -}): string[] { - const mentions = (params.content?.entities ?? []) - .filter((entity): entity is InlineMessageEntitySummary & { userId: string } => - entity.type === "mention" && - Boolean(entity.userId) && - Number.isInteger(entity.offset) && - entity.offset >= 0 && - Number.isInteger(entity.length) && - entity.length > 0) - .sort((a, b) => a.offset - b.offset) - // Inline entity offsets use UTF-16 code units, which matches JavaScript slice. - const text = params.messageText ?? "" - const leading: string[] = [] - let cursor = 0 - for (const mention of mentions) { - if (mention.offset < cursor) break - if (text.slice(cursor, mention.offset).replace(/,/g, "").trim()) break - leading.push(mention.userId) - cursor = mention.offset + mention.length - } - return leading -} - -function isInlineDialogFollowing(followMode: unknown): boolean { - if (followMode == null || typeof followMode === "boolean") return false - if (typeof followMode === "number") return followMode === DialogFollowMode.FOLLOWING - if (typeof followMode === "bigint") return followMode === BigInt(DialogFollowMode.FOLLOWING) - const text = String(followMode).trim().toLowerCase() - return ( - text === String(DialogFollowMode.FOLLOWING) || - text === "following" || - text === "follow_mode_following" || - text === "dialog_following" - ) -} - -function isInlineDialogExplicitlyUnfollowed(followMode: unknown): boolean { - if (followMode == null || typeof followMode === "boolean") return false - if (typeof followMode === "number") return followMode === DIALOG_FOLLOW_MODE_UNFOLLOWED - if (typeof followMode === "bigint") { - return followMode === BigInt(DIALOG_FOLLOW_MODE_UNFOLLOWED) - } - const text = String(followMode).trim().toLowerCase() - return ( - text === String(DIALOG_FOLLOW_MODE_UNFOLLOWED) || - text === "unfollowed" || - text === "follow_mode_unfollowed" || - text === "dialog_unfollowed" - ) -} - -// Only structured Inline addressing wakes the agent; words such as its name never do. -function resolveInlineMentionSource(params: { - nativeMentioned: boolean - implicitMention: boolean - shouldBypassMention: boolean -}): InlineMentionSource { - if (params.nativeMentioned) return "explicit_bot" - if (params.shouldBypassMention) return "command_bypass" - if (params.implicitMention) return "implicit_thread" - return "none" -} - -function callbackDataToBase64(data: Uint8Array): string { - return Buffer.from(data).toString("base64") -} - -function callbackDataToUtf8(data: Uint8Array): string | undefined { - try { - const decoded = new TextDecoder("utf-8", { fatal: true }).decode(data).trim() - return decoded || undefined - } catch { - return undefined - } -} - -type InlineModelPickerCallback = - | { type: "providers" | "back" } - | { type: "list"; provider: string; page: number } - | { type: "select"; provider?: string; model: string } - -type InlineModelPickerSelection = - | { kind: "resolved"; provider: string; model: string } - | { kind: "ambiguous"; model: string } - function buildInlineInboundMessageSid(params: { msgId: bigint callbackActionEvent?: { @@ -1654,1670 +755,187 @@ function claimInlineInboundMessageInstance(params: { String(message.entities?.entities.length ?? 0), ].join("\u0000") const key = `${String(params.chatId)}:${String(message.id)}:${String(message.date ?? 0n)}:${content}` - if (params.seen.has(key)) return false - params.seen.set(key, true) - const maximum = params.maximum ?? 5_000 - while (params.seen.size > maximum) { - const oldest = params.seen.keys().next().value - if (oldest == null) break - params.seen.delete(oldest) - } - return true -} - -function buildInlineDebounceKey(params: { - accountId: string - chatId: bigint - senderId: bigint | null | undefined -}): string | null { - if (params.senderId == null) return null - return `inline:${params.accountId}:${String(params.chatId)}:${String(params.senderId)}` -} - -function buildInlineInboundTaskKey(params: { - accountId: string - chatId: bigint -}): string { - return `inline:${params.accountId}:${String(params.chatId)}` -} - -function buildSyntheticInlineTextMessage(params: { - base: Message - text: string - mentioned?: boolean -}): Message { - return { - ...params.base, - message: params.text, - ...(params.mentioned !== undefined ? { mentioned: params.mentioned } : {}), - } -} - -function buildInlineTranscriptTextMessage(params: { - base: Message - text: string -}): Message { - const { media: _media, ...base } = params.base as Message & { media?: unknown } - return { - ...base, - message: params.text, - } as Message -} - -function buildInlineVoicePendingKey(params: { chatId: bigint; messageId: bigint }): string { - return `${String(params.chatId)}:${String(params.messageId)}` -} - -function resolveInlineVoiceTranscriptWaitMs(config: { voiceTranscriptWaitMs?: unknown }): number { - const raw = config.voiceTranscriptWaitMs - if (typeof raw !== "number" || !Number.isFinite(raw)) { - return DEFAULT_INLINE_VOICE_TRANSCRIPT_WAIT_MS - } - return Math.max(0, Math.min(MAX_INLINE_VOICE_TRANSCRIPT_WAIT_MS, Math.trunc(raw))) -} - -function resolveInlineInboundDebounceMsOverride(config: { debounceMs?: unknown }): number | undefined { - const raw = config.debounceMs - if (typeof raw !== "number" || !Number.isFinite(raw)) { - return undefined - } - return Math.max(0, Math.trunc(raw)) -} - -function shouldWaitForInlineVoiceTranscript(message: Message): boolean { - const content = summarizeInlineMessageContent(message) - return content.media?.kind === "voice" && !content.rawText -} - -function extractInlineVoiceTranscriptText(message: Message): string | null { - const text = message.message?.trim() - return text ? text : null -} - -const INLINE_ACTION_MAX_ROWS = 8 -const INLINE_ACTION_MAX_PER_ROW = 8 -const INLINE_CALLBACK_TOAST_MAX_LENGTH = 256 - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value) -} - -function normalizeInlineCallbackToastText(raw: unknown): string | undefined { - if (typeof raw !== "string") return undefined - - const visible = sanitizeInlineVisibleText(raw) - if (visible.shouldSkip) return undefined - - const text = visible.text.replace(/\s+/g, " ").trim() - if (!text) return undefined - - if (text.length <= INLINE_CALLBACK_TOAST_MAX_LENGTH) { - return text - } - - return text.slice(0, INLINE_CALLBACK_TOAST_MAX_LENGTH).trimEnd() -} - -function resolveInlineCallbackResponseUi(data: Uint8Array): MessageActionResponseUi | undefined { - const decoded = callbackDataToUtf8(data) - if (!decoded) return undefined - - let parsed: unknown - try { - parsed = JSON.parse(decoded) - } catch { - return undefined - } - - if (!isRecord(parsed)) return undefined - - const text = - normalizeInlineCallbackToastText(parsed.callbackToast) ?? - normalizeInlineCallbackToastText(parsed.callback_toast) ?? - normalizeInlineCallbackToastText(parsed.toast) - - if (!text) return undefined - - return { - kind: { - oneofKind: "toast", - toast: { text }, - }, - } -} - -function normalizeOptionalString(value: unknown): string | undefined { - return typeof value === "string" && value.trim() ? value.trim() : undefined -} - -type InlineStreamingMode = "off" | "partial" | "block" | "progress" - -function normalizeInlineStreamingMode(value: unknown): InlineStreamingMode | null { - if (typeof value !== "string") return null - const normalized = normalizeLowercaseStringOrEmpty(value) - switch (normalized) { - case "off": - case "partial": - case "block": - case "progress": - return normalized - default: - return null - } -} - -function resolveExplicitInlineStreamingMode(config: { - streaming?: unknown - streamMode?: unknown -}): InlineStreamingMode | null { - const streaming = isRecord(config.streaming) ? config.streaming : null - return ( - normalizeInlineStreamingMode(streaming?.mode) ?? - normalizeInlineStreamingMode(config.streaming) ?? - normalizeInlineStreamingMode(config.streamMode) - ) -} - -function resolveInlineStreamingMode(config: { - streaming?: unknown - streamMode?: unknown - streamViaEditMessage?: boolean | undefined -}): InlineStreamingMode { - const mode = resolveExplicitInlineStreamingMode(config) - if (mode) return mode - if (typeof config.streaming === "boolean") { - return config.streaming ? "partial" : "off" - } - return config.streamViaEditMessage === true ? "partial" : "off" -} - -function resolveInlineProgressPlaceholderEnabled(config: { - streaming?: unknown - streamMode?: unknown - streamViaEditMessage?: boolean | undefined -}): boolean { - const mode = resolveExplicitInlineStreamingMode(config) - if (mode === "off") return false - if (mode === "progress") return true - if (config.streaming === false) return false - if (isRecord(config.streaming) && isRecord(config.streaming.progress)) return true - - return ( - config.streaming === undefined && - config.streamMode === undefined && - config.streamViaEditMessage !== true - ) -} - -function resolveInlineBlockStreamingEnabled(config: { - streaming?: unknown - streamMode?: unknown - blockStreaming?: boolean | undefined - streamViaEditMessage?: boolean | undefined -}): boolean | undefined { - const mode = resolveInlineStreamingMode(config) - const streaming = isRecord(config.streaming) ? config.streaming : null - const block = streaming && isRecord(streaming.block) ? streaming.block : null - if (typeof block?.enabled === "boolean") return block.enabled - if (typeof config.blockStreaming === "boolean") return config.blockStreaming - if (mode === "block") return true - if (mode === "off") return false - return undefined -} - -function shouldStartInlineProgressPlaceholderNow( - line: string | ChannelProgressDraftLine | undefined, -): boolean { - return typeof line === "object" && line?.kind === "patch" && Boolean(line.detail) -} - -function resolveInlineParentSessionKeyCandidate( - sessionKey: string, - entry: { parentSessionKey?: string } | undefined, -): string | undefined { - const explicit = normalizeOptionalString(entry?.parentSessionKey) - if (explicit && explicit !== sessionKey) return explicit - - const inlineThreadMarker = ":thread:" - const markerIndex = sessionKey.lastIndexOf(inlineThreadMarker) - if (markerIndex > 0) return sessionKey.slice(0, markerIndex) - - return undefined -} - -type InlineSessionModelEntry = NonNullable> & { - fallbackNoticeSelectedModel?: string - fallbackNoticeActiveModel?: string -} - -type InlineProviderModelRef = { - provider: string - model: string -} - -function resolveInlineProviderModelRef( - providerValue: unknown, - modelValue: unknown, -): InlineProviderModelRef | undefined { - let provider = normalizeOptionalString(providerValue) - let model = normalizeOptionalString(modelValue) - if (!model) return undefined - - const slash = model.indexOf("/") - if (!provider && slash > 0 && slash < model.length - 1) { - provider = model.slice(0, slash) - model = model.slice(slash + 1) - } else if (provider && model.startsWith(`${provider}/`)) { - model = model.slice(provider.length + 1) - } - - return provider && model ? { provider, model } : undefined -} - -function resolveInlineFallbackModelContext(params: { - entry?: InlineSessionModelEntry - parentEntry?: InlineSessionModelEntry - defaultModel: InlineProviderModelRef - storedOverride?: InlineProviderModelRef | null -}): { - selected: InlineProviderModelRef - activeFallback?: InlineProviderModelRef -} { - const entryHasOverride = Boolean( - normalizeOptionalString(params.entry?.providerOverride) || - normalizeOptionalString(params.entry?.modelOverride), - ) - const parentHasOverride = Boolean( - normalizeOptionalString(params.parentEntry?.providerOverride) || - normalizeOptionalString(params.parentEntry?.modelOverride), - ) - const overrideEntry = entryHasOverride - ? params.entry - : parentHasOverride - ? params.parentEntry - : undefined - const fallbackEntry = overrideEntry?.modelOverrideSource === "auto" || - (normalizeOptionalString(overrideEntry?.modelOverrideFallbackOriginProvider) && - normalizeOptionalString(overrideEntry?.modelOverrideFallbackOriginModel)) - ? overrideEntry - : undefined - - const selected = fallbackEntry - ? resolveInlineProviderModelRef( - fallbackEntry.modelOverrideFallbackOriginProvider, - fallbackEntry.modelOverrideFallbackOriginModel, - ) ?? resolveInlineProviderModelRef(undefined, fallbackEntry.fallbackNoticeSelectedModel) - ?? params.defaultModel - : params.storedOverride ?? params.defaultModel - if (!fallbackEntry) return { selected } - - const activeFallback = resolveInlineProviderModelRef( - params.entry?.modelProvider, - params.entry?.model, - ) ?? resolveInlineProviderModelRef( - fallbackEntry.providerOverride, - fallbackEntry.modelOverride, - ) ?? resolveInlineProviderModelRef(undefined, fallbackEntry.fallbackNoticeActiveModel) - if ( - !activeFallback || - (activeFallback.provider === selected.provider && activeFallback.model === selected.model) - ) { - return { selected } - } - return { selected, activeFallback } -} - -function resolveInlineCommandMenuModelContext(params: { - cfg: OpenClawConfig - agentId: string - sessionKey: string -}): { - provider?: string - model?: string - activeProvider?: string - activeModel?: string - thinkingLevel?: string -} { - if (!params.sessionKey.trim()) { - return {} - } - try { - const storePath = resolveStorePath(params.cfg.session?.store, { agentId: params.agentId }) - const defaultModel = resolveDefaultModelForAgent({ - cfg: params.cfg, - agentId: params.agentId, - }) - const entry = getSessionEntry({ - agentId: params.agentId, - sessionKey: params.sessionKey, - storePath, - readConsistency: "latest", - }) as InlineSessionModelEntry | undefined - const thinkingLevel = normalizeOptionalString(entry?.thinkingLevel) - const parentSessionKey = resolveInlineParentSessionKeyCandidate(params.sessionKey, entry) - const parentEntry = parentSessionKey - ? getSessionEntry({ - agentId: params.agentId, - sessionKey: parentSessionKey, - storePath, - readConsistency: "latest", - }) as InlineSessionModelEntry | undefined - : undefined - const override = resolveStoredModelOverride({ - ...(entry ? { sessionEntry: entry } : {}), - ...(parentSessionKey ? { parentSessionKey } : {}), - ...(parentSessionKey && parentEntry ? { sessionStore: { [parentSessionKey]: parentEntry } } : {}), - sessionKey: params.sessionKey, - defaultProvider: defaultModel.provider, - }) - const modelContext = resolveInlineFallbackModelContext({ - ...(entry ? { entry } : {}), - ...(parentEntry ? { parentEntry } : {}), - defaultModel, - ...(override?.model - ? { - storedOverride: { - provider: override.provider || defaultModel.provider, - model: override.model, - }, - } - : {}), - }) - return { - provider: modelContext.selected.provider, - model: modelContext.selected.model, - ...(modelContext.activeFallback - ? { - activeProvider: modelContext.activeFallback.provider, - activeModel: modelContext.activeFallback.model, - } - : {}), - ...(thinkingLevel ? { thinkingLevel } : {}), - } - } catch { - return {} - } -} - -function resolveInlineThinkMenuCurrentLevel(params: { - cfg: OpenClawConfig - agentId: string - provider?: string - model?: string - thinkingLevel?: string -}): string { - const explicit = normalizeOptionalString(params.thinkingLevel) - if (explicit) return explicit - - const agentThinkingDefault = normalizeOptionalString( - resolveAgentConfig(params.cfg, params.agentId)?.thinkingDefault, - ) - if (agentThinkingDefault) return agentThinkingDefault - - const defaultModel = resolveDefaultModelForAgent({ - cfg: params.cfg, - agentId: params.agentId, - }) - return resolveThinkingDefault({ - cfg: params.cfg, - provider: params.provider ?? defaultModel.provider, - model: params.model ?? defaultModel.model, - catalog: buildConfiguredModelCatalog({ cfg: params.cfg }), - }) -} - -function formatInlineCommandArgMenuTitle(params: { - command: NonNullable> - menu: NonNullable> - currentThinkingLevel?: string -}): string { - const title = formatCommandArgMenuTitle({ command: params.command, menu: params.menu }) - if (params.command.key !== "think" || !params.currentThinkingLevel) { - return title - } - return `Current thinking level: ${params.currentThinkingLevel}.\n${title}` -} - -function resolveInlineNativeCommandMenu(params: { - commandBody: string - cfg: OpenClawConfig - agentId: string - sessionKey: string -}): { - title: string - buttons: InlineReplyMarkupButton[][] -} | null { - const normalized = params.commandBody.trim() - const match = normalized.match(/^\/([^\s]+)(?:\s+([\s\S]+))?$/) - if (!match?.[1]) return null - - const command = findInlineNativeCommandFromBody(normalized) - if (!command) return null - - const args = parseCommandArgs(command, match[2]) - const menuNeedsModelContext = - command.argsMenu && - !(args?.raw && !args.values) && - command.args?.some( - (arg) => typeof arg.choices === "function" && args?.values?.[arg.name] == null, - ) - const menuModelContext = menuNeedsModelContext - ? resolveInlineCommandMenuModelContext({ - cfg: params.cfg, - agentId: params.agentId, - sessionKey: params.sessionKey, - }) - : {} - const menu = resolveCommandArgMenu({ - command, - ...(args ? { args } : {}), - cfg: params.cfg, - ...menuModelContext, - }) - if (!menu) return null - - const currentThinkingLevel = - command.key === "think" - ? resolveInlineThinkMenuCurrentLevel({ - cfg: params.cfg, - agentId: params.agentId, - ...menuModelContext, - }) - : undefined - const title = formatInlineCommandArgMenuTitle({ - command, - menu, - ...(currentThinkingLevel ? { currentThinkingLevel } : {}), - }) - const rows: InlineReplyMarkupButton[][] = [] - for (let index = 0; index < menu.choices.length; index += 2) { - const slice = menu.choices.slice(index, index + 2) - rows.push( - slice.map((choice) => ({ - text: choice.label, - callback_data: buildInlineNativeCommandCallbackData( - buildCommandTextFromArgs(command, { - values: { [menu.arg.name]: choice.value }, - }), - ), - })), - ) - } - - return { title, buttons: rows } -} - -function mapInlineModelPickerCallbackToCommand(raw: string): string | undefined { - const callback = parseInlineModelPickerCallback(raw) - if (!callback) return undefined - switch (callback.type) { - case "providers": - case "back": - return "/models" - case "list": - return `/models ${callback.provider} ${String(callback.page)}` - case "select": - return callback.provider ? `/model ${callback.provider}/${callback.model}` : `/model ${callback.model}` - } -} - -function parseInlineModelPickerCallback(raw: string): InlineModelPickerCallback | null { - const trimmed = raw.trim() - if (!trimmed) return null - - if (trimmed === "mdl_prov") return { type: "providers" } - if (trimmed === "mdl_back") return { type: "back" } - - const listMatch = trimmed.match(/^mdl_list_([a-z0-9_.-]+)_(\d+)$/i) - if (listMatch?.[1] && listMatch[2]) { - const provider = listMatch[1].trim() - const page = Number.parseInt(listMatch[2], 10) - if (provider && Number.isFinite(page) && page > 0) { - return { type: "list", provider, page } - } - } - - const standardSelectionMatch = trimmed.match(/^mdl_sel_(.+)$/) - if (standardSelectionMatch?.[1]?.trim()) { - const modelRef = standardSelectionMatch[1].trim() - const slashIndex = modelRef.indexOf("/") - if (slashIndex > 0 && slashIndex < modelRef.length - 1) { - return { - type: "select", - provider: modelRef.slice(0, slashIndex), - model: modelRef.slice(slashIndex + 1), - } - } - } - - const compactSelectionMatch = trimmed.match(/^mdl_sel\/(.+)$/) - if (compactSelectionMatch?.[1]?.trim()) { - return { type: "select", model: compactSelectionMatch[1].trim() } - } - - return null -} - -function resolveInlineModelPickerSelection(params: { - callback: Extract - providers: readonly string[] - byProvider: ReadonlyMap> -}): InlineModelPickerSelection { - if (params.callback.provider) { - return { - kind: "resolved", - provider: params.callback.provider, - model: params.callback.model, - } - } - - const matchingProviders = params.providers.filter((id) => params.byProvider.get(id)?.has(params.callback.model)) - if (matchingProviders.length === 1 && matchingProviders[0]) { - return { - kind: "resolved", - provider: matchingProviders[0], - model: params.callback.model, - } - } - - return { - kind: "ambiguous", - model: params.callback.model, - } -} - -function normalizeInlineActionCallbackData(raw: string): string { - const trimmed = raw.trim() - if (!trimmed) return "" - const nativeCommandText = parseInlineNativeCommandCallbackData(trimmed) - if (nativeCommandText) return nativeCommandText - return mapInlineModelPickerCallbackToCommand(trimmed) ?? trimmed -} - -function normalizeCompatibleButtonCallbackData(raw: string): string { - const trimmed = raw.trim() - if (!trimmed) return "" - if (parseInlineModelPickerCallback(trimmed)) return trimmed - return normalizeInlineActionCallbackData(trimmed) -} - -type InlineParsedReplyActionButton = - | { - text: string - kind: "callback" - callbackData: string - } - | { - text: string - kind: "copyText" - copyText: string - } - -function normalizeReplyMarkupButtonsWith( - raw: unknown, - options?: { mapCallbackData?: (value: string) => string }, -): InlineParsedReplyActionButton[][] { - if (!Array.isArray(raw)) return [] - - const rows: InlineParsedReplyActionButton[][] = [] - for (const candidateRow of raw) { - if (!Array.isArray(candidateRow)) continue - const row: InlineParsedReplyActionButton[] = [] - for (const candidateButton of candidateRow) { - if (!isRecord(candidateButton)) continue - const text = sanitizeInlineActionLabel( - typeof candidateButton.text === "string" ? candidateButton.text : "", - ) - const callbackDataRaw = - typeof candidateButton.callback_data === "string" ? candidateButton.callback_data.trim() : "" - const mappedCallbackData = options?.mapCallbackData - ? options.mapCallbackData(callbackDataRaw) - : callbackDataRaw - const callbackData = sanitizeInlineActionCallbackData(mappedCallbackData) - const copyText = sanitizeInlineActionCopyText( - typeof candidateButton.copy_text === "string" - ? candidateButton.copy_text - : typeof candidateButton.copyText === "string" - ? candidateButton.copyText - : "", - ) - if (!text) continue - if (callbackData) { - row.push({ text, kind: "callback", callbackData }) - } else if (copyText) { - row.push({ text, kind: "copyText", copyText }) - } - if (row.length >= INLINE_ACTION_MAX_PER_ROW) break - } - if (row.length === 0) continue - rows.push(row) - if (rows.length >= INLINE_ACTION_MAX_ROWS) break - } - - return rows -} - -function resolveInlineReplyActions( - payload: Record, - options?: { owner?: InlineMessageActionOwner }, -): MessageActions | undefined { - const owner = options?.owner ?? "agent" - const channelData = isRecord(payload.channelData) ? payload.channelData : undefined - const inlineData = channelData && isRecord(channelData.inline) ? channelData.inline : undefined - const telegramData = channelData && isRecord(channelData.telegram) ? channelData.telegram : undefined - - let rawButtons: unknown = undefined - let hasExplicitButtons = false - let mapCallbackData: ((value: string) => string) | undefined - - if (inlineData && Object.prototype.hasOwnProperty.call(inlineData, "buttons")) { - rawButtons = inlineData.buttons - hasExplicitButtons = true - } else if (telegramData && Object.prototype.hasOwnProperty.call(telegramData, "buttons")) { - // Backwards compatibility for replies emitted before Inline had its own command surface. - rawButtons = telegramData.buttons - hasExplicitButtons = true - mapCallbackData = normalizeCompatibleButtonCallbackData - } else if (Object.prototype.hasOwnProperty.call(payload, "buttons")) { - rawButtons = payload.buttons - hasExplicitButtons = true - } - - if (!hasExplicitButtons) { - return resolveInlineMessageActionsParam(payload, { owner }) - } - - const rows = normalizeReplyMarkupButtonsWith( - rawButtons, - mapCallbackData ? { mapCallbackData } : undefined, - ) - return { - rows: rows.map((row, rowIndex) => ({ - actions: row.map((button, buttonIndex) => ({ - actionId: buildInlineMessageActionId(owner, rowIndex, buttonIndex), - text: button.text, - style: 0, - requiredGrant: 0, - action: - button.kind === "callback" - ? { - oneofKind: "callback" as const, - callback: { - data: new TextEncoder().encode(button.callbackData), - }, - } - : { - oneofKind: "copyText" as const, - copyText: { - text: button.copyText, - }, - }, - })), - })), - } -} - -async function answerInlineMessageAction( - client: InlineSdkClient, - interactionId: bigint, - ui?: MessageActionResponseUi, -): Promise { - const withAnswer = client as InlineSdkClient & { - answerMessageAction?: (params: { interactionId: bigint; ui?: MessageActionResponseUi }) => Promise - invokeUncheckedRaw?: ( - method: Method, - input?: { - oneofKind?: string - answerMessageAction?: { interactionId: bigint; ui?: MessageActionResponseUi } - }, - ) => Promise - } - - if (typeof withAnswer.answerMessageAction === "function") { - await withAnswer.answerMessageAction({ interactionId, ...(ui ? { ui } : {}) }) - return - } - - const answerMethod = (Method as Record)["ANSWER_MESSAGE_ACTION"] - if (typeof answerMethod === "number" && typeof withAnswer.invokeUncheckedRaw === "function") { - await withAnswer.invokeUncheckedRaw(answerMethod as Method, { - oneofKind: "answerMessageAction", - answerMessageAction: { interactionId, ...(ui ? { ui } : {}) }, - }) - } -} - -function resolveCallbackCommandBodyFromActionData(params: { - data: Uint8Array - botUsername?: string -}): string | undefined { - const decoded = callbackDataToUtf8(params.data) - if (!decoded) return undefined - const normalized = normalizeInlineActionCallbackData(decoded) - if (!normalized.startsWith("/")) return undefined - return normalizeInlineCommandBody(normalized, params.botUsername) -} - -function resolveInlineSystemPrompt(params: { - account: ResolvedInlineAccount - groupId?: string - replyThread?: boolean - botHandBack?: boolean -}): string { - const groupPrompt = params.groupId - ? resolveInlineGroupSystemPrompt({ - groups: params.account.config.groups, - groupId: params.groupId, - }) - : undefined - return [ - buildInlineSystemPrompt(params.account.config.systemPrompt), - groupPrompt, - params.replyThread ? INLINE_REPLY_THREAD_SYSTEM_PROMPT : undefined, - params.botHandBack ? INLINE_BOT_HAND_BACK_SYSTEM_PROMPT : undefined, - ] - .filter((entry): entry is string => Boolean(entry)) - .join("\n\n") -} - -function rewriteNumericMentionsToUsernames(text: string, senderProfilesById: SenderProfileLookup): string { - if (!text.includes("@")) return text - return text.replace(/(^|[^\w])@([0-9]+)\b/g, (full, prefix: string, userId: string) => { - const username = senderProfilesById.get(userId)?.username - if (!username) return full - return `${prefix}@${username}` - }) -} - -function rememberBotMessageId(cache: Map, chatId: bigint, messageId: bigint): void { - const key = String(chatId) - const list = cache.get(key) ?? [] - const nextId = String(messageId) - if (!list.includes(nextId)) list.push(nextId) - if (list.length > BOT_MESSAGE_CACHE_LIMIT) { - list.splice(0, list.length - BOT_MESSAGE_CACHE_LIMIT) - } - cache.set(key, list) -} - -function rememberInlineBotDelivery(params: { - botMessageIdsByChat: Map - chatId: bigint - messageId: bigint - accountId: string - agentId: string - inlineAgentId?: string - replyThreadContext?: InlineReplyThreadContext | null -}): void { - rememberBotMessageId(params.botMessageIdsByChat, params.chatId, params.messageId) - if (!params.replyThreadContext || params.chatId !== params.replyThreadContext.childChatId) return - rememberInlineReplyThreadRoute({ - accountId: params.accountId, - parentChatId: params.replyThreadContext.parentChatId, - threadId: params.replyThreadContext.childChatId, - parentMessageId: params.replyThreadContext.parentMessageId, - threadLabel: params.replyThreadContext.threadLabel, - agentId: params.agentId, - ...(params.inlineAgentId ? { inlineAgentId: params.inlineAgentId } : {}), - }) -} - -function hasBotMessageId(cache: Map, chatId: bigint, messageId: bigint): boolean { - const key = String(chatId) - return (cache.get(key) ?? []).includes(String(messageId)) -} - -function rememberBotMessagesFromList(params: { - messages: Message[] - meId: bigint - chatId: bigint - botMessageIdsByChat: Map -}): void { - for (const item of params.messages) { - if (item.fromId === params.meId) { - rememberBotMessageId(params.botMessageIdsByChat, params.chatId, item.id) - } - } -} - -function buildChatPeer(chatId: bigint): { - type: { - oneofKind: "chat" - chat: { chatId: bigint } - } -} { - return { - type: { - oneofKind: "chat", - chat: { chatId }, - }, - } -} - -async function loadChatHistoryMessages(params: { - client: InlineSdkClient - chatId: bigint - limit: number - offsetId?: bigint -}): Promise { - const result = await params.client.invokeRaw(Method.GET_CHAT_HISTORY, { - oneofKind: "getChatHistory", - getChatHistory: { - peerId: buildChatPeer(params.chatId), - ...(params.offsetId != null ? { offsetId: params.offsetId } : {}), - limit: params.limit, - }, - }) - if (result.oneofKind !== "getChatHistory") { - return null - } - return result.getChatHistory.messages ?? [] -} - -async function loadInlineJoinWindowMessages(params: { - client: InlineSdkClient - chatId: bigint - participantDate: bigint -}): Promise { - const cutoff = params.participantDate - INLINE_JOIN_MENTION_LOOKBACK_SECONDS - const messages = new Map() - let offsetId: bigint | undefined - - for (;;) { - const page = await loadChatHistoryMessages({ - client: params.client, - chatId: params.chatId, - limit: 100, - ...(offsetId != null ? { offsetId } : {}), - }) - if (!page) return null - if (page.length === 0) break - - let oldest: Message | undefined - let added = 0 - for (const message of page) { - const key = `${String(message.id)}:${String(message.date ?? 0n)}` - if (!messages.has(key)) { - messages.set(key, message) - added += 1 - } - if (!oldest || message.date < oldest.date || (message.date === oldest.date && message.id < oldest.id)) { - oldest = message - } - } - if (!oldest || oldest.date < cutoff || page.length < 100 || added === 0) break - offsetId = oldest.id - } - - return [...messages.values()] -} - -async function findChatMessageById(params: { - client: InlineSdkClient - chatId: bigint - messageId: bigint - limit: number - meId: bigint - botMessageIdsByChat: Map -}): Promise { - const directResult = - GET_MESSAGES_METHOD == null - ? null - : await params.client - .invokeRaw(GET_MESSAGES_METHOD, { - oneofKind: "getMessages", - getMessages: { - peerId: buildChatPeer(params.chatId), - messageIds: [params.messageId], - }, - }) - .catch(() => null) - if (directResult?.oneofKind === "getMessages") { - const directMessages = directResult.getMessages.messages ?? [] - rememberBotMessagesFromList({ - messages: directMessages, - meId: params.meId, - chatId: params.chatId, - botMessageIdsByChat: params.botMessageIdsByChat, - }) - const directTarget = directMessages.find((item) => item.id === params.messageId) ?? null - if (directTarget) { - return directTarget - } - } - - // Compatibility fallback for older servers without GET_MESSAGES. - const historyMessages = await loadChatHistoryMessages({ - client: params.client, - chatId: params.chatId, - offsetId: params.messageId + 1n, - limit: params.limit, - }) - if (!historyMessages) { - return null - } - - rememberBotMessagesFromList({ - messages: historyMessages, - meId: params.meId, - chatId: params.chatId, - botMessageIdsByChat: params.botMessageIdsByChat, - }) - - return historyMessages.find((item) => item.id === params.messageId) ?? null -} - -async function isReactionTargetBotMessage(params: { - client: InlineSdkClient - chatId: bigint - messageId: bigint - meId: bigint - botMessageIdsByChat: Map -}): Promise { - const target = await findChatMessageById({ - client: params.client, - chatId: params.chatId, - messageId: params.messageId, - limit: REACTION_TARGET_LOOKUP_LIMIT, - meId: params.meId, - botMessageIdsByChat: params.botMessageIdsByChat, - }) - if (!target) { - return hasBotMessageId(params.botMessageIdsByChat, params.chatId, params.messageId) - } - return target.fromId === params.meId -} - -function normalizeHistoryText(raw: string | undefined): string { - const compact = (raw ?? "").replace(/\s+/g, " ").trim() - if (!compact) return "" - if (compact.length <= HISTORY_LINE_MAX_CHARS) return compact - // Keep full URLs for media/file discoverability in history context. - if (URL_LIKE_PATTERN.test(compact)) return compact - return `${compact.slice(0, HISTORY_LINE_MAX_CHARS - 1)}…` -} - -function buildInlineReplyThreadLabel(params: { - title: string | null | undefined - anchorMessage: Message | null | undefined -}): string | null { - const title = params.title?.trim() - if (title) return title - - if (!params.anchorMessage) return "Re: Message" - const anchorText = buildInlineInboundBodyText(summarizeInlineMessageContent(params.anchorMessage)) - .replace(/\s+/g, " ") - .trim() - if (!anchorText) return "Re: Message" - - return `Re: ${anchorText.slice(0, REPLY_THREAD_LABEL_MAX_CHARS)}` -} - -function resolveHistorySenderLabel(params: { - senderId: bigint - meId: bigint - senderProfilesById: SenderProfileLookup -}): string { - if (params.senderId === params.meId) return "assistant" - const senderId = String(params.senderId) - const profile = params.senderProfilesById.get(senderId) - return resolveInlineSenderLabel({ - senderId, - senderName: profile?.name, - senderUsername: profile?.username, - }) -} - -function resolveInlineSenderLabel(params: { - senderId: string - senderName?: string | null | undefined - senderUsername?: string | null | undefined -}): string { - const senderId = params.senderId.trim() - const name = params.senderName?.trim() - const usernameRaw = params.senderUsername?.trim().replace(/^@+/, "") - const username = usernameRaw ? `@${usernameRaw}` : undefined - - let label = name - if (name && username) { - label = `${name} (${username})` - } else if (!name && username) { - label = username - } - - const id = senderId ? `id:${senderId}` : undefined - const fallback = senderId ? `user:${senderId}` : undefined - if (label && id) return `${label} ${id}` - if (label) return label - return fallback ?? "id:unknown" -} - -function resolveInlineDirectSenderName(params: { - profileName: string | null | undefined - chatTitle: string | null | undefined - senderId: string -}): string | undefined { - const profileName = params.profileName?.trim() - if (profileName) return profileName - const chatTitle = params.chatTitle?.trim() - if (!chatTitle || chatTitle === `user:${params.senderId}` || chatTitle === params.senderId) return undefined - return chatTitle -} - -function resolveInlineConversationLabel(params: { - isGroup: boolean - groupTitle?: string | null - groupId: string - senderLabel: string -}): string { - if (!params.isGroup) return params.senderLabel - const title = params.groupTitle?.trim() - return title ? `${title} id:${params.groupId}` : `group:${params.groupId}` -} - -function resolveHistoryLimit(params: { - cfg: OpenClawConfig - isGroup: boolean - historyLimit: number | undefined - dmHistoryLimit: number | undefined -}): number { - if (params.isGroup) { - return Math.max(0, params.historyLimit ?? params.cfg.messages?.groupChat?.historyLimit ?? DEFAULT_GROUP_HISTORY_LIMIT) - } - return Math.max(0, params.dmHistoryLimit ?? params.historyLimit ?? DEFAULT_DM_HISTORY_LIMIT) -} - -function historyEntryDedupeKey(entry: InlinePendingHistoryEntry): string { - if (entry.messageId) return `id:${entry.messageId}:ts:${entry.timestamp ?? "unknown"}` - return `ts:${entry.timestamp ?? "unknown"}:${entry.sender}:${entry.body}` -} - -function mergeInboundHistoryEntries(params: { - historyContextEntries: InlinePendingHistoryEntry[] - pendingEntries: InlinePendingHistoryEntry[] - limit: number -}): Array<{ sender: string; body: string; timestamp?: number }> { - if (params.limit <= 0) return [] - - const deduped: InlinePendingHistoryEntry[] = [] - const seen = new Set() - for (const entry of [...params.historyContextEntries, ...params.pendingEntries]) { - const key = historyEntryDedupeKey(entry) - if (seen.has(key)) continue - seen.add(key) - deduped.push(entry) - } - - return deduped.slice(-params.limit).map((entry) => ({ - sender: entry.sender, - body: entry.body, - ...(entry.timestamp != null ? { timestamp: entry.timestamp } : {}), - })) -} - -function buildInlineHistoryEntryPayload(params: { - message: Message - senderProfilesById: SenderProfileLookup - meId: bigint - syntheticMessageId?: string -}): InlineHistoryEntryPayload { - const content = summarizeInlineMessageContent(params.message) - const text = normalizeHistoryText(content.text) - if (!text) { - return { - line: null, - attachmentLine: null, - entityLine: null, - inboundEntry: null, - } - } - - const label = resolveHistorySenderLabel({ - senderId: params.message.fromId, - meId: params.meId, - senderProfilesById: params.senderProfilesById, - }) - const replySuffix = params.message.replyToMsgId != null ? ` ->${String(params.message.replyToMsgId)}` : "" - const messageId = params.syntheticMessageId ?? String(params.message.id) - const attachmentText = normalizeHistoryText(content.attachmentText) - const entityText = normalizeHistoryText(content.entityText) - - return { - line: `#${String(params.message.id)}${replySuffix} ${label}: ${text}`, - attachmentLine: attachmentText ? `#${String(params.message.id)}${replySuffix} ${label}: ${attachmentText}` : null, - entityLine: entityText ? `#${String(params.message.id)}${replySuffix} ${label}: ${entityText}` : null, - inboundEntry: { - sender: label, - body: text, - ...(params.message.date != null ? { timestamp: Number(params.message.date) * 1000 } : {}), - messageId, - }, - } -} - -function appendInlineHistoryEntry( - target: { - lines: string[] - attachmentLines: string[] - entityLines: string[] - inboundHistory: InlinePendingHistoryEntry[] - }, - entry: InlineHistoryEntryPayload, -): void { - if (!entry.inboundEntry || !entry.line) return - target.inboundHistory.push(entry.inboundEntry) - target.lines.push(entry.line) - if (entry.attachmentLine) { - target.attachmentLines.push(entry.attachmentLine) - } - if (entry.entityLine) { - target.entityLines.push(entry.entityLine) + if (params.seen.has(key)) return false + params.seen.set(key, true) + const maximum = params.maximum ?? 5_000 + while (params.seen.size > maximum) { + const oldest = params.seen.keys().next().value + if (oldest == null) break + params.seen.delete(oldest) } + return true } -function prependLabeledHistoryLine(params: { - existing: string | null - heading: string - line: string | null +function buildInlineDebounceKey(params: { + accountId: string + chatId: bigint + senderId: bigint | null | undefined }): string | null { - if (!params.line) return params.existing - const prefix = `${params.heading}\n` - const existingBody = params.existing?.startsWith(prefix) ? params.existing.slice(prefix.length) : params.existing - return existingBody ? `${prefix}${params.line}\n${existingBody}` : `${prefix}${params.line}` -} - -function stripLabeledHistoryHeading(text: string | null, heading: string): string | null { - if (!text) return null - const prefix = `${heading}\n` - const body = text.startsWith(prefix) ? text.slice(prefix.length) : text - return body.trim() || null + if (params.senderId == null) return null + return `inline:${params.accountId}:${String(params.chatId)}:${String(params.senderId)}` } -function prependLabeledHistoryBlock(params: { - existing: string | null - heading: string - block: string | null -}): string | null { - const block = stripLabeledHistoryHeading(params.block, params.heading) - if (!block) return params.existing - const prefix = `${params.heading}\n` - const existingBody = stripLabeledHistoryHeading(params.existing, params.heading) - return existingBody ? `${prefix}${block}\n${existingBody}` : `${prefix}${block}` +function buildInlineInboundTaskKey(params: { + accountId: string + chatId: bigint +}): string { + return `inline:${params.accountId}:${String(params.chatId)}` } -function prependInlineReplyThreadAnchor(params: { - historyContext: HistoryContext - anchorMessage: Message - parentChatId: bigint - senderProfilesById: SenderProfileLookup - meId: bigint -}): HistoryContext { - const entry = buildInlineHistoryEntryPayload({ - message: params.anchorMessage, - senderProfilesById: params.senderProfilesById, - meId: params.meId, - syntheticMessageId: `anchor:${String(params.parentChatId)}:${String(params.anchorMessage.id)}`, - }) - if (!entry.inboundEntry || !entry.line) { - return { - ...params.historyContext, - } - } - +function buildSyntheticInlineTextMessage(params: { + base: Message + text: string + mentioned?: boolean +}): Message { return { - ...params.historyContext, - inboundHistory: [entry.inboundEntry, ...params.historyContext.inboundHistory], - historyText: prependLabeledHistoryLine({ - existing: params.historyContext.historyText, - heading: "Recent thread messages (oldest -> newest):", - line: entry.line, - }), - attachmentText: prependLabeledHistoryLine({ - existing: params.historyContext.attachmentText, - heading: "Recent media/attachments:", - line: entry.attachmentLine, - }), - entityText: prependLabeledHistoryLine({ - existing: params.historyContext.entityText, - heading: "Recent message entities:", - line: entry.entityLine, - }), + ...params.base, + message: params.text, + ...(params.mentioned !== undefined ? { mentioned: params.mentioned } : {}), } } -function prependInlineParentHistoryContext(params: { - historyContext: HistoryContext - parentHistoryContext: HistoryContext -}): HistoryContext { +function buildInlineTranscriptTextMessage(params: { + base: Message + text: string +}): Message { + const { media: _media, ...base } = params.base as Message & { media?: unknown } return { - ...params.historyContext, - inboundHistory: [ - ...params.parentHistoryContext.inboundHistory, - ...params.historyContext.inboundHistory, - ], - historyText: prependLabeledHistoryBlock({ - existing: params.historyContext.historyText, - heading: "Recent thread messages (oldest -> newest):", - block: params.parentHistoryContext.historyText, - }), - attachmentText: prependLabeledHistoryBlock({ - existing: params.historyContext.attachmentText, - heading: "Recent media/attachments:", - block: params.parentHistoryContext.attachmentText, - }), - entityText: prependLabeledHistoryBlock({ - existing: params.historyContext.entityText, - heading: "Recent message entities:", - block: params.parentHistoryContext.entityText, - }), - } -} - -function buildInlineChannelStructuredContext(params: { - currentAttachmentText: string | null - currentEntityText: string | null - currentBody: string - historyAttachmentText: string | null - historyEntityText: string | null - actionContext?: InlineChannelStructuredContextEntry -}): InlineChannelStructuredContextEntry[] { - const entries: InlineChannelStructuredContextEntry[] = [] - const append = (label: string, type: string, summary: string | null) => { - if (!summary) return - const normalized = normalizeHistoryText(summary) - if (!normalized || normalized === params.currentBody) return - entries.push({ - label, - source: CHANNEL_ID, - type, - payload: { summary: normalized }, - }) - } - - append("Current Inline media/attachments", "current_media_attachments", params.currentAttachmentText) - append("Current Inline message entities", "current_message_entities", params.currentEntityText) - append("Recent Inline media/attachments", "recent_media_attachments", params.historyAttachmentText) - append("Recent Inline message entities", "recent_message_entities", params.historyEntityText) - if (params.actionContext) entries.push(params.actionContext) - return entries -} - -function resolveInlineMediaMaxBytes(params: { - cfg: OpenClawConfig - account: ResolvedInlineAccount -}): number { - return ( - resolveChannelMediaMaxBytes({ - cfg: params.cfg, - accountId: params.account.accountId, - resolveChannelLimitMb: ({ accountId }) => { - if (accountId != null && accountId !== params.account.accountId) return undefined - return params.account.config.mediaMaxMb - }, - }) ?? DEFAULT_INLINE_MEDIA_MAX_BYTES - ) + ...base, + message: params.text, + } as Message } -function buildInlineInboundMediaFacts(media: InlineInboundMediaInfo[]) { - return toInboundMediaFacts(media.map((item) => ({ - ...(item.path ? { path: item.path } : {}), - ...(item.contentType?.trim() ? { contentType: item.contentType.trim() } : {}), - ...(item.fileName?.trim() ? { fileName: item.fileName.trim() } : {}), - ...(item.kind ? { kind: item.kind } : {}), - ...(item.messageId ? { messageId: item.messageId } : {}), - }))) +function buildInlineVoicePendingKey(params: { chatId: bigint; messageId: bigint }): string { + return `${String(params.chatId)}:${String(params.messageId)}` } -function buildInlineInboundBodyText(content: ReturnType): string { - if (content.rawText) return content.rawText - if (content.media?.kind === "nudge" || (!content.media && content.attachmentText)) { - return content.text +function resolveInlineVoiceTranscriptWaitMs(config: { voiceTranscriptWaitMs?: unknown }): number { + const raw = config.voiceTranscriptWaitMs + if (typeof raw !== "number" || !Number.isFinite(raw)) { + return DEFAULT_INLINE_VOICE_TRANSCRIPT_WAIT_MS } - return "" + return Math.max(0, Math.min(MAX_INLINE_VOICE_TRANSCRIPT_WAIT_MS, Math.trunc(raw))) } -function resolveFilePathHint(params: { sourceUrl: string; preferredName?: string | null | undefined }): string | undefined { - const preferred = params.preferredName?.trim() - if (preferred) return preferred - - try { - const pathname = new URL(params.sourceUrl).pathname - const base = path.basename(pathname).trim() - if (base) return base - } catch { - // ignore malformed urls and let the media pipeline choose a filename +function resolveInlineInboundDebounceMsOverride(config: { debounceMs?: unknown }): number | undefined { + const raw = config.debounceMs + if (typeof raw !== "number" || !Number.isFinite(raw)) { + return undefined } - - return undefined + return Math.max(0, Math.trunc(raw)) } -async function resolveInlineInboundMedia(params: { - core: ReturnType - message: Message - maxBytes: number - log?: { warn?: (msg: string) => void; debug?: (msg: string) => void } | undefined -}): Promise { - const content = summarizeInlineMessageContent(params.message) - const candidates = new Map< - string, - { - fileName?: string | null - mimeType?: string | null - kind: "image" | "video" | "audio" | "document" - } - >() - - const out: InlineInboundMediaInfo[] = [] - const messageId = String(params.message.id) - const inlineMediaKind = content.media?.kind === "photo" - ? "image" - : content.media?.kind === "video" - ? "video" - : content.media?.kind === "voice" - ? "audio" - : content.media?.kind === "document" - ? "document" - : null - - if (content.media?.url) { - candidates.set(content.media.url, { - fileName: content.media.fileName ?? null, - mimeType: content.media.mimeType ?? null, - kind: inlineMediaKind ?? "document", - }) - } else if (inlineMediaKind) { - out.push({ - kind: inlineMediaKind, - ...(content.media?.fileName ? { fileName: content.media.fileName } : {}), - ...(content.media?.mimeType ? { contentType: content.media.mimeType } : {}), - messageId, - }) - } - - for (const attachment of content.attachments) { - if (attachment.kind !== "urlPreview" || !attachment.previewImageUrl) continue - candidates.set(attachment.previewImageUrl, { - mimeType: null, - kind: "image", - }) - } - - for (const [url, candidate] of candidates.entries()) { - try { - const filePathHint = resolveFilePathHint({ sourceUrl: url, preferredName: candidate.fileName }) - const fetched = await params.core.channel.media.fetchRemoteMedia({ - url, - maxBytes: params.maxBytes, - ...(filePathHint ? { filePathHint } : {}), - }) - const fetchedContentType = fetched.contentType ?? candidate.mimeType ?? undefined - const saved = await params.core.channel.media.saveMediaBuffer( - fetched.buffer, - fetchedContentType, - "inbound", - params.maxBytes, - fetched.fileName ?? candidate.fileName ?? undefined, - ) - const contentType = saved.contentType ?? fetchedContentType - const fileName = fetched.fileName ?? candidate.fileName - out.push({ - path: saved.path, - ...(contentType ? { contentType } : {}), - ...(fileName ? { fileName } : {}), - // Documents may contain image/audio bytes; keep the host's MIME inference. - ...(candidate.kind !== "document" ? { kind: candidate.kind } : {}), - messageId, - }) - } catch (err) { - params.log?.warn?.(`inline: failed to download inbound media ${url}: ${String(err)}`) - out.push({ - kind: candidate.kind, - ...(candidate.fileName ? { fileName: candidate.fileName } : {}), - ...(candidate.mimeType ? { contentType: candidate.mimeType } : {}), - messageId, - }) - } - } - - return out +function shouldWaitForInlineVoiceTranscript(message: Message): boolean { + const content = summarizeInlineMessageContent(message) + return content.media?.kind === "voice" && !content.rawText } -async function resolveInlineInboundReplyThreadContext(params: { - replyThreadsEnabled: boolean - client: InlineSdkClient - chatId: bigint - chatInfo: CachedChatInfo - chatCache: Map - useCachedChatMetadata?: boolean -}): Promise { - if (params.chatInfo.kind === "direct" || !params.replyThreadsEnabled) { - return null - } - - const metadata = params.useCachedChatMetadata - ? params.chatInfo.parentChatId != null - ? { - childChatId: params.chatId, - parentChatId: params.chatInfo.parentChatId, - parentMessageId: params.chatInfo.parentMessageId ?? null, - title: params.chatInfo.title ?? null, - } - : null - : await loadInlineReplyThreadMetadata({ - client: params.client, - chatId: params.chatId, - }) - if (!metadata) { - return null - } +function extractInlineVoiceTranscriptText(message: Message): string | null { + const text = message.message?.trim() + return text ? text : null +} - const parentChatInfo = - metadata.parentChatId === params.chatId - ? params.chatInfo - : await resolveChatInfo(params.client, params.chatCache, metadata.parentChatId) - const anchorMessage = - metadata.parentMessageId != null - ? await loadInlineReplyThreadAnchorMessage({ - client: params.client, - parentChatId: metadata.parentChatId, - parentMessageId: metadata.parentMessageId, - }).catch(() => null) - : null +type InlineStreamingMode = "off" | "partial" | "block" | "progress" - return { - childChatId: metadata.childChatId, - parentChatId: metadata.parentChatId, - parentChatKind: parentChatInfo.kind, - ...(parentChatInfo.peerUserId != null ? { parentPeerUserId: parentChatInfo.peerUserId } : {}), - parentChatTitle: parentChatInfo.title ?? null, - parentMessageId: metadata.parentMessageId, - threadLabel: buildInlineReplyThreadLabel({ - title: metadata.title ?? params.chatInfo.title ?? null, - anchorMessage, - }), - anchorMessage, +function normalizeInlineStreamingMode(value: unknown): InlineStreamingMode | null { + if (typeof value !== "string") return null + const normalized = normalizeLowercaseStringOrEmpty(value) + switch (normalized) { + case "off": + case "partial": + case "block": + case "progress": + return normalized + default: + return null } } -function parseCachedInlineId(raw: string | undefined): bigint | null { - if (!raw || !/^\d+$/.test(raw)) return null - try { - return BigInt(raw) - } catch { - return null - } +function resolveExplicitInlineStreamingMode(config: { + streaming?: unknown + streamMode?: unknown +}): InlineStreamingMode | null { + const streaming = isRecord(config.streaming) ? config.streaming : null + return ( + normalizeInlineStreamingMode(streaming?.mode) ?? + normalizeInlineStreamingMode(config.streaming) ?? + normalizeInlineStreamingMode(config.streamMode) + ) } -async function resolveInlineReplyThreadContextFromRoute(params: { - route: InlineReplyThreadRouteRecord - client: InlineSdkClient - chatCache: Map - fallbackParentChatInfo: CachedChatInfo -}): Promise { - const parentChatId = parseCachedInlineId(params.route.parentChatId) - const childChatId = parseCachedInlineId(params.route.threadId) - if (parentChatId == null || childChatId == null) { - return null - } - const parentMessageId = parseCachedInlineId(params.route.parentMessageId) - const parentChatInfo = await resolveChatInfo(params.client, params.chatCache, parentChatId) - .catch(() => params.fallbackParentChatInfo) - const anchorMessage = - parentMessageId != null - ? await loadInlineReplyThreadAnchorMessage({ - client: params.client, - parentChatId, - parentMessageId, - }).catch(() => null) - : null - const labelSource = params.route.threadLabel ?? params.route.title ?? null - - return { - childChatId, - parentChatId, - parentChatKind: parentChatInfo.kind, - ...(parentChatInfo.peerUserId != null ? { parentPeerUserId: parentChatInfo.peerUserId } : {}), - parentChatTitle: parentChatInfo.title ?? params.fallbackParentChatInfo.title, - parentMessageId, - threadLabel: buildInlineReplyThreadLabel({ - title: labelSource, - anchorMessage, - }), - anchorMessage, +function resolveInlineStreamingMode(config: { + streaming?: unknown + streamMode?: unknown + streamViaEditMessage?: boolean | undefined +}): InlineStreamingMode { + const mode = resolveExplicitInlineStreamingMode(config) + if (mode) return mode + if (typeof config.streaming === "boolean") { + return config.streaming ? "partial" : "off" } + return config.streamViaEditMessage === true ? "partial" : "off" } -async function buildReplyThreadParentHistoryContext(params: { - client: InlineSdkClient - replyThreadContext: InlineReplyThreadContext - parentHistoryLimit: number - senderProfilesById: SenderProfileLookup - meId: bigint - botMessageIdsByChat: Map -}): Promise { - if (params.parentHistoryLimit <= 0 || params.replyThreadContext.parentMessageId == null) { - return null - } +function resolveInlineProgressPlaceholderEnabled(config: { + streaming?: unknown + streamMode?: unknown + streamViaEditMessage?: boolean | undefined +}): boolean { + const mode = resolveExplicitInlineStreamingMode(config) + if (mode === "off") return false + if (mode === "progress") return true + if (config.streaming === false) return false + if (isRecord(config.streaming) && isRecord(config.streaming.progress)) return true - return buildHistoryContext({ - client: params.client, - chatId: params.replyThreadContext.parentChatId, - currentMessageId: params.replyThreadContext.parentMessageId, - replyToMsgId: params.replyThreadContext.anchorMessage?.replyToMsgId, - senderProfilesById: params.senderProfilesById, - meId: params.meId, - historyLimit: params.parentHistoryLimit, - botMessageIdsByChat: params.botMessageIdsByChat, - }) + return ( + config.streaming === undefined && + config.streamMode === undefined && + config.streamViaEditMessage !== true + ) } -async function buildHistoryContext(params: { - client: InlineSdkClient - chatId: bigint - currentMessageId: bigint - replyToMsgId: bigint | undefined - senderProfilesById: SenderProfileLookup - meId: bigint - historyLimit: number - botMessageIdsByChat: Map -}): Promise { - const cachedReplyToBot = - params.replyToMsgId != null && - hasBotMessageId(params.botMessageIdsByChat, params.chatId, params.replyToMsgId) - let repliedToBot = cachedReplyToBot - let replyToSenderId: string | null = null - let foundReplyTargetInHistory = false - const lines: string[] = [] - const attachmentLines: string[] = [] - const entityLines: string[] = [] - const inboundHistory: InlinePendingHistoryEntry[] = [] - - if (params.historyLimit > 0) { - const messages = await loadChatHistoryMessages({ - client: params.client, - chatId: params.chatId, - offsetId: params.currentMessageId, - limit: params.historyLimit, - }) - - if (messages) { - for (const item of messages) { - if (item.fromId === params.meId) { - rememberBotMessageId(params.botMessageIdsByChat, params.chatId, item.id) - } - } - - const sortedMessages = messages - .filter((item) => item.id !== params.currentMessageId) - .sort((a, b) => { - const byDate = Number(a.date - b.date) - if (byDate !== 0) return byDate - if (a.id === b.id) return 0 - return a.id < b.id ? -1 : 1 - }) - - for (const item of sortedMessages) { - if (params.replyToMsgId != null && item.id === params.replyToMsgId) { - foundReplyTargetInHistory = true - replyToSenderId = String(item.fromId) - repliedToBot = item.fromId === params.meId - } - appendInlineHistoryEntry( - { - lines, - attachmentLines, - entityLines, - inboundHistory, - }, - buildInlineHistoryEntryPayload({ - message: item, - senderProfilesById: params.senderProfilesById, - meId: params.meId, - }), - ) - } - } - } +function resolveInlineBlockStreamingEnabled(config: { + streaming?: unknown + streamMode?: unknown + blockStreaming?: boolean | undefined + streamViaEditMessage?: boolean | undefined +}): boolean | undefined { + const mode = resolveInlineStreamingMode(config) + const streaming = isRecord(config.streaming) ? config.streaming : null + const block = streaming && isRecord(streaming.block) ? streaming.block : null + if (typeof block?.enabled === "boolean") return block.enabled + if (typeof config.blockStreaming === "boolean") return config.blockStreaming + if (mode === "block") return true + if (mode === "off") return false + return undefined +} - if (params.replyToMsgId != null && !foundReplyTargetInHistory) { - const replyTarget = await findChatMessageById({ - client: params.client, - chatId: params.chatId, - messageId: params.replyToMsgId, - limit: REPLY_TARGET_LOOKUP_LIMIT, - meId: params.meId, - botMessageIdsByChat: params.botMessageIdsByChat, - }) - if (replyTarget) { - replyToSenderId = String(replyTarget.fromId) - repliedToBot = replyTarget.fromId === params.meId - } else if (!cachedReplyToBot) { - repliedToBot = false - } - } +function shouldStartInlineProgressPlaceholderNow( + line: string | ChannelProgressDraftLine | undefined, +): boolean { + return typeof line === "object" && line?.kind === "patch" && Boolean(line.detail) +} - if (!lines.length) { - return { - historyText: null, - attachmentText: attachmentLines.length ? attachmentLines.slice(-ATTACHMENT_CONTEXT_LIMIT).join("\n") : null, - entityText: entityLines.length ? entityLines.slice(-ATTACHMENT_CONTEXT_LIMIT).join("\n") : null, - inboundHistory, - repliedToBot, - replyToSenderId, - } - } - return { - historyText: `Recent thread messages (oldest -> newest):\n${lines.join("\n")}`, - attachmentText: attachmentLines.length - ? `Recent media/attachments:\n${attachmentLines.slice(-ATTACHMENT_CONTEXT_LIMIT).join("\n")}` - : null, - entityText: entityLines.length - ? `Recent message entities:\n${entityLines.slice(-ATTACHMENT_CONTEXT_LIMIT).join("\n")}` - : null, - inboundHistory, - repliedToBot, - replyToSenderId, - } +function resolveInlineSystemPrompt(params: { + account: ResolvedInlineAccount + groupId?: string + replyThread?: boolean + botHandBack?: boolean +}): string { + const groupPrompt = params.groupId + ? resolveInlineGroupSystemPrompt({ + groups: params.account.config.groups, + groupId: params.groupId, + }) + : undefined + return [ + buildInlineSystemPrompt(params.account.config.systemPrompt), + groupPrompt, + params.replyThread ? INLINE_REPLY_THREAD_SYSTEM_PROMPT : undefined, + params.botHandBack ? INLINE_BOT_HAND_BACK_SYSTEM_PROMPT : undefined, + ] + .filter((entry): entry is string => Boolean(entry)) + .join("\n\n") } export async function monitorInlineProvider(params: { @@ -3452,6 +1070,13 @@ export async function monitorInlineProvider(params: { if (!meUser) { throw new Error("inline startup getMe failed: missing user") } + const unshareClient = shareInlineAccountClient({ + accountId: account.accountId, + baseUrl: account.baseUrl, + token, + client, + }) + const sendMessage = createInlineMessageSender(client, abortSignal) const meId = meUser.id const botUsername = normalizeInlineUsername(meUser.username)?.toLowerCase() void registerBotCapabilitiesWithRetry({ @@ -4244,7 +1869,7 @@ export async function monitorInlineProvider(params: { cfg, botUsername ? { botUsername } : undefined, ) - const isRegisteredNativeCommand = isInlineNativeCommandBody({ + const isRegisteredNativeCommand = await isInlineNativeCommandBody({ cfg, account, commandBody: normalizedCommandBody, @@ -4288,7 +1913,7 @@ export async function monitorInlineProvider(params: { allowTextCommands, hasControlCommand, }) - const commandAuthorized = resolveInlineCommandAuthorized({ + const commandAuthorized = await resolveInlineCommandAuthorized({ cfg, accountId: account.accountId, isGroup, @@ -4355,7 +1980,7 @@ export async function monitorInlineProvider(params: { }) if (created) { try { - await client.sendMessage({ + await sendMessage({ chatId, text: core.channel.pairing.buildPairingReply({ channel: CHANNEL_ID, @@ -4580,7 +2205,7 @@ export async function monitorInlineProvider(params: { } } - const sent = await client.sendMessage({ + const sent = await sendMessage({ chatId, text: responseText, replyToMsgId: msg.id, @@ -4672,7 +2297,7 @@ export async function monitorInlineProvider(params: { } } if (!delivered) { - const sent = await client.sendMessage({ + const sent = await sendMessage({ chatId, text, ...(actions ? { actions } : {}), @@ -4698,7 +2323,7 @@ export async function monitorInlineProvider(params: { const agentId = commandsPageCallback.agentId ?? route.agentId const paginated = buildCommandsMessagePaginated( cfg, - listSkillCommandsForAgents({ + await prepareSkillCommandsForAgents({ cfg, agentIds: [agentId], }), @@ -4735,7 +2360,7 @@ export async function monitorInlineProvider(params: { } } catch (error) { runtime.error?.(`inline commands pagination edit failed; falling back to send (${String(error)})`) - const sent = await client.sendMessage({ + const sent = await sendMessage({ chatId, text, actions, @@ -4747,7 +2372,7 @@ export async function monitorInlineProvider(params: { return } - const nativeCommandMenu = resolveInlineNativeCommandMenu({ + const nativeCommandMenu = await resolveInlineNativeCommandMenu({ commandBody: normalizedCommandBody, cfg, agentId: route.agentId, @@ -4786,7 +2411,7 @@ export async function monitorInlineProvider(params: { } } if (!deliveredNativeMenu) { - const sent = await client.sendMessage({ + const sent = await sendMessage({ chatId, text: menuText, ...(menuActions ? { actions: menuActions } : {}), @@ -4838,7 +2463,7 @@ export async function monitorInlineProvider(params: { } } catch (error) { runtime.error?.(`inline model picker edit failed; falling back to send (${String(error)})`) - const sent = await client.sendMessage({ + const sent = await sendMessage({ chatId, text: outboundText, actions, @@ -5036,7 +2661,7 @@ export async function monitorInlineProvider(params: { if (shouldCreateDeliveryThread && !deliveryReplyThreadContext) { await setParentThreadCreationTyping(false) runtime.error?.("inline create reply thread failed; answer withheld from parent chat") - const sent = await client.sendMessage({ + const sent = await sendMessage({ chatId, text: "I couldn't confirm the reply thread, so I haven't sent the answer. Please check the thread or try again.", replyToMsgId: msg.id, @@ -5477,7 +3102,7 @@ export async function monitorInlineProvider(params: { if (abortSignal.aborted || progressState.closing || text === progressState.text) return if (progressState.messageId == null) { progressState.startedAt ??= performance.now() - const sent = await client.sendMessage({ + const sent = await sendMessage({ chatId: deliveryChatId, text, sendMode: "silent", @@ -5671,7 +3296,7 @@ export async function monitorInlineProvider(params: { if (editStreamState.messageId == null) { let sent: Awaited> try { - sent = await client.sendMessage({ + sent = await sendMessage({ chatId: deliveryChatId, text: snapshot.text, ...(defaultReplyToMsgId != null ? { replyToMsgId: defaultReplyToMsgId } : {}), @@ -5822,7 +3447,7 @@ export async function monitorInlineProvider(params: { actions: MessageActions | undefined }): Promise => { try { - const sent = await client.sendMessage({ + const sent = await sendMessage({ chatId: deliveryChatId, text: `Attachment: ${params.mediaUrl}`, ...(params.isFirstMediaForAnswer && params.replyToMsgId != null @@ -5890,7 +3515,7 @@ export async function monitorInlineProvider(params: { } try { - const sent = await client.sendMessage({ + const sent = await sendMessage({ chatId: deliveryChatId, media, ...(params.isFirstMediaForAnswer && params.replyToMsgId != null @@ -6454,12 +4079,14 @@ export async function monitorInlineProvider(params: { const payloadRecord = visiblePayload as InlineReplyPayload & { interactive?: unknown presentation?: unknown + presentationTextMode?: "fallback" } const rawText = resolveInlineInteractiveTextFallback({ text: visiblePayload.text, interactive: payloadRecord.interactive, presentation: payloadRecord.presentation, + presentationTextMode: payloadRecord.presentationTextMode, }) ?? visiblePayload.text ?? "" @@ -6504,7 +4131,7 @@ export async function monitorInlineProvider(params: { const outbound = sanitizeInlineDeliveryText(text) if (!outbound.trim()) return try { - const sent = await client.sendMessage({ + const sent = await sendMessage({ chatId: deliveryChatId, text: outbound, ...(includeReplyTo && replyToMsgId != null ? { replyToMsgId } : {}), @@ -6722,7 +4349,7 @@ export async function monitorInlineProvider(params: { } try { - const sent = await client.sendMessage({ + const sent = await sendMessage({ chatId: deliveryChatId, ...(caption ? { text: caption } : {}), media, @@ -6846,7 +4473,7 @@ export async function monitorInlineProvider(params: { : dispatchError != null ? INLINE_REQUEST_ERROR_FALLBACK : EMPTY_RESPONSE_FALLBACK - const sent = await client.sendMessage({ + const sent = await sendMessage({ chatId: deliveryChatId, text: fallbackText, ...(defaultReplyToMsgId != null ? { replyToMsgId: defaultReplyToMsgId } : {}), @@ -6956,12 +4583,11 @@ export async function monitorInlineProvider(params: { runtime.error?.(`inline debounce flush failed: ${String(err)}`) const chatId = items[0]?.chatId if (chatId == null) return - void client - .sendMessage({ - chatId, - text: INLINE_DEBOUNCE_ERROR_FALLBACK, - parseMarkdown: account.config.parseMarkdown ?? true, - }) + void sendMessage({ + chatId, + text: INLINE_DEBOUNCE_ERROR_FALLBACK, + parseMarkdown: account.config.parseMarkdown ?? true, + }) .then(() => { publishStatus({ lastOutboundAt: Date.now() }) }) @@ -7092,7 +4718,7 @@ export async function monitorInlineProvider(params: { hasControlCommand: true, }) - return resolveInlineCommandAuthorized({ + return await resolveInlineCommandAuthorized({ cfg, accountId: account.accountId, isGroup: true, @@ -7225,15 +4851,15 @@ export async function monitorInlineProvider(params: { return [...options.values()] } - const resolveOpenClawBotSettingsReasoningOptions = (params: { + const resolveOpenClawBotSettingsReasoningOptions = async (params: { cfg: OpenClawConfig provider: string model: string currentLevel: string - }): OpenClawBotChatSettingsOption[] => { + }): Promise => { const command = findInlineNativeCommandFromBody("/think") const menu = command - ? resolveCommandArgMenu({ + ? await resolveCommandArgMenuAsync({ command, cfg: params.cfg, provider: params.provider, @@ -7280,7 +4906,7 @@ export async function monitorInlineProvider(params: { resolvedContext: ingress, }) const currentConfig = core.config.current() as OpenClawConfig - const modelContext = resolveInlineCommandMenuModelContext({ + const modelContext = await resolveInlineCommandMenuModelContext({ cfg: currentConfig, agentId: ingress.agentId, sessionKey: ingress.sessionKey, @@ -7330,7 +4956,7 @@ export async function monitorInlineProvider(params: { defaultModel: defaultModelValue, modelOptions, reasoningLevel, - reasoningOptions: resolveOpenClawBotSettingsReasoningOptions({ + reasoningOptions: await resolveOpenClawBotSettingsReasoningOptions({ cfg: currentConfig, provider, model, @@ -7435,7 +5061,7 @@ export async function monitorInlineProvider(params: { }) const currentConfig = core.config.current() as OpenClawConfig - const applied = resolveInlineCommandMenuModelContext({ + const applied = await resolveInlineCommandMenuModelContext({ cfg: currentConfig, agentId: params.ingress.agentId, sessionKey: params.ingress.sessionKey, @@ -7460,14 +5086,13 @@ export async function monitorInlineProvider(params: { ): Promise => { const currentConfig = core.config.current() as OpenClawConfig const storePath = resolveStorePath(currentConfig.session?.store, { agentId: ingress.agentId }) - await patchSessionEntry({ + await applySessionEntryPatch({ agentId: ingress.agentId, sessionKey: ingress.sessionKey, storePath, fallbackEntry: { sessionId: ingress.sessionKey, updatedAt: Date.now() }, preserveActivity: true, - replaceEntry: true, - update: (entry) => ({ ...entry, thinkingLevel: reasoningLevel }), + patch: { thinkingLevel: reasoningLevel }, }) } @@ -7886,6 +5511,7 @@ export async function monitorInlineProvider(params: { try { await client.consumeEvents(async (event) => { if (abortSignal.aborted) { + unshareClient() await client.close() return } @@ -7931,6 +5557,7 @@ export async function monitorInlineProvider(params: { chatId: event.chatId, } as Message if (msg.out || msg.fromId === meId) return + if (isInlineServiceMessage(msg)) return if (!claimInlineInboundMessageInstance({ seen: seenInboundMessageInstances, chatId: event.chatId, @@ -7968,6 +5595,7 @@ export async function monitorInlineProvider(params: { chatId: event.chatId, } as Message if (msg.out || msg.fromId === meId) return + if (isInlineServiceMessage(msg)) return if (await handlePendingInlineVoiceEdit({ chatId: event.chatId, msg })) { return } @@ -8103,6 +5731,7 @@ export async function monitorInlineProvider(params: { } catch (err) { publishStatus({ connected: false, lastError: String(err) }) runtime.error?.(`inline monitor loop crashed: ${String(err)}`) + unshareClient() await client.close().catch(() => {}) throw err } @@ -8123,6 +5752,7 @@ export async function monitorInlineProvider(params: { shutdown.abort() clearInterval(diagnosticsTimer) clearPendingInlineVoiceMessages() + unshareClient() await client.close().catch(() => {}) await loop.catch(() => {}) await drainInboundTasks() diff --git a/plugins/openclaw/src/inline/parent-context-tool.ts b/plugins/openclaw/src/inline/parent-context-tool.ts index d01c54d23..e22c0528c 100644 --- a/plugins/openclaw/src/inline/parent-context-tool.ts +++ b/plugins/openclaw/src/inline/parent-context-tool.ts @@ -1,6 +1,7 @@ import type { AnyAgentTool, OpenClawConfig } from "openclaw/plugin-sdk/core" import { InlineSdkClient, Method, type Message } from "@inline-chat/realtime-sdk" import { resolveInlineAccount, resolveInlineToken } from "./accounts.js" +import { withInlineAccountClient } from "./client-registry.js" import { summarizeInlineMessageContent } from "./message-content.js" import { loadInlineReplyThreadMetadata } from "./reply-threads.js" import { @@ -258,16 +259,10 @@ async function withInlineClient(params: { throw new Error(`Inline not configured for account "${account.accountId}" (missing token or baseUrl)`) } const token = await resolveInlineToken(account) - const client = new InlineSdkClient({ - baseUrl: account.baseUrl, - token, - }) - await client.connect() - try { - return await params.fn(client, account.accountId) - } finally { - await client.close().catch(() => {}) - } + return await withInlineAccountClient( + { accountId: account.accountId, baseUrl: account.baseUrl, token }, + (client) => params.fn(client, account.accountId), + ) } async function loadParentHistory(params: { diff --git a/plugins/openclaw/src/inline/profile-tool.ts b/plugins/openclaw/src/inline/profile-tool.ts index e525d48a3..367775b8c 100644 --- a/plugins/openclaw/src/inline/profile-tool.ts +++ b/plugins/openclaw/src/inline/profile-tool.ts @@ -1,6 +1,7 @@ import type { AnyAgentTool, OpenClawConfig } from "openclaw/plugin-sdk/core" import { InlineSdkClient, Method } from "@inline-chat/realtime-sdk" import { resolveInlineAccount, resolveInlineToken } from "./accounts.js" +import { withInlineAccountClient } from "./client-registry.js" import { sanitizeInlineVisibleText } from "./outbound-sanitize.js" import { getInlineRuntime } from "../runtime.js" @@ -68,16 +69,10 @@ async function withInlineClient(params: { throw new Error(`Inline not configured for account "${account.accountId}" (missing token or baseUrl)`) } const token = await resolveInlineToken(account) - const client = new InlineSdkClient({ - baseUrl: account.baseUrl, - token, - }) - await client.connect() - try { - return await params.fn(client, account.accountId) - } finally { - await client.close().catch(() => {}) - } + return await withInlineAccountClient( + { accountId: account.accountId, baseUrl: account.baseUrl, token }, + (client) => params.fn(client, account.accountId), + ) } function readTrimmedString(value: unknown): string | undefined { diff --git a/plugins/openclaw/src/manifest.test.ts b/plugins/openclaw/src/manifest.test.ts index 2d976758b..14ee05e52 100644 --- a/plugins/openclaw/src/manifest.test.ts +++ b/plugins/openclaw/src/manifest.test.ts @@ -337,10 +337,10 @@ describe("plugin manifest", () => { expect(json.openclaw?.install?.clawhubSpec).toBe("clawhub:@inline-openclaw/inline") expect(json.openclaw?.install?.defaultChoice).toBe("clawhub") - expect(json.openclaw?.install?.minHostVersion).toBe(">=2026.8.2") - expect(json.peerDependencies?.openclaw).toBe(">=2026.8.2") + expect(json.openclaw?.install?.minHostVersion).toBe(">=2026.11.1") + expect(json.peerDependencies?.openclaw).toBe(">=2026.11.1") expect(json.peerDependenciesMeta?.openclaw?.optional).toBe(true) - expect(json.openclaw?.compat?.pluginApi).toBe(">=2026.8.2") + expect(json.openclaw?.compat?.pluginApi).toBe(">=2026.11.1") expect(json.devDependencies?.openclaw).toBe("2026.11.1") expect(json.openclaw?.build?.openclawVersion).toBe(json.devDependencies?.openclaw) expect(json.openclaw?.release?.publishToClawHub).toBe(true) diff --git a/plugins/openclaw/src/openclaw-compat.ts b/plugins/openclaw/src/openclaw-compat.ts index b9c237b33..ae6be08c3 100644 --- a/plugins/openclaw/src/openclaw-compat.ts +++ b/plugins/openclaw/src/openclaw-compat.ts @@ -1,4 +1,4 @@ -import { z, type ZodTypeAny } from "zod" +import { z, type ZodType } from "zod" import type { OpenClawConfig } from "openclaw/plugin-sdk/core" const MB = 1024 * 1024 @@ -38,7 +38,7 @@ export function setOpenClawAgentDefaultModel( } } -type ZodSchemaWithToJsonSchema = ZodTypeAny & { +type ZodSchemaWithToJsonSchema = ZodType & { toJSONSchema?: (params?: Record) => unknown } @@ -88,7 +88,7 @@ export function emptyPluginConfigSchema() { } } -export function buildChannelConfigSchema(schema: ZodTypeAny) { +export function buildChannelConfigSchema(schema: ZodType) { const schemaWithJson = schema as ZodSchemaWithToJsonSchema if (typeof schemaWithJson.toJSONSchema === "function") { return { @@ -132,7 +132,7 @@ const ToolPolicyBaseSchema = z export const ToolPolicySchema = ToolPolicyBaseSchema.superRefine((value, ctx) => { if (value.allow && value.allow.length > 0 && value.alsoAllow && value.alsoAllow.length > 0) { ctx.addIssue({ - code: z.ZodIssueCode.custom, + code: "custom", message: "tools policy cannot set both allow and alsoAllow in the same scope (merge alsoAllow into allow, or remove allow and use profile + alsoAllow)", }) @@ -154,7 +154,7 @@ export function requireOpenAllowFrom(params: { return } params.ctx.addIssue({ - code: z.ZodIssueCode.custom, + code: "custom", path: params.path, message: params.message, })