From 1f956492978c72de969f2dd448a12fc1809f177c Mon Sep 17 00:00:00 2001 From: sawka Date: Wed, 15 Oct 2025 13:02:45 -0700 Subject: [PATCH 01/11] rpc call to add AI context --- frontend/app/aipanel/waveai-model.tsx | 12 ++++++++++++ frontend/app/store/tabrpcclient.ts | 21 +++++++++++++++++++++ frontend/app/store/wshclientapi.ts | 5 +++++ frontend/types/gotypes.d.ts | 15 +++++++++++++++ pkg/wshrpc/wshclient/wshclient.go | 6 ++++++ pkg/wshrpc/wshrpctypes.go | 15 +++++++++++++++ 6 files changed, 74 insertions(+) diff --git a/frontend/app/aipanel/waveai-model.tsx b/frontend/app/aipanel/waveai-model.tsx index 8b17c24103..a8e8a2d421 100644 --- a/frontend/app/aipanel/waveai-model.tsx +++ b/frontend/app/aipanel/waveai-model.tsx @@ -173,6 +173,18 @@ export class WaveAIModel { return input != null && input.trim().length > 0; } + appendText(text: string) { + const currentInput = globalStore.get(this.inputAtom); + let newInput = currentInput; + + if (newInput.length > 0 && !newInput.endsWith(" ") && !newInput.endsWith("\n")) { + newInput += " "; + } + + newInput += text; + globalStore.set(this.inputAtom, newInput); + } + setModel(model: string) { const tabId = globalStore.get(atoms.staticTabId); RpcApi.SetMetaCommand(TabRpcClient, { diff --git a/frontend/app/store/tabrpcclient.ts b/frontend/app/store/tabrpcclient.ts index b74d7e8120..c8e3c0121a 100644 --- a/frontend/app/store/tabrpcclient.ts +++ b/frontend/app/store/tabrpcclient.ts @@ -1,6 +1,7 @@ // Copyright 2025, Command Line Inc. // SPDX-License-Identifier: Apache-2.0 +import { WaveAIModel } from "@/app/aipanel/waveai-model"; import { getApi } from "@/app/store/global"; import { getLayoutModelForStaticTab } from "@/layout/index"; import { RpcResponseHelper, WshClient } from "./wshclient"; @@ -56,4 +57,24 @@ export class TabClient extends WshClient { return await getApi().captureScreenshot(electronRect); } + + async handle_waveaiaddcontext(rh: RpcResponseHelper, data: CommandWaveAIAddContextData): Promise { + const model = WaveAIModel.getInstance(); + + if (data.files && data.files.length > 0) { + for (const fileData of data.files) { + const blob = new Blob([fileData.data], { type: fileData.type }); + const file = new File([blob], fileData.name, { type: fileData.type }); + await model.addFile(file); + } + } + + if (data.text) { + model.appendText(data.text); + } + + if (data.submit) { + model.focusInput(); + } + } } diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index dc81b159ac..e2220ea827 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -487,6 +487,11 @@ class RpcApiType { return client.wshRpcCall("waitforroute", data, opts); } + // command "waveaiaddcontext" [call] + WaveAIAddContextCommand(client: WshClient, data: CommandWaveAIAddContextData, opts?: RpcOpts): Promise { + return client.wshRpcCall("waveaiaddcontext", data, opts); + } + // command "waveaienabletelemetry" [call] WaveAIEnableTelemetryCommand(client: WshClient, opts?: RpcOpts): Promise { return client.wshRpcCall("waveaienabletelemetry", null, opts); diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index fd0cfd0a4e..88da69bbfb 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -5,6 +5,14 @@ declare global { + // wshrpc.AIAttachedFile + type AIAttachedFile = { + name: string; + type: string; + size: number; + data: string; + }; + // wshrpc.ActivityDisplayType type ActivityDisplayType = { width: number; @@ -320,6 +328,13 @@ declare global { waitms: number; }; + // wshrpc.CommandWaveAIAddContextData + type CommandWaveAIAddContextData = { + files?: AIAttachedFile[]; + text?: string; + submit?: boolean; + }; + // wshrpc.CommandWaveAIToolApproveData type CommandWaveAIToolApproveData = { toolcallid: string; diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index 02d0c7242b..47af277974 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -581,6 +581,12 @@ func WaitForRouteCommand(w *wshutil.WshRpc, data wshrpc.CommandWaitForRouteData, return resp, err } +// command "waveaiaddcontext", wshserver.WaveAIAddContextCommand +func WaveAIAddContextCommand(w *wshutil.WshRpc, data wshrpc.CommandWaveAIAddContextData, opts *wshrpc.RpcOpts) error { + _, err := sendRpcRequestCallHelper[any](w, "waveaiaddcontext", data, opts) + return err +} + // command "waveaienabletelemetry", wshserver.WaveAIEnableTelemetryCommand func WaveAIEnableTelemetryCommand(w *wshutil.WshRpc, opts *wshrpc.RpcOpts) error { _, err := sendRpcRequestCallHelper[any](w, "waveaienabletelemetry", nil, opts) diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 754b0fa4cf..1c63f83802 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -143,6 +143,7 @@ const ( Command_GetWaveAIChat = "getwaveaichat" Command_GetWaveAIRateLimit = "getwaveairatelimit" Command_WaveAIToolApprove = "waveaitoolapprove" + Command_WaveAIAddContext = "waveaiaddcontext" Command_CaptureBlockScreenshot = "captureblockscreenshot" @@ -273,6 +274,7 @@ type WshRpcInterface interface { GetWaveAIChatCommand(ctx context.Context, data CommandGetWaveAIChatData) (*uctypes.UIChat, error) GetWaveAIRateLimitCommand(ctx context.Context) (*uctypes.RateLimitInfo, error) WaveAIToolApproveCommand(ctx context.Context, data CommandWaveAIToolApproveData) error + WaveAIAddContextCommand(ctx context.Context, data CommandWaveAIAddContextData) error // screenshot CaptureBlockScreenshotCommand(ctx context.Context, data CommandCaptureBlockScreenshotData) (string, error) @@ -734,6 +736,19 @@ type CommandWaveAIToolApproveData struct { Approval string `json:"approval,omitempty"` } +type AIAttachedFile struct { + Name string `json:"name"` + Type string `json:"type"` + Size int `json:"size"` + Data []byte `json:"data"` +} + +type CommandWaveAIAddContextData struct { + Files []AIAttachedFile `json:"files,omitempty"` + Text string `json:"text,omitempty"` + Submit bool `json:"submit,omitempty"` +} + type CommandCaptureBlockScreenshotData struct { BlockId string `json:"blockid" wshcontext:"BlockId"` } From 895f6512160568b5368112a6ba99caf8d2c5b106 Mon Sep 17 00:00:00 2001 From: sawka Date: Wed, 15 Oct 2025 13:45:59 -0700 Subject: [PATCH 02/11] increase message limit to 10m --- pkg/web/ws.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/web/ws.go b/pkg/web/ws.go index f33eee9f84..fcf110556d 100644 --- a/pkg/web/ws.go +++ b/pkg/web/ws.go @@ -27,7 +27,7 @@ const wsReadWaitTimeout = 15 * time.Second const wsWriteWaitTimeout = 10 * time.Second const wsPingPeriodTickTime = 10 * time.Second const wsInitialPingTime = 1 * time.Second -const wsMaxMessageSize = 8 * 1024 * 1024 +const wsMaxMessageSize = 10 * 1024 * 1024 const DefaultCommandTimeout = 2 * time.Second From 7647b042378dc602ff7bd5e045556adf943bdfb1 Mon Sep 17 00:00:00 2001 From: sawka Date: Wed, 15 Oct 2025 16:06:29 -0700 Subject: [PATCH 03/11] re-implement `wsh ai` to work with Wave AI. fix the text_file encoding so AI recognizes it as a file and doesn't try to re-read it. implement RPC calls to interact with Wave AI. --- cmd/wsh/cmd/wshcmd-ai.go | 221 ++++++++++-------- docs/docs/wsh-reference.mdx | 42 +++- docs/docs/wsh.mdx | 35 ++- frontend/app/aipanel/aipanel.tsx | 107 ++------- frontend/app/aipanel/aitypes.ts | 32 ++- frontend/app/aipanel/waveai-model.tsx | 103 +++++++- frontend/app/store/tabrpcclient.ts | 16 +- .../app/workspace/workspace-layout-model.ts | 12 +- frontend/types/gotypes.d.ts | 3 +- frontend/util/util.ts | 23 +- package-lock.json | 4 +- pkg/aiusechat/openai/openai-convertmessage.go | 82 ++++++- pkg/aiusechat/usechat.go | 5 + pkg/util/utilfn/marshal.go | 39 ++++ pkg/wshrpc/wshrpctypes.go | 15 +- 15 files changed, 501 insertions(+), 238 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-ai.go b/cmd/wsh/cmd/wshcmd-ai.go index 15648f297b..e90ef6d50f 100644 --- a/cmd/wsh/cmd/wshcmd-ai.go +++ b/cmd/wsh/cmd/wshcmd-ai.go @@ -4,47 +4,69 @@ package cmd import ( + "encoding/base64" "fmt" "io" + "mime" "os" - "strings" + "path/filepath" "github.com/spf13/cobra" - "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" "github.com/wavetermdev/waveterm/pkg/wshutil" ) var aiCmd = &cobra.Command{ - Use: "ai [-] [message...]", - Short: "Send a message to an AI block", + Use: "ai [options] [files...]", + Short: "Append content to Wave AI sidebar prompt", + Long: `Append content to Wave AI sidebar prompt (does not auto-submit by default) + +Arguments: + files... Files to attach (use '-' for stdin) + +Examples: + git diff | wsh ai - # Pipe diff to AI, ask question in UI + wsh ai main.go # Attach file, ask question in UI + wsh ai *.go -m "find bugs" # Attach files with message + wsh ai -s - -m "review" < log.txt # Stdin + message, auto-submit + wsh ai -n config.json # New chat with file attached`, RunE: aiRun, PreRunE: preRunSetupRpcClient, DisableFlagsInUseLine: true, } -var aiFileFlags []string +var aiMessageFlag string +var aiSubmitFlag bool var aiNewBlockFlag bool func init() { rootCmd.AddCommand(aiCmd) - aiCmd.Flags().BoolVarP(&aiNewBlockFlag, "new", "n", false, "create a new AI block") - aiCmd.Flags().StringArrayVarP(&aiFileFlags, "file", "f", nil, "attach file content (use '-' for stdin)") + aiCmd.Flags().StringVarP(&aiMessageFlag, "message", "m", "", "optional message/question to append after files") + aiCmd.Flags().BoolVarP(&aiSubmitFlag, "submit", "s", false, "submit the prompt immediately after appending") + aiCmd.Flags().BoolVarP(&aiNewBlockFlag, "new", "n", false, "create a new AI chat instead of using existing") } -func encodeFile(builder *strings.Builder, file io.Reader, fileName string) error { - data, err := io.ReadAll(file) - if err != nil { - return fmt.Errorf("error reading file: %w", err) +func getMimeType(filename string) string { + ext := filepath.Ext(filename) + if ext == "" { + return "text/plain" } - // Start delimiter with the file name - builder.WriteString(fmt.Sprintf("\n@@@start file %q\n", fileName)) - // Read the file content and write it to the builder - builder.Write(data) - // End delimiter with the file name - builder.WriteString(fmt.Sprintf("\n@@@end file %q\n\n", fileName)) - return nil + mimeType := mime.TypeByExtension(ext) + if mimeType == "" { + return "text/plain" + } + return mimeType +} + +func getMaxFileSize(mimeType string) (int, string) { + if mimeType == "application/pdf" { + return 5 * 1024 * 1024, "5MB" + } + if mimeType[:6] == "image/" { + return 7 * 1024 * 1024, "7MB" + } + return 200 * 1024, "200KB" } func aiRun(cmd *cobra.Command, args []string) (rtnErr error) { @@ -52,118 +74,127 @@ func aiRun(cmd *cobra.Command, args []string) (rtnErr error) { sendActivity("ai", rtnErr == nil) }() - if len(args) == 0 { + if len(args) == 0 && aiMessageFlag == "" { OutputHelpMessage(cmd) - return fmt.Errorf("no message provided") + return fmt.Errorf("no files or message provided") } + const maxBatchSize = 7 * 1024 * 1024 + const largeFileThreshold = 1 * 1024 * 1024 + const maxFileCount = 15 + const rpcTimeout = 30000 + + var allFiles []wshrpc.AIAttachedFile var stdinUsed bool - var message strings.Builder - // Handle file attachments first - for _, file := range aiFileFlags { - if file == "-" { + if len(args) > maxFileCount { + return fmt.Errorf("too many files (maximum %d files allowed)", maxFileCount) + } + + for _, filePath := range args { + var data []byte + var fileName string + var mimeType string + var err error + + if filePath == "-" { if stdinUsed { return fmt.Errorf("stdin (-) can only be used once") } stdinUsed = true - if err := encodeFile(&message, os.Stdin, ""); err != nil { + + data, err = io.ReadAll(os.Stdin) + if err != nil { return fmt.Errorf("reading from stdin: %w", err) } + fileName = "stdin" + mimeType = "text/plain" } else { - fd, err := os.Open(file) + fileInfo, err := os.Stat(filePath) if err != nil { - return fmt.Errorf("opening file %s: %w", file, err) + return fmt.Errorf("accessing file %s: %w", filePath, err) } - defer fd.Close() - if err := encodeFile(&message, fd, file); err != nil { - return fmt.Errorf("reading file %s: %w", file, err) + if fileInfo.IsDir() { + return fmt.Errorf("%s is a directory, not a file", filePath) } + + data, err = os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("reading file %s: %w", filePath, err) + } + fileName = filepath.Base(filePath) + mimeType = getMimeType(filePath) } - } - // Default to "waveai" block - isDefaultBlock := blockArg == "" - if isDefaultBlock { - blockArg = "view@waveai" + maxSize, sizeStr := getMaxFileSize(mimeType) + if len(data) > maxSize { + return fmt.Errorf("file %s exceeds maximum size of %s for %s files", fileName, sizeStr, mimeType) + } + + allFiles = append(allFiles, wshrpc.AIAttachedFile{ + Name: fileName, + Type: mimeType, + Size: len(data), + Data64: base64.StdEncoding.EncodeToString(data), + }) } - var fullORef *waveobj.ORef - var err error - if !aiNewBlockFlag { - fullORef, err = resolveSimpleId(blockArg) + + tabId := os.Getenv("WAVETERM_TABID") + if tabId == "" { + return fmt.Errorf("WAVETERM_TABID environment variable not set") } - if (err != nil && isDefaultBlock) || aiNewBlockFlag { - // Create new AI block if default block doesn't exist - data := &wshrpc.CommandCreateBlockData{ - BlockDef: &waveobj.BlockDef{ - Meta: map[string]interface{}{ - waveobj.MetaKey_View: "waveai", - }, - }, - Focused: true, - } - newORef, err := wshclient.CreateBlockCommand(RpcClient, *data, &wshrpc.RpcOpts{Timeout: 2000}) - if err != nil { - return fmt.Errorf("creating AI block: %w", err) + route := wshutil.MakeTabRouteId(tabId) + + if aiNewBlockFlag { + newChatData := wshrpc.CommandWaveAIAddContextData{ + NewChat: true, } - fullORef = &newORef - // Wait for the block's route to be available - gotRoute, err := wshclient.WaitForRouteCommand(RpcClient, wshrpc.CommandWaitForRouteData{ - RouteId: wshutil.MakeFeBlockRouteId(fullORef.OID), - WaitMs: 4000, - }, &wshrpc.RpcOpts{Timeout: 5000}) + err := wshclient.WaveAIAddContextCommand(RpcClient, newChatData, &wshrpc.RpcOpts{ + Route: route, + Timeout: rpcTimeout, + }) if err != nil { - return fmt.Errorf("waiting for AI block: %w", err) + return fmt.Errorf("creating new chat: %w", err) } - if !gotRoute { - return fmt.Errorf("AI block route could not be established") - } - } else if err != nil { - return fmt.Errorf("resolving block: %w", err) } - // Create the route for this block - route := wshutil.MakeFeBlockRouteId(fullORef.OID) + var smallFiles []wshrpc.AIAttachedFile + var smallFilesSize int - // Then handle main message - if args[0] == "-" { - if stdinUsed { - return fmt.Errorf("stdin (-) can only be used once") - } - data, err := io.ReadAll(os.Stdin) - if err != nil { - return fmt.Errorf("reading from stdin: %w", err) - } - message.Write(data) - - // Also include any remaining arguments (excluding the "-" itself) - if len(args) > 1 { - if message.Len() > 0 { - message.WriteString(" ") + for _, file := range allFiles { + if file.Size > largeFileThreshold { + contextData := wshrpc.CommandWaveAIAddContextData{ + Files: []wshrpc.AIAttachedFile{file}, + } + err := wshclient.WaveAIAddContextCommand(RpcClient, contextData, &wshrpc.RpcOpts{ + Route: route, + Timeout: rpcTimeout, + }) + if err != nil { + return fmt.Errorf("adding file %s: %w", file.Name, err) } - message.WriteString(strings.Join(args[1:], " ")) + } else { + smallFilesSize += file.Size + if smallFilesSize > maxBatchSize { + return fmt.Errorf("small files total size exceeds maximum batch size of 7MB") + } + smallFiles = append(smallFiles, file) } - } else { - message.WriteString(strings.Join(args, " ")) } - if message.Len() == 0 { - return fmt.Errorf("message is empty") - } - if message.Len() > 50*1024 { - return fmt.Errorf("current max message size is 50k") + finalContextData := wshrpc.CommandWaveAIAddContextData{ + Files: smallFiles, + Text: aiMessageFlag, + Submit: aiSubmitFlag, } - messageData := wshrpc.AiMessageData{ - Message: message.String(), - } - err = wshclient.AiSendMessageCommand(RpcClient, messageData, &wshrpc.RpcOpts{ + err := wshclient.WaveAIAddContextCommand(RpcClient, finalContextData, &wshrpc.RpcOpts{ Route: route, - Timeout: 2000, + Timeout: rpcTimeout, }) if err != nil { - return fmt.Errorf("sending message: %w", err) + return fmt.Errorf("adding context: %w", err) } return nil diff --git a/docs/docs/wsh-reference.mdx b/docs/docs/wsh-reference.mdx index 0903cde982..d484d9fe3f 100644 --- a/docs/docs/wsh-reference.mdx +++ b/docs/docs/wsh-reference.mdx @@ -110,25 +110,45 @@ wsh getmeta -b [other-tab-id] "bg:*" --clear-prefix | wsh setmeta -b tab --json ## ai -Send messages to new or existing AI blocks directly from the CLI. `-f` passes a file. note that there is a maximum size of 10k for messages and files, so use a tail/grep to cut down file sizes before passing. The `-f` option works great for small files though like shell scripts or `.zshrc` etc. You can use "-" to read input from stdin. +Append content to the Wave AI sidebar. Files are attached as proper file attachments (supporting images, PDFs, and text), not encoded as text. By default, content is added to the sidebar without auto-submitting, allowing you to review and add more context before sending to the AI. -By default the messages get sent to the first AI block (by blocknum). If no AI block exists, then a new one will be created. Use `-n` to force creation of a new AI block. Use `-b` to target a specific AI block. +You can attach multiple files at once (up to 15 files). Use `-m` to add a message along with files, `-s` to auto-submit immediately, and `-n` to start a new chat conversation. Use "-" to read from stdin. ```sh -wsh ai "how do i write an ls command that sorts files in reverse size order" -wsh ai -f <(tail -n 20 "my.log") -- "any idea what these error messages mean" -wsh ai -f README.md "help me update this readme file" +# Pipe command output to AI (ask question in UI) +git diff | wsh ai - +docker logs mycontainer | wsh ai - -# creates a new AI block -wsh ai -n "tell me a story" +# Attach files without auto-submit (review in UI first) +wsh ai main.go utils.go +wsh ai screenshot.png logs.txt -# targets block number 5 -wsh ai -b 5 "tell me more" +# Attach files with message +wsh ai app.py -m "find potential bugs" +wsh ai *.log -m "analyze these error logs" -# read from stdin and also supply a message -tail -n 50 mylog.log | wsh ai - "can you tell me what this error means?" +# Auto-submit immediately +wsh ai config.json -s -m "explain this configuration" +tail -n 50 app.log | wsh ai -s - -m "what's causing these errors?" + +# Start new chat and attach files +wsh ai -n report.pdf data.csv -m "summarize these reports" + +# Attach different file types (images, PDFs, code) +wsh ai architecture.png api-spec.pdf server.go -m "review the system design" ``` +**File Size Limits:** +- Text files: 200KB maximum +- PDF files: 5MB maximum +- Image files: 7MB maximum (accounts for base64 encoding overhead) +- Maximum 15 files per command + +**Flags:** +- `-m, --message ` - Add message text along with files +- `-s, --submit` - Auto-submit immediately (default waits for user) +- `-n, --new` - Clear current chat and start fresh conversation + --- ## editconfig diff --git a/docs/docs/wsh.mdx b/docs/docs/wsh.mdx index ac9b007510..37f5be1fd8 100644 --- a/docs/docs/wsh.mdx +++ b/docs/docs/wsh.mdx @@ -116,17 +116,40 @@ wsh setvar -b tab SHARED_ENV=staging ### AI-Assisted Development +The `wsh ai` command appends content to the Wave AI sidebar. By default, files are attached without auto-submitting, allowing you to review and add more context before sending. + ```bash -# Get AI help with code (uses "-" to read from stdin) -git diff | wsh ai - "review these changes" +# Pipe output to AI sidebar (ask question in UI) +git diff | wsh ai - + +# Attach files with a message +wsh ai main.go utils.go -m "find bugs in these files" + +# Auto-submit with message +wsh ai config.json -s -m "explain this config" -# Get help with a file -wsh ai -f .zshrc "help me add ~/bin to my path" +# Start new chat with attached files +wsh ai -n *.log -m "analyze these logs" -# Debug issues (uses "-" to read from stdin) -dmesg | wsh ai - "help me understand these errors" +# Attach multiple file types (images, PDFs, code) +wsh ai screenshot.png report.pdf app.py -m "review these" + +# Debug with stdin and auto-submit +dmesg | wsh ai -s - -m "help me understand these errors" ``` +**Flags:** +- `-` - Read from stdin instead of a file +- `-m, --message` - Add message text along with files +- `-s, --submit` - Auto-submit immediately (default is to wait for user) +- `-n, --new` - Clear chat and start fresh conversation + +**File Limits:** +- Text files: 200KB max +- PDFs: 5MB max +- Images: 7MB max +- Maximum 15 files per command + ## Tips & Features 1. **Working with Blocks** diff --git a/frontend/app/aipanel/aipanel.tsx b/frontend/app/aipanel/aipanel.tsx index 4231d3add4..7d40d83ce2 100644 --- a/frontend/app/aipanel/aipanel.tsx +++ b/frontend/app/aipanel/aipanel.tsx @@ -1,7 +1,6 @@ // Copyright 2025, Command Line Inc. // SPDX-License-Identifier: Apache-2.0 -import { WaveUIMessagePart } from "@/app/aipanel/aitypes"; import { waveAIHasSelection } from "@/app/aipanel/waveai-focus-utils"; import { ErrorBoundary } from "@/app/element/errorboundary"; import { ContextMenuModel } from "@/app/store/contextmenu"; @@ -17,14 +16,14 @@ import { useChat } from "@ai-sdk/react"; import { DefaultChatTransport } from "ai"; import * as jotai from "jotai"; import { memo, useCallback, useEffect, useRef, useState } from "react"; -import { createDataUrl, formatFileSizeError, isAcceptableFile, normalizeMimeType, validateFileSize } from "./ai-utils"; +import { formatFileSizeError, isAcceptableFile, validateFileSize } from "./ai-utils"; import { AIDroppedFiles } from "./aidroppedfiles"; import { AIPanelHeader } from "./aipanelheader"; import { AIPanelInput } from "./aipanelinput"; import { AIPanelMessages } from "./aipanelmessages"; import { AIRateLimitStrip } from "./airatelimitstrip"; import { TelemetryRequiredMessage } from "./telemetryrequired"; -import { WaveAIModel, type DroppedFile } from "./waveai-model"; +import { WaveAIModel } from "./waveai-model"; const AIBlockMask = memo(() => { return ( @@ -195,11 +194,10 @@ interface AIPanelProps { const AIPanelComponentInner = memo(({ className, onClose }: AIPanelProps) => { const [isDragOver, setIsDragOver] = useState(false); - const [isLoadingChat, setIsLoadingChat] = useState(true); + const [initialLoadDone, setInitialLoadDone] = useState(false); const model = WaveAIModel.getInstance(); const containerRef = useRef(null); const errorMessage = jotai.useAtomValue(model.errorMessage); - const realMessageRef = useRef(null); const isLayoutMode = jotai.useAtomValue(atoms.controlShiftDelayAtom); const showOverlayBlockNums = jotai.useAtomValue(getSettingsKeyAtom("app:showoverlayblocknums")) ?? true; const focusType = jotai.useAtomValue(focusManager.focusType); @@ -211,8 +209,7 @@ const AIPanelComponentInner = memo(({ className, onClose }: AIPanelProps) => { transport: new DefaultChatTransport({ api: `${getWebServerEndpoint()}/api/post-chat-message`, prepareSendMessagesRequest: (opts) => { - const msg = realMessageRef.current; - realMessageRef.current = null; + const msg = model.getAndClearMessage(); return { body: { msg, @@ -235,16 +232,17 @@ const AIPanelComponentInner = memo(({ className, onClose }: AIPanelProps) => { }, }); + model.registerUseChatData(sendMessage, setMessages, status); + // console.log("AICHAT messages", messages); - const clearChat = () => { + const handleClearChat = useCallback(() => { model.clearChat(); - setMessages([]); - }; + }, [model]); const handleKeyDown = (waveEvent: WaveKeyboardEvent): boolean => { if (checkKeyPressed(waveEvent, "Cmd:k")) { - clearChat(); + model.clearChat(); return true; } return false; @@ -259,16 +257,12 @@ const AIPanelComponentInner = memo(({ className, onClose }: AIPanelProps) => { }, []); useEffect(() => { - const loadMessages = async () => { - const messages = await model.loadChat(); - setMessages(messages as any); - setIsLoadingChat(false); - setTimeout(() => { - model.scrollToBottom(); - }, 100); + const loadChat = async () => { + await model.uiLoadChat(); + setInitialLoadDone(true); }; - loadMessages(); - }, [model, setMessages]); + loadChat(); + }, [model]); useEffect(() => { const updateWidth = () => { @@ -295,72 +289,7 @@ const AIPanelComponentInner = memo(({ className, onClose }: AIPanelProps) => { const handleSubmit = async (e: React.FormEvent) => { e.preventDefault(); - const input = globalStore.get(model.inputAtom); - if (!input.trim() || status !== "ready" || isLoadingChat) return; - - if (input.trim() === "/clear" || input.trim() === "/new") { - clearChat(); - globalStore.set(model.inputAtom, ""); - return; - } - - model.clearError(); - - const droppedFiles = globalStore.get(model.droppedFiles) as DroppedFile[]; - - // Prepare AI message parts (for backend) - const aiMessageParts: AIMessagePart[] = [{ type: "text", text: input.trim() }]; - - // Prepare UI message parts (for frontend display) - const uiMessageParts: WaveUIMessagePart[] = []; - - if (input.trim()) { - uiMessageParts.push({ type: "text", text: input.trim() }); - } - - // Process files - for (const droppedFile of droppedFiles) { - const normalizedMimeType = normalizeMimeType(droppedFile.file); - const dataUrl = await createDataUrl(droppedFile.file); - - // For AI message (backend) - use data URL - aiMessageParts.push({ - type: "file", - filename: droppedFile.name, - mimetype: normalizedMimeType, - url: dataUrl, - size: droppedFile.file.size, - previewurl: droppedFile.previewUrl, - }); - - uiMessageParts.push({ - type: "data-userfile", - data: { - filename: droppedFile.name, - mimetype: normalizedMimeType, - size: droppedFile.file.size, - previewurl: droppedFile.previewUrl, - }, - }); - } - - // realMessage uses AIMessageParts - const realMessage: AIMessage = { - messageid: crypto.randomUUID(), - parts: aiMessageParts, - }; - realMessageRef.current = realMessage; - - // sendMessage uses UIMessageParts - sendMessage({ parts: uiMessageParts }); - - model.isChatEmpty = false; - globalStore.set(model.inputAtom, ""); - model.clearFiles(); - - setTimeout(() => { - model.focusInput(); - }, 100); + await model.handleSubmit(true); }; const hasFilesDragged = (dataTransfer: DataTransfer): boolean => { @@ -473,7 +402,7 @@ const AIPanelComponentInner = memo(({ className, onClose }: AIPanelProps) => { menu.push({ label: "New Chat", click: () => { - clearChat(); + model.clearChat(); }, }); @@ -516,7 +445,7 @@ const AIPanelComponentInner = memo(({ className, onClose }: AIPanelProps) => { > {isDragOver && } {showBlockMask && } - +
@@ -524,7 +453,7 @@ const AIPanelComponentInner = memo(({ className, onClose }: AIPanelProps) => { ) : ( <> - {messages.length === 0 && !isLoadingChat ? ( + {messages.length === 0 && initialLoadDone ? (
diff --git a/frontend/app/aipanel/aitypes.ts b/frontend/app/aipanel/aitypes.ts index f16c1f6e3c..5fcb25cd3f 100644 --- a/frontend/app/aipanel/aitypes.ts +++ b/frontend/app/aipanel/aitypes.ts @@ -1,7 +1,7 @@ // Copyright 2025, Command Line Inc. // SPDX-License-Identifier: Apache-2.0 -import { UIMessage, UIMessagePart } from "ai"; +import { ChatRequestOptions, FileUIPart, UIMessage, UIMessagePart } from "ai"; type WaveUIDataTypes = { userfile: { @@ -22,3 +22,33 @@ type WaveUIDataTypes = { export type WaveUIMessage = UIMessage; export type WaveUIMessagePart = UIMessagePart; + +export type UseChatSetMessagesType = ( + messages: WaveUIMessage[] | ((messages: WaveUIMessage[]) => WaveUIMessage[]) +) => void; + +export type UseChatSendMessageType = ( + message?: + | (Omit & { + id?: string; + role?: "system" | "user" | "assistant"; + } & { + text?: never; + files?: never; + messageId?: string; + }) + | { + text: string; + files?: FileList | FileUIPart[]; + metadata?: unknown; + parts?: never; + messageId?: string; + } + | { + files: FileList | FileUIPart[]; + metadata?: unknown; + parts?: never; + messageId?: string; + }, + options?: ChatRequestOptions +) => Promise; diff --git a/frontend/app/aipanel/waveai-model.tsx b/frontend/app/aipanel/waveai-model.tsx index a8e8a2d421..d55b36a6f3 100644 --- a/frontend/app/aipanel/waveai-model.tsx +++ b/frontend/app/aipanel/waveai-model.tsx @@ -1,15 +1,17 @@ // Copyright 2025, Command Line Inc. // SPDX-License-Identifier: Apache-2.0 +import { UseChatSendMessageType, UseChatSetMessagesType, WaveUIMessagePart } from "@/app/aipanel/aitypes"; import { atoms, getTabMetaKeyAtom } from "@/app/store/global"; import { globalStore } from "@/app/store/jotaiStore"; import * as WOS from "@/app/store/wos"; import { RpcApi } from "@/app/store/wshclientapi"; import { TabRpcClient } from "@/app/store/wshrpcutil"; import { WorkspaceLayoutModel } from "@/app/workspace/workspace-layout-model"; +import { ChatStatus } from "ai"; import * as jotai from "jotai"; import type React from "react"; -import { createImagePreview, resizeImage } from "./ai-utils"; +import { createDataUrl, createImagePreview, normalizeMimeType, resizeImage } from "./ai-utils"; import type { AIPanelInputRef } from "./aipanelinput"; export interface DroppedFile { @@ -25,6 +27,11 @@ export class WaveAIModel { private static instance: WaveAIModel | null = null; private inputRef: React.RefObject | null = null; private scrollToBottomCallback: (() => void) | null = null; + private useChatSendMessage: UseChatSendMessageType | null = null; + private useChatSetMessages: UseChatSetMessagesType | null = null; + private useChatStatus: ChatStatus = "ready"; + // Used for injecting Wave-specific message data into DefaultChatTransport's prepareSendMessagesRequest + realMessage: AIMessage | null = null; widgetAccessAtom!: jotai.Atom; droppedFiles: jotai.PrimitiveAtom = jotai.atom([]); @@ -34,6 +41,7 @@ export class WaveAIModel { containerWidth: jotai.PrimitiveAtom = jotai.atom(0); codeBlockMaxWidth!: jotai.Atom; inputAtom: jotai.PrimitiveAtom = jotai.atom(""); + isLoadingChatAtom: jotai.PrimitiveAtom = jotai.atom(false); isChatEmpty: boolean = true; private constructor() { @@ -137,6 +145,8 @@ export class WaveAIModel { oref: WOS.makeORef("tab", tabId), meta: { "waveai:chatid": newChatId }, }); + + this.useChatSetMessages?.([]); } setError(message: string) { @@ -155,6 +165,12 @@ export class WaveAIModel { this.scrollToBottomCallback = callback; } + registerUseChatData(sendMessage: UseChatSendMessageType, setMessages: UseChatSetMessagesType, status: ChatStatus) { + this.useChatSendMessage = sendMessage; + this.useChatSetMessages = setMessages; + this.useChatStatus = status; + } + scrollToBottom() { this.scrollToBottomCallback?.(); } @@ -168,6 +184,12 @@ export class WaveAIModel { } } + getAndClearMessage(): AIMessage | null { + const msg = this.realMessage; + this.realMessage = null; + return msg; + } + hasNonEmptyInput(): boolean { const input = globalStore.get(this.inputAtom); return input != null && input.trim().length > 0; @@ -226,6 +248,85 @@ export class WaveAIModel { } } + async handleSubmit(shouldFocus: boolean) { + const input = globalStore.get(this.inputAtom); + if (!input.trim() || this.useChatStatus !== "ready" || globalStore.get(this.isLoadingChatAtom)) { + return; + } + + if (input.trim() === "/clear" || input.trim() === "/new") { + this.clearChat(); + globalStore.set(this.inputAtom, ""); + return; + } + + this.clearError(); + + const droppedFiles = globalStore.get(this.droppedFiles); + + const aiMessageParts: AIMessagePart[] = [{ type: "text", text: input.trim() }]; + + const uiMessageParts: WaveUIMessagePart[] = []; + + if (input.trim()) { + uiMessageParts.push({ type: "text", text: input.trim() }); + } + + for (const droppedFile of droppedFiles) { + const normalizedMimeType = normalizeMimeType(droppedFile.file); + const dataUrl = await createDataUrl(droppedFile.file); + + aiMessageParts.push({ + type: "file", + filename: droppedFile.name, + mimetype: normalizedMimeType, + url: dataUrl, + size: droppedFile.file.size, + previewurl: droppedFile.previewUrl, + }); + + uiMessageParts.push({ + type: "data-userfile", + data: { + filename: droppedFile.name, + mimetype: normalizedMimeType, + size: droppedFile.file.size, + previewurl: droppedFile.previewUrl, + }, + }); + } + + const realMessage: AIMessage = { + messageid: crypto.randomUUID(), + parts: aiMessageParts, + }; + this.realMessage = realMessage; + + // console.log("SUBMIT MESSAGE", realMessage); + + this.useChatSendMessage?.({ parts: uiMessageParts }); + + this.isChatEmpty = false; + globalStore.set(this.inputAtom, ""); + this.clearFiles(); + + if (shouldFocus) { + setTimeout(() => { + this.focusInput(); + }, 100); + } + } + + async uiLoadChat() { + globalStore.set(this.isLoadingChatAtom, true); + const messages = await this.loadChat(); + this.useChatSetMessages?.(messages as any); + globalStore.set(this.isLoadingChatAtom, false); + setTimeout(() => { + this.scrollToBottom(); + }, 100); + } + async ensureRateLimitSet() { const currentInfo = globalStore.get(atoms.waveAIRateLimitInfoAtom); if (currentInfo != null) { diff --git a/frontend/app/store/tabrpcclient.ts b/frontend/app/store/tabrpcclient.ts index c8e3c0121a..7b6449c18b 100644 --- a/frontend/app/store/tabrpcclient.ts +++ b/frontend/app/store/tabrpcclient.ts @@ -3,7 +3,9 @@ import { WaveAIModel } from "@/app/aipanel/waveai-model"; import { getApi } from "@/app/store/global"; +import { WorkspaceLayoutModel } from "@/app/workspace/workspace-layout-model"; import { getLayoutModelForStaticTab } from "@/layout/index"; +import { base64ToArray } from "@/util/util"; import { RpcResponseHelper, WshClient } from "./wshclient"; export class TabClient extends WshClient { @@ -59,11 +61,21 @@ export class TabClient extends WshClient { } async handle_waveaiaddcontext(rh: RpcResponseHelper, data: CommandWaveAIAddContextData): Promise { + const workspaceLayoutModel = WorkspaceLayoutModel.getInstance(); + if (!workspaceLayoutModel.getAIPanelVisible()) { + workspaceLayoutModel.setAIPanelVisible(true, { nofocus: true }); + } + const model = WaveAIModel.getInstance(); + if (data.newchat) { + model.clearChat(); + } + if (data.files && data.files.length > 0) { for (const fileData of data.files) { - const blob = new Blob([fileData.data], { type: fileData.type }); + const decodedData = base64ToArray(fileData.data64); + const blob = new Blob([decodedData], { type: fileData.type }); const file = new File([blob], fileData.name, { type: fileData.type }); await model.addFile(file); } @@ -74,7 +86,7 @@ export class TabClient extends WshClient { } if (data.submit) { - model.focusInput(); + await model.handleSubmit(false); } } } diff --git a/frontend/app/workspace/workspace-layout-model.ts b/frontend/app/workspace/workspace-layout-model.ts index da1ffa4a1c..9b5437316f 100644 --- a/frontend/app/workspace/workspace-layout-model.ts +++ b/frontend/app/workspace/workspace-layout-model.ts @@ -218,7 +218,7 @@ class WorkspaceLayoutModel { return this.aiPanelVisible; } - setAIPanelVisible(visible: boolean): void { + setAIPanelVisible(visible: boolean, opts?: { nofocus?: boolean }): void { if (this.focusTimeoutRef != null) { clearTimeout(this.focusTimeoutRef); this.focusTimeoutRef = null; @@ -238,10 +238,12 @@ class WorkspaceLayoutModel { this.syncAIPanelRef(); if (visible) { - this.focusTimeoutRef = setTimeout(() => { - WaveAIModel.getInstance().focusInput(); - this.focusTimeoutRef = null; - }, 350); + if (!opts?.nofocus) { + this.focusTimeoutRef = setTimeout(() => { + WaveAIModel.getInstance().focusInput(); + this.focusTimeoutRef = null; + }, 350); + } } else { const layoutModel = getLayoutModelForStaticTab(); const focusedNode = globalStore.get(layoutModel.focusedNode); diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index 88da69bbfb..3b88370c4a 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -10,7 +10,7 @@ declare global { name: string; type: string; size: number; - data: string; + data64: string; }; // wshrpc.ActivityDisplayType @@ -333,6 +333,7 @@ declare global { files?: AIAttachedFile[]; text?: string; submit?: boolean; + newchat?: boolean; }; // wshrpc.CommandWaveAIToolApproveData diff --git a/frontend/util/util.ts b/frontend/util/util.ts index 4a081ad64d..69e17fceb2 100644 --- a/frontend/util/util.ts +++ b/frontend/util/util.ts @@ -4,8 +4,8 @@ import base64 from "base64-js"; import clsx, { type ClassValue } from "clsx"; import { Atom, atom, Getter, SetStateAction, Setter, useAtomValue } from "jotai"; -import { debounce, throttle } from "throttle-debounce"; import { twMerge } from "tailwind-merge"; +import { debounce, throttle } from "throttle-debounce"; const prevValueCache = new WeakMap(); // stores a previous value for a deep equal comparison (used with the deepCompareReturnPrev function) function isBlank(str: string): boolean { @@ -28,7 +28,7 @@ function stringToBase64(input: string): string { return base64.fromByteArray(stringBytes); } -function base64ToArray(b64: string): Uint8Array { +function base64ToArray(b64: string): Uint8Array { const rawStr = atob(b64); const rtnArr = new Uint8Array(new ArrayBuffer(rawStr.length)); for (let i = 0; i < rawStr.length; i++) { @@ -379,17 +379,22 @@ function mergeMeta(meta: MetaType, metaUpdate: MetaType, prefix?: string): MetaT } function escapeBytes(str: string): string { - return str.replace(/[\s\S]/g, ch => { + return str.replace(/[\s\S]/g, (ch) => { const code = ch.charCodeAt(0); switch (ch) { - case "\n": return "\\n"; - case "\r": return "\\r"; - case "\t": return "\\t"; - case "\b": return "\\b"; - case "\f": return "\\f"; + case "\n": + return "\\n"; + case "\r": + return "\\r"; + case "\t": + return "\\t"; + case "\b": + return "\\b"; + case "\f": + return "\\f"; } if (code === 0x1b) return "\\x1b"; // escape - if (code < 0x20 || code === 0x7f) return `\\x${code.toString(16).padStart(2,"0")}`; + if (code < 0x20 || code === 0x7f) return `\\x${code.toString(16).padStart(2, "0")}`; return ch; }); } diff --git a/package-lock.json b/package-lock.json index 74aec0396c..54a16f19ba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "waveterm", - "version": "0.12.0-beta.2", + "version": "0.12.0-beta.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "waveterm", - "version": "0.12.0-beta.2", + "version": "0.12.0-beta.3", "hasInstallScript": true, "license": "Apache-2.0", "workspaces": [ diff --git a/pkg/aiusechat/openai/openai-convertmessage.go b/pkg/aiusechat/openai/openai-convertmessage.go index 4fccea00a0..1065423ab4 100644 --- a/pkg/aiusechat/openai/openai-convertmessage.go +++ b/pkg/aiusechat/openai/openai-convertmessage.go @@ -12,10 +12,12 @@ import ( "fmt" "log" "net/http" + "strconv" "strings" "github.com/google/uuid" "github.com/wavetermdev/waveterm/pkg/aiusechat/uctypes" + "github.com/wavetermdev/waveterm/pkg/util/utilfn" ) const ( @@ -23,6 +25,38 @@ const ( OpenAIDefaultMaxTokens = 4096 ) +// extractXmlAttribute extracts an attribute value from an XML-like tag. +// Expects double-quoted strings where internal quotes are encoded as ". +// Returns the unquoted value and true if found, or empty string and false if not found or invalid. +func extractXmlAttribute(tag, attrName string) (string, bool) { + attrStart := strings.Index(tag, attrName+"=") + if attrStart == -1 { + return "", false + } + + pos := attrStart + len(attrName+"=") + start := strings.Index(tag[pos:], `"`) + if start == -1 { + return "", false + } + start += pos + + end := strings.Index(tag[start+1:], `"`) + if end == -1 { + return "", false + } + end += start + 1 + + quotedValue := tag[start : end+1] + value, err := strconv.Unquote(quotedValue) + if err != nil { + return "", false + } + + value = strings.ReplaceAll(value, """, `"`) + return value, true +} + // ---------- OpenAI Request Types ---------- type StreamOptionsType struct { @@ -292,24 +326,34 @@ func convertFileAIMessagePart(part uctypes.AIMessagePart) (*OpenAIMessageContent }, nil case part.MimeType == "text/plain": - // Handle text/plain files as input_text with special formatting var textContent string if len(part.Data) > 0 { textContent = string(part.Data) } else if part.URL != "" { - return nil, fmt.Errorf("dropping text/plain file with URL (must be fetched and converted to data)") + if strings.HasPrefix(part.URL, "data:") { + _, decodedData, err := utilfn.DecodeDataURL(part.URL) + if err != nil { + return nil, fmt.Errorf("failed to decode data URL for text/plain file: %w", err) + } + textContent = string(decodedData) + } else { + return nil, fmt.Errorf("dropping text/plain file with URL (must be fetched and converted to data)") + } } else { return nil, fmt.Errorf("text/plain file part missing data") } - // Format as: file "filename" (mimetype)\n\nfile-content fileName := part.FileName if fileName == "" { fileName = "untitled.txt" } - formattedText := fmt.Sprintf("file %q (%s)\n\n%s", fileName, part.MimeType, textContent) + encodedFileName := strings.ReplaceAll(fileName, `"`, """) + quotedFileName := strconv.Quote(encodedFileName) + + randomSuffix := uuid.New().String()[0:8] + formattedText := fmt.Sprintf("\n%s\n", randomSuffix, quotedFileName, textContent, randomSuffix) return &OpenAIMessageContent{ Type: "input_text", @@ -435,11 +479,31 @@ func (m *OpenAIChatMessage) ConvertToUIMessage() *uctypes.UIMessage { for _, block := range m.Message.Content { switch block.Type { case "input_text", "output_text": - // Convert text blocks to UIMessagePart - parts = append(parts, uctypes.UIMessagePart{ - Type: "text", - Text: block.Text, - }) + if strings.HasPrefix(block.Text, "") + if openTagEnd == -1 { + continue + } + + openTag := block.Text[:openTagEnd] + fileName, ok := extractXmlAttribute(openTag, "file_name") + if !ok { + continue + } + + parts = append(parts, uctypes.UIMessagePart{ + Type: "data-userfile", + Data: uctypes.UIMessageDataUserFile{ + FileName: fileName, + MimeType: "text/plain", + }, + }) + } else { + parts = append(parts, uctypes.UIMessagePart{ + Type: "text", + Text: block.Text, + }) + } case "input_image": // Convert image blocks to data-userfile UIMessagePart (only for user role) if role == "user" { diff --git a/pkg/aiusechat/usechat.go b/pkg/aiusechat/usechat.go index a7e54e27a9..58f8b9676b 100644 --- a/pkg/aiusechat/usechat.go +++ b/pkg/aiusechat/usechat.go @@ -65,6 +65,11 @@ var SystemPromptText_OpenAI = strings.Join([]string{ // Crisp behavior `Be concise and direct. Prefer determinism over speculation. If a brief clarifying question eliminates guesswork, ask it.`, + // Attached text files + `User-attached text files may appear inline as content.`, + `If multiple attached files exist, treat each as a separate source file with its own file_name.`, + `When the user refers to these files, use their inline content directly; do NOT call any read_text_file or file-access tools to re-read them unless asked.`, + // Output & formatting `When presenting commands or any runnable multi-line code, always use fenced Markdown code blocks.`, `Use an appropriate language hint after the opening fence (e.g., "bash" for shell commands, "go" for Go, "json" for JSON).`, diff --git a/pkg/util/utilfn/marshal.go b/pkg/util/utilfn/marshal.go index 63c4617895..7789fa9ba7 100644 --- a/pkg/util/utilfn/marshal.go +++ b/pkg/util/utilfn/marshal.go @@ -5,8 +5,10 @@ package utilfn import ( "bytes" + "encoding/base64" "encoding/json" "fmt" + "net/url" "reflect" "strings" @@ -162,3 +164,40 @@ func setValue(field reflect.Value, value any) error { return fmt.Errorf("cannot set value of type %v to field of type %v", valueRef.Type(), field.Type()) } + +// DecodeDataURL decodes a data URL and returns the mimetype and raw data bytes +func DecodeDataURL(dataURL string) (mimeType string, data []byte, err error) { + if !strings.HasPrefix(dataURL, "data:") { + return "", nil, fmt.Errorf("invalid data URL: must start with 'data:'") + } + + parts := strings.SplitN(dataURL, ",", 2) + if len(parts) != 2 { + return "", nil, fmt.Errorf("invalid data URL format: missing comma separator") + } + + header := parts[0] + dataStr := parts[1] + + // Parse mimetype from header: "data:text/plain;base64" -> "text/plain" + headerWithoutPrefix := strings.TrimPrefix(header, "data:") + mimeType = strings.Split(headerWithoutPrefix, ";")[0] + if mimeType == "" { + mimeType = "text/plain" // default mimetype + } + + if strings.Contains(header, ";base64") { + decoded, decodeErr := base64.StdEncoding.DecodeString(dataStr) + if decodeErr != nil { + return "", nil, fmt.Errorf("failed to decode base64 data: %w", decodeErr) + } + return mimeType, decoded, nil + } + + // Non-base64 data URLs are percent-encoded + decoded, decodeErr := url.QueryUnescape(dataStr) + if decodeErr != nil { + return "", nil, fmt.Errorf("failed to decode percent-encoded data: %w", decodeErr) + } + return mimeType, []byte(decoded), nil +} diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 1c63f83802..f1abf21e04 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -737,16 +737,17 @@ type CommandWaveAIToolApproveData struct { } type AIAttachedFile struct { - Name string `json:"name"` - Type string `json:"type"` - Size int `json:"size"` - Data []byte `json:"data"` + Name string `json:"name"` + Type string `json:"type"` + Size int `json:"size"` + Data64 string `json:"data64"` } type CommandWaveAIAddContextData struct { - Files []AIAttachedFile `json:"files,omitempty"` - Text string `json:"text,omitempty"` - Submit bool `json:"submit,omitempty"` + Files []AIAttachedFile `json:"files,omitempty"` + Text string `json:"text,omitempty"` + Submit bool `json:"submit,omitempty"` + NewChat bool `json:"newchat,omitempty"` } type CommandCaptureBlockScreenshotData struct { From 8de7e5de91b993016d864ad8168a939f275860c6 Mon Sep 17 00:00:00 2001 From: sawka Date: Wed, 15 Oct 2025 16:14:55 -0700 Subject: [PATCH 04/11] fix submit gate. can submit with empty input if we have dropped files. also allow submitting when we have an error status --- frontend/app/aipanel/waveai-model.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/app/aipanel/waveai-model.tsx b/frontend/app/aipanel/waveai-model.tsx index d55b36a6f3..62a24673c3 100644 --- a/frontend/app/aipanel/waveai-model.tsx +++ b/frontend/app/aipanel/waveai-model.tsx @@ -250,7 +250,9 @@ export class WaveAIModel { async handleSubmit(shouldFocus: boolean) { const input = globalStore.get(this.inputAtom); - if (!input.trim() || this.useChatStatus !== "ready" || globalStore.get(this.isLoadingChatAtom)) { + const droppedFiles = globalStore.get(this.droppedFiles); + + if ((!input.trim() && droppedFiles.length === 0) || (this.useChatStatus !== "ready" && this.useChatStatus !== "error") || globalStore.get(this.isLoadingChatAtom)) { return; } @@ -262,8 +264,6 @@ export class WaveAIModel { this.clearError(); - const droppedFiles = globalStore.get(this.droppedFiles); - const aiMessageParts: AIMessagePart[] = [{ type: "text", text: input.trim() }]; const uiMessageParts: WaveUIMessagePart[] = []; From 1e25e6d3873f0b2b40a07904d03b2e8b51b56206 Mon Sep 17 00:00:00 2001 From: sawka Date: Wed, 15 Oct 2025 16:19:07 -0700 Subject: [PATCH 05/11] fix issue with empty text parts when input.trim() was empty... --- frontend/app/aipanel/waveai-model.tsx | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/frontend/app/aipanel/waveai-model.tsx b/frontend/app/aipanel/waveai-model.tsx index 62a24673c3..12803d6d88 100644 --- a/frontend/app/aipanel/waveai-model.tsx +++ b/frontend/app/aipanel/waveai-model.tsx @@ -251,8 +251,12 @@ export class WaveAIModel { async handleSubmit(shouldFocus: boolean) { const input = globalStore.get(this.inputAtom); const droppedFiles = globalStore.get(this.droppedFiles); - - if ((!input.trim() && droppedFiles.length === 0) || (this.useChatStatus !== "ready" && this.useChatStatus !== "error") || globalStore.get(this.isLoadingChatAtom)) { + + if ( + (!input.trim() && droppedFiles.length === 0) || + (this.useChatStatus !== "ready" && this.useChatStatus !== "error") || + globalStore.get(this.isLoadingChatAtom) + ) { return; } @@ -264,11 +268,11 @@ export class WaveAIModel { this.clearError(); - const aiMessageParts: AIMessagePart[] = [{ type: "text", text: input.trim() }]; - + const aiMessageParts: AIMessagePart[] = []; const uiMessageParts: WaveUIMessagePart[] = []; if (input.trim()) { + aiMessageParts.push({ type: "text", text: input.trim() }); uiMessageParts.push({ type: "text", text: input.trim() }); } From bbf81dec244efb3803443d3d3afa9dc2a371dd8d Mon Sep 17 00:00:00 2001 From: sawka Date: Wed, 15 Oct 2025 16:25:08 -0700 Subject: [PATCH 06/11] cleaner focus handling after submit, allow clear and new to run before input validation --- frontend/app/aipanel/aipanel.tsx | 5 ++++- frontend/app/aipanel/waveai-model.tsx | 20 +++++++------------- frontend/app/store/tabrpcclient.ts | 2 +- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/frontend/app/aipanel/aipanel.tsx b/frontend/app/aipanel/aipanel.tsx index 7d40d83ce2..04d75714bc 100644 --- a/frontend/app/aipanel/aipanel.tsx +++ b/frontend/app/aipanel/aipanel.tsx @@ -289,7 +289,10 @@ const AIPanelComponentInner = memo(({ className, onClose }: AIPanelProps) => { const handleSubmit = async (e: React.FormEvent) => { e.preventDefault(); - await model.handleSubmit(true); + await model.handleSubmit(); + setTimeout(() => { + model.focusInput(); + }, 100); }; const hasFilesDragged = (dataTransfer: DataTransfer): boolean => { diff --git a/frontend/app/aipanel/waveai-model.tsx b/frontend/app/aipanel/waveai-model.tsx index 12803d6d88..3f99e5a7d5 100644 --- a/frontend/app/aipanel/waveai-model.tsx +++ b/frontend/app/aipanel/waveai-model.tsx @@ -248,10 +248,16 @@ export class WaveAIModel { } } - async handleSubmit(shouldFocus: boolean) { + async handleSubmit() { const input = globalStore.get(this.inputAtom); const droppedFiles = globalStore.get(this.droppedFiles); + if (input.trim() === "/clear" || input.trim() === "/new") { + this.clearChat(); + globalStore.set(this.inputAtom, ""); + return; + } + if ( (!input.trim() && droppedFiles.length === 0) || (this.useChatStatus !== "ready" && this.useChatStatus !== "error") || @@ -260,12 +266,6 @@ export class WaveAIModel { return; } - if (input.trim() === "/clear" || input.trim() === "/new") { - this.clearChat(); - globalStore.set(this.inputAtom, ""); - return; - } - this.clearError(); const aiMessageParts: AIMessagePart[] = []; @@ -313,12 +313,6 @@ export class WaveAIModel { this.isChatEmpty = false; globalStore.set(this.inputAtom, ""); this.clearFiles(); - - if (shouldFocus) { - setTimeout(() => { - this.focusInput(); - }, 100); - } } async uiLoadChat() { diff --git a/frontend/app/store/tabrpcclient.ts b/frontend/app/store/tabrpcclient.ts index 7b6449c18b..f0c6da55ce 100644 --- a/frontend/app/store/tabrpcclient.ts +++ b/frontend/app/store/tabrpcclient.ts @@ -86,7 +86,7 @@ export class TabClient extends WshClient { } if (data.submit) { - await model.handleSubmit(false); + await model.handleSubmit(); } } } From dc3c2b57dcf3089004d06631bc1e4478d5d78156 Mon Sep 17 00:00:00 2001 From: sawka Date: Wed, 15 Oct 2025 16:38:40 -0700 Subject: [PATCH 07/11] correctly stop streams when creating a new chat --- frontend/app/aipanel/aipanel.tsx | 4 ++-- frontend/app/aipanel/waveai-model.tsx | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/frontend/app/aipanel/aipanel.tsx b/frontend/app/aipanel/aipanel.tsx index 04d75714bc..aca6f17f93 100644 --- a/frontend/app/aipanel/aipanel.tsx +++ b/frontend/app/aipanel/aipanel.tsx @@ -205,7 +205,7 @@ const AIPanelComponentInner = memo(({ className, onClose }: AIPanelProps) => { const telemetryEnabled = jotai.useAtomValue(getSettingsKeyAtom("telemetry:enabled")) ?? false; const isPanelVisible = jotai.useAtomValue(WorkspaceLayoutModel.getInstance().panelVisibleAtom); - const { messages, sendMessage, status, setMessages, error } = useChat({ + const { messages, sendMessage, status, setMessages, error, stop } = useChat({ transport: new DefaultChatTransport({ api: `${getWebServerEndpoint()}/api/post-chat-message`, prepareSendMessagesRequest: (opts) => { @@ -232,7 +232,7 @@ const AIPanelComponentInner = memo(({ className, onClose }: AIPanelProps) => { }, }); - model.registerUseChatData(sendMessage, setMessages, status); + model.registerUseChatData(sendMessage, setMessages, status, stop); // console.log("AICHAT messages", messages); diff --git a/frontend/app/aipanel/waveai-model.tsx b/frontend/app/aipanel/waveai-model.tsx index 3f99e5a7d5..405e0943b1 100644 --- a/frontend/app/aipanel/waveai-model.tsx +++ b/frontend/app/aipanel/waveai-model.tsx @@ -30,6 +30,7 @@ export class WaveAIModel { private useChatSendMessage: UseChatSendMessageType | null = null; private useChatSetMessages: UseChatSetMessagesType | null = null; private useChatStatus: ChatStatus = "ready"; + private useChatStop: (() => void) | null = null; // Used for injecting Wave-specific message data into DefaultChatTransport's prepareSendMessagesRequest realMessage: AIMessage | null = null; @@ -135,6 +136,7 @@ export class WaveAIModel { } clearChat() { + this.useChatStop?.(); this.clearFiles(); this.isChatEmpty = true; const newChatId = crypto.randomUUID(); @@ -165,10 +167,16 @@ export class WaveAIModel { this.scrollToBottomCallback = callback; } - registerUseChatData(sendMessage: UseChatSendMessageType, setMessages: UseChatSetMessagesType, status: ChatStatus) { + registerUseChatData( + sendMessage: UseChatSendMessageType, + setMessages: UseChatSetMessagesType, + status: ChatStatus, + stop: () => void + ) { this.useChatSendMessage = sendMessage; this.useChatSetMessages = setMessages; this.useChatStatus = status; + this.useChatStop = stop; } scrollToBottom() { From b338b49f4111df9253fee12357369d55a5fe4995 Mon Sep 17 00:00:00 2001 From: sawka Date: Wed, 15 Oct 2025 16:50:31 -0700 Subject: [PATCH 08/11] fix mimetypes... just detect images/pdfs... everything else is text. text files get scanned for binary chars. so we can upload dotfiles more effectively instead of dying because we cant figure out the mimetype. --- cmd/wsh/cmd/wshcmd-ai.go | 31 ++++++++++++++++++------------- pkg/util/utilfn/marshal.go | 14 ++++++++++++++ 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-ai.go b/cmd/wsh/cmd/wshcmd-ai.go index e90ef6d50f..53bf4d4efe 100644 --- a/cmd/wsh/cmd/wshcmd-ai.go +++ b/cmd/wsh/cmd/wshcmd-ai.go @@ -7,11 +7,13 @@ import ( "encoding/base64" "fmt" "io" - "mime" + "net/http" "os" "path/filepath" + "strings" "github.com/spf13/cobra" + "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" "github.com/wavetermdev/waveterm/pkg/wshutil" @@ -47,23 +49,16 @@ func init() { aiCmd.Flags().BoolVarP(&aiNewBlockFlag, "new", "n", false, "create a new AI chat instead of using existing") } -func getMimeType(filename string) string { - ext := filepath.Ext(filename) - if ext == "" { - return "text/plain" - } - mimeType := mime.TypeByExtension(ext) - if mimeType == "" { - return "text/plain" - } - return mimeType +func detectMimeType(data []byte) string { + mimeType := http.DetectContentType(data) + return strings.Split(mimeType, ";")[0] } func getMaxFileSize(mimeType string) (int, string) { if mimeType == "application/pdf" { return 5 * 1024 * 1024, "5MB" } - if mimeType[:6] == "image/" { + if strings.HasPrefix(mimeType, "image/") { return 7 * 1024 * 1024, "7MB" } return 200 * 1024, "200KB" @@ -123,7 +118,17 @@ func aiRun(cmd *cobra.Command, args []string) (rtnErr error) { return fmt.Errorf("reading file %s: %w", filePath, err) } fileName = filepath.Base(filePath) - mimeType = getMimeType(filePath) + mimeType = detectMimeType(data) + } + + isPDF := mimeType == "application/pdf" + isImage := strings.HasPrefix(mimeType, "image/") + + if !isPDF && !isImage { + mimeType = "text/plain" + if utilfn.ContainsBinaryData(data) { + return fmt.Errorf("file %s contains binary data and cannot be uploaded as text", fileName) + } } maxSize, sizeStr := getMaxFileSize(mimeType) diff --git a/pkg/util/utilfn/marshal.go b/pkg/util/utilfn/marshal.go index 7789fa9ba7..c284260ae9 100644 --- a/pkg/util/utilfn/marshal.go +++ b/pkg/util/utilfn/marshal.go @@ -201,3 +201,17 @@ func DecodeDataURL(dataURL string) (mimeType string, data []byte, err error) { } return mimeType, []byte(decoded), nil } + + +// ContainsBinaryData checks if the provided data contains binary (non-text) content +func ContainsBinaryData(data []byte) bool { + for _, b := range data { + if b == 0 { + return true + } + if b < 32 && b != 9 && b != 10 && b != 13 { + return true + } + } + return false +} From 3d7097c6a864a3663d03f8692d38ac7b302fbfd5 Mon Sep 17 00:00:00 2001 From: sawka Date: Wed, 15 Oct 2025 16:57:44 -0700 Subject: [PATCH 09/11] use newline to find the end of the start tag instead of ">" to avoid issues with the filename containing ">" chars... --- pkg/aiusechat/openai/openai-convertmessage.go | 4 ++-- pkg/aiusechat/usechat.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/aiusechat/openai/openai-convertmessage.go b/pkg/aiusechat/openai/openai-convertmessage.go index 1065423ab4..8c9559d699 100644 --- a/pkg/aiusechat/openai/openai-convertmessage.go +++ b/pkg/aiusechat/openai/openai-convertmessage.go @@ -480,8 +480,8 @@ func (m *OpenAIChatMessage) ConvertToUIMessage() *uctypes.UIMessage { switch block.Type { case "input_text", "output_text": if strings.HasPrefix(block.Text, "") - if openTagEnd == -1 { + openTagEnd := strings.Index(block.Text, "\n") + if openTagEnd == -1 || block.Text[openTagEnd-1] != '>' { continue } diff --git a/pkg/aiusechat/usechat.go b/pkg/aiusechat/usechat.go index f58619eec3..379eab8fba 100644 --- a/pkg/aiusechat/usechat.go +++ b/pkg/aiusechat/usechat.go @@ -66,7 +66,7 @@ var SystemPromptText_OpenAI = strings.Join([]string{ `Be concise and direct. Prefer determinism over speculation. If a brief clarifying question eliminates guesswork, ask it.`, // Attached text files - `User-attached text files may appear inline as content.`, + `User-attached text files may appear inline as \ncontent\n.`, `If multiple attached files exist, treat each as a separate source file with its own file_name.`, `When the user refers to these files, use their inline content directly; do NOT call any read_text_file or file-access tools to re-read them unless asked.`, From 0fcacc8231e2d4507b1b9c036cc00e7ef80c6d11 Mon Sep 17 00:00:00 2001 From: sawka Date: Wed, 15 Oct 2025 17:05:13 -0700 Subject: [PATCH 10/11] simplify file handling, just upload each file in order in its own RPC call --- cmd/wsh/cmd/wshcmd-ai.go | 55 +++++++++++++++------------------------- 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-ai.go b/cmd/wsh/cmd/wshcmd-ai.go index 53bf4d4efe..b1f4ae3af0 100644 --- a/cmd/wsh/cmd/wshcmd-ai.go +++ b/cmd/wsh/cmd/wshcmd-ai.go @@ -74,8 +74,6 @@ func aiRun(cmd *cobra.Command, args []string) (rtnErr error) { return fmt.Errorf("no files or message provided") } - const maxBatchSize = 7 * 1024 * 1024 - const largeFileThreshold = 1 * 1024 * 1024 const maxFileCount = 15 const rpcTimeout = 30000 @@ -164,42 +162,31 @@ func aiRun(cmd *cobra.Command, args []string) (rtnErr error) { } } - var smallFiles []wshrpc.AIAttachedFile - var smallFilesSize int - for _, file := range allFiles { - if file.Size > largeFileThreshold { - contextData := wshrpc.CommandWaveAIAddContextData{ - Files: []wshrpc.AIAttachedFile{file}, - } - err := wshclient.WaveAIAddContextCommand(RpcClient, contextData, &wshrpc.RpcOpts{ - Route: route, - Timeout: rpcTimeout, - }) - if err != nil { - return fmt.Errorf("adding file %s: %w", file.Name, err) - } - } else { - smallFilesSize += file.Size - if smallFilesSize > maxBatchSize { - return fmt.Errorf("small files total size exceeds maximum batch size of 7MB") - } - smallFiles = append(smallFiles, file) + contextData := wshrpc.CommandWaveAIAddContextData{ + Files: []wshrpc.AIAttachedFile{file}, + } + err := wshclient.WaveAIAddContextCommand(RpcClient, contextData, &wshrpc.RpcOpts{ + Route: route, + Timeout: rpcTimeout, + }) + if err != nil { + return fmt.Errorf("adding file %s: %w", file.Name, err) } } - finalContextData := wshrpc.CommandWaveAIAddContextData{ - Files: smallFiles, - Text: aiMessageFlag, - Submit: aiSubmitFlag, - } - - err := wshclient.WaveAIAddContextCommand(RpcClient, finalContextData, &wshrpc.RpcOpts{ - Route: route, - Timeout: rpcTimeout, - }) - if err != nil { - return fmt.Errorf("adding context: %w", err) + if aiMessageFlag != "" || aiSubmitFlag { + finalContextData := wshrpc.CommandWaveAIAddContextData{ + Text: aiMessageFlag, + Submit: aiSubmitFlag, + } + err := wshclient.WaveAIAddContextCommand(RpcClient, finalContextData, &wshrpc.RpcOpts{ + Route: route, + Timeout: rpcTimeout, + }) + if err != nil { + return fmt.Errorf("adding context: %w", err) + } } return nil From 881d607a5221335ad6b0c372f11b9f8aac60a77b Mon Sep 17 00:00:00 2001 From: sawka Date: Wed, 15 Oct 2025 17:06:43 -0700 Subject: [PATCH 11/11] 8 x's not 6 --- pkg/aiusechat/usechat.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/aiusechat/usechat.go b/pkg/aiusechat/usechat.go index 379eab8fba..cdad22db6e 100644 --- a/pkg/aiusechat/usechat.go +++ b/pkg/aiusechat/usechat.go @@ -66,7 +66,7 @@ var SystemPromptText_OpenAI = strings.Join([]string{ `Be concise and direct. Prefer determinism over speculation. If a brief clarifying question eliminates guesswork, ask it.`, // Attached text files - `User-attached text files may appear inline as \ncontent\n.`, + `User-attached text files may appear inline as \ncontent\n.`, `If multiple attached files exist, treat each as a separate source file with its own file_name.`, `When the user refers to these files, use their inline content directly; do NOT call any read_text_file or file-access tools to re-read them unless asked.`,