Skip to content

Add Pi invocation execution with fixture-based integration tests #832

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
dersam wants to merge 1 commit into from
Closed

Add Pi invocation execution with fixture-based integration tests #832

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions examples/simple_pi_agent.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# typed: true
# frozen_string_literal: true

#: self as Roast::Workflow

config do
agent do
provider :pi
model "sonnet"
append_system_prompt "Always respond concisely in one sentence"
show_prompt!
dump_raw_agent_messages_to "tmp/pi-messages.log"
end
end

execute do
agent { "What is the capital of France?" }
end
6 changes: 6 additions & 0 deletions lib/roast/cogs/agent/providers/pi/pi_invocation.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,15 +55,21 @@ def run!
raise PiAlreadyStartedError if started?

@started = true
start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)

_stdout, stderr, status = CommandRunner.execute(
command_line,
working_directory: @working_directory,
stdin_content: @prompt,
stdout_handler: lambda { |line| handle_stdout(line) },
)

duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round

if status.success?
@completed = true
@result.stats ||= Stats.new
@result.stats.duration_ms = duration_ms
else
@failed = true
@result.success = false
Expand Down
32 changes: 32 additions & 0 deletions test/examples/functional/roast_examples_test.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -642,6 +642,38 @@ class RoastExamplesTest < FunctionalTest
assert_empty logged_stderr
end

test "simple_pi_agent.rb workflow runs successfully" do
use_command_runner_fixture(
"agent_transcripts/pi_simple_response",
expected_args: [
"pi",
"-p",
"--mode",
"json",
"--model",
"sonnet",
"--append-system-prompt",
"Always respond concisely in one sentence",
],
expected_stdin_content: "What is the capital of France?",
)

stdout, stderr = in_sandbox :simple_pi_agent do
Roast::Workflow.from_file("examples/simple_pi_agent.rb", EMPTY_PARAMS)
end

assert_empty stdout
assert_empty stderr

logged_stdout, logged_stderr = original_streams_from_logger_output

assert_includes logged_stdout, "[USER PROMPT] What is the capital of France?"
assert_includes logged_stdout, "[AGENT STATS] Turns:"
assert_includes logged_stdout, "Tokens (claude-sonnet-4-20250514):"
assert_includes logged_stdout, "Session ID: test-session-abc-123"
assert_empty logged_stderr
end

test "working_directory.rb workflow runs successfully" do
stdout, stderr = in_sandbox :working_directory do
Roast::Workflow.from_file("examples/working_directory.rb", EMPTY_PARAMS)
Expand Down
12 changes: 12 additions & 0 deletions test/fixtures/agent_transcripts/pi_simple_response.stdout.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{"type":"session","version":3,"id":"test-session-abc-123","timestamp":"2026-01-15T10:00:00.000Z","cwd":"/tmp/test"}
{"type":"agent_start"}
{"type":"turn_start"}
{"type":"message_start","message":{"role":"user","content":[{"type":"text","text":"What is the capital of France?"}],"timestamp":1737000000000}}
{"type":"message_end","message":{"role":"user","content":[{"type":"text","text":"What is the capital of France?"}],"timestamp":1737000000000}}
{"type":"message_start","message":{"role":"assistant","content":[],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1737000000100}}
{"type":"message_update","assistantMessageEvent":{"type":"text_start","contentIndex":0,"partial":{"role":"assistant","content":[{"type":"text","text":"","index":0}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":1,"cacheRead":0,"cacheWrite":1000,"totalTokens":1006,"cost":{"input":0.000025,"output":0.000025,"cacheRead":0,"cacheWrite":0.00625,"total":0.0063}},"stopReason":"stop","timestamp":1737000000100}},"message":{"role":"assistant","content":[{"type":"text","text":"","index":0}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":1,"cacheRead":0,"cacheWrite":1000,"totalTokens":1006,"cost":{"input":0.000025,"output":0.000025,"cacheRead":0,"cacheWrite":0.00625,"total":0.0063}},"stopReason":"stop","timestamp":1737000000100}}
{"type":"message_update","assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"The capital of France is Paris.","partial":{"role":"assistant","content":[{"type":"text","text":"The capital of France is Paris.","index":0}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":10,"cacheRead":0,"cacheWrite":1000,"totalTokens":1015,"cost":{"input":0.000025,"output":0.00025,"cacheRead":0,"cacheWrite":0.00625,"total":0.006525}},"stopReason":"stop","timestamp":1737000000100}},"message":{"role":"assistant","content":[{"type":"text","text":"The capital of France is Paris.","index":0}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":10,"cacheRead":0,"cacheWrite":1000,"totalTokens":1015,"cost":{"input":0.000025,"output":0.00025,"cacheRead":0,"cacheWrite":0.00625,"total":0.006525}},"stopReason":"stop","timestamp":1737000000100}}
{"type":"message_update","assistantMessageEvent":{"type":"text_end","contentIndex":0,"content":"The capital of France is Paris.","partial":{"role":"assistant","content":[{"type":"text","text":"The capital of France is Paris."}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":10,"cacheRead":0,"cacheWrite":1000,"totalTokens":1015,"cost":{"input":0.000025,"output":0.00025,"cacheRead":0,"cacheWrite":0.00625,"total":0.006525}},"stopReason":"stop","timestamp":1737000000100}},"message":{"role":"assistant","content":[{"type":"text","text":"The capital of France is Paris."}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":10,"cacheRead":0,"cacheWrite":1000,"totalTokens":1015,"cost":{"input":0.000025,"output":0.00025,"cacheRead":0,"cacheWrite":0.00625,"total":0.006525}},"stopReason":"stop","timestamp":1737000000100}}
{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"The capital of France is Paris."}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":10,"cacheRead":0,"cacheWrite":1000,"totalTokens":1015,"cost":{"input":0.000025,"output":0.00025,"cacheRead":0,"cacheWrite":0.00625,"total":0.006525}},"stopReason":"stop","timestamp":1737000000100}}
{"type":"turn_end","message":{"role":"assistant","content":[{"type":"text","text":"The capital of France is Paris."}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":10,"cacheRead":0,"cacheWrite":1000,"totalTokens":1015,"cost":{"input":0.000025,"output":0.00025,"cacheRead":0,"cacheWrite":0.00625,"total":0.006525}},"stopReason":"stop","timestamp":1737000000100},"toolResults":[]}
{"type":"agent_end","messages":[{"role":"user","content":[{"type":"text","text":"What is the capital of France?"}],"timestamp":1737000000000},{"role":"assistant","content":[{"type":"text","text":"The capital of France is Paris."}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":10,"cacheRead":0,"cacheWrite":1000,"totalTokens":1015,"cost":{"input":0.000025,"output":0.00025,"cacheRead":0,"cacheWrite":0.00625,"total":0.006525}},"stopReason":"stop","timestamp":1737000000100}]}
21 changes: 21 additions & 0 deletions test/fixtures/agent_transcripts/pi_tool_use_response.stdout.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{"type":"session","version":3,"id":"test-session-tool-456","timestamp":"2026-01-15T10:05:00.000Z","cwd":"/tmp/test"}
{"type":"agent_start"}
{"type":"turn_start"}
{"type":"message_start","message":{"role":"user","content":[{"type":"text","text":"Read the file /tmp/hello.txt"}],"timestamp":1737000300000}}
{"type":"message_end","message":{"role":"user","content":[{"type":"text","text":"Read the file /tmp/hello.txt"}],"timestamp":1737000300000}}
{"type":"message_start","message":{"role":"assistant","content":[],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1737000300100}}
{"type":"message_update","assistantMessageEvent":{"type":"toolcall_start","contentIndex":0,"partial":{"role":"assistant","content":[{"type":"toolCall","id":"toolu_01ABC123","name":"read","arguments":{}}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":15,"cacheRead":0,"cacheWrite":2000,"totalTokens":2020,"cost":{"input":0.000025,"output":0.000375,"cacheRead":0,"cacheWrite":0.0125,"total":0.0129}},"stopReason":"stop","timestamp":1737000300100}},"message":{"role":"assistant","content":[{"type":"toolCall","id":"toolu_01ABC123","name":"read","arguments":{}}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":15,"cacheRead":0,"cacheWrite":2000,"totalTokens":2020,"cost":{"input":0.000025,"output":0.000375,"cacheRead":0,"cacheWrite":0.0125,"total":0.0129}},"stopReason":"stop","timestamp":1737000300100}}
{"type":"message_update","assistantMessageEvent":{"type":"toolcall_delta","contentIndex":0,"delta":"{\"path\": \"/tmp/hello.txt\"}","partial":{"role":"assistant","content":[{"type":"toolCall","id":"toolu_01ABC123","name":"read","arguments":{"path":"/tmp/hello.txt"}}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":20,"cacheRead":0,"cacheWrite":2000,"totalTokens":2025,"cost":{"input":0.000025,"output":0.0005,"cacheRead":0,"cacheWrite":0.0125,"total":0.013025}},"stopReason":"stop","timestamp":1737000300100}},"message":{"role":"assistant","content":[{"type":"toolCall","id":"toolu_01ABC123","name":"read","arguments":{"path":"/tmp/hello.txt"}}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":20,"cacheRead":0,"cacheWrite":2000,"totalTokens":2025,"cost":{"input":0.000025,"output":0.0005,"cacheRead":0,"cacheWrite":0.0125,"total":0.013025}},"stopReason":"stop","timestamp":1737000300100}}
{"type":"message_update","assistantMessageEvent":{"type":"toolcall_end","contentIndex":0,"toolCall":{"type":"toolCall","id":"toolu_01ABC123","name":"read","arguments":{"path":"/tmp/hello.txt"}},"partial":{"role":"assistant","content":[{"type":"toolCall","id":"toolu_01ABC123","name":"read","arguments":{"path":"/tmp/hello.txt"}}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":20,"cacheRead":0,"cacheWrite":2000,"totalTokens":2025,"cost":{"input":0.000025,"output":0.0005,"cacheRead":0,"cacheWrite":0.0125,"total":0.013025}},"stopReason":"stop","timestamp":1737000300100}},"message":{"role":"assistant","content":[{"type":"toolCall","id":"toolu_01ABC123","name":"read","arguments":{"path":"/tmp/hello.txt"}}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":20,"cacheRead":0,"cacheWrite":2000,"totalTokens":2025,"cost":{"input":0.000025,"output":0.0005,"cacheRead":0,"cacheWrite":0.0125,"total":0.013025}},"stopReason":"stop","timestamp":1737000300100}}
{"type":"message_end","message":{"role":"assistant","content":[{"type":"toolCall","id":"toolu_01ABC123","name":"read","arguments":{"path":"/tmp/hello.txt"}}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":20,"cacheRead":0,"cacheWrite":2000,"totalTokens":2025,"cost":{"input":0.000025,"output":0.0005,"cacheRead":0,"cacheWrite":0.0125,"total":0.013025}},"stopReason":"stop","timestamp":1737000300100}}
{"type":"turn_end","message":{"role":"assistant","content":[{"type":"toolCall","id":"toolu_01ABC123","name":"read","arguments":{"path":"/tmp/hello.txt"}}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":20,"cacheRead":0,"cacheWrite":2000,"totalTokens":2025,"cost":{"input":0.000025,"output":0.0005,"cacheRead":0,"cacheWrite":0.0125,"total":0.013025}},"stopReason":"stop","timestamp":1737000300100},"toolResults":[]}
{"type":"tool_execution_start"}
{"type":"tool_execution_end"}
{"type":"turn_start"}
{"type":"message_start","message":{"role":"assistant","content":[],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"totalTokens":0,"cost":{"input":0,"output":0,"cacheRead":0,"cacheWrite":0,"total":0}},"stopReason":"stop","timestamp":1737000300500}}
{"type":"message_update","assistantMessageEvent":{"type":"text_start","contentIndex":0,"partial":{"role":"assistant","content":[{"type":"text","text":"","index":0}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":25,"output":1,"cacheRead":2000,"cacheWrite":500,"totalTokens":2526,"cost":{"input":0.000125,"output":0.000025,"cacheRead":0.0025,"cacheWrite":0.003125,"total":0.005775}},"stopReason":"stop","timestamp":1737000300500}},"message":{"role":"assistant","content":[{"type":"text","text":"","index":0}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":25,"output":1,"cacheRead":2000,"cacheWrite":500,"totalTokens":2526,"cost":{"input":0.000125,"output":0.000025,"cacheRead":0.0025,"cacheWrite":0.003125,"total":0.005775}},"stopReason":"stop","timestamp":1737000300500}}
{"type":"message_update","assistantMessageEvent":{"type":"text_delta","contentIndex":0,"delta":"The file contains: Hello, World!","partial":{"role":"assistant","content":[{"type":"text","text":"The file contains: Hello, World!","index":0}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":25,"output":12,"cacheRead":2000,"cacheWrite":500,"totalTokens":2537,"cost":{"input":0.000125,"output":0.0003,"cacheRead":0.0025,"cacheWrite":0.003125,"total":0.00605}},"stopReason":"stop","timestamp":1737000300500}},"message":{"role":"assistant","content":[{"type":"text","text":"The file contains: Hello, World!","index":0}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":25,"output":12,"cacheRead":2000,"cacheWrite":500,"totalTokens":2537,"cost":{"input":0.000125,"output":0.0003,"cacheRead":0.0025,"cacheWrite":0.003125,"total":0.00605}},"stopReason":"stop","timestamp":1737000300500}}
{"type":"message_update","assistantMessageEvent":{"type":"text_end","contentIndex":0,"content":"The file contains: Hello, World!","partial":{"role":"assistant","content":[{"type":"text","text":"The file contains: Hello, World!"}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":25,"output":12,"cacheRead":2000,"cacheWrite":500,"totalTokens":2537,"cost":{"input":0.000125,"output":0.0003,"cacheRead":0.0025,"cacheWrite":0.003125,"total":0.00605}},"stopReason":"stop","timestamp":1737000300500}},"message":{"role":"assistant","content":[{"type":"text","text":"The file contains: Hello, World!"}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":25,"output":12,"cacheRead":2000,"cacheWrite":500,"totalTokens":2537,"cost":{"input":0.000125,"output":0.0003,"cacheRead":0.0025,"cacheWrite":0.003125,"total":0.00605}},"stopReason":"stop","timestamp":1737000300500}}
{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"The file contains: Hello, World!"}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":25,"output":12,"cacheRead":2000,"cacheWrite":500,"totalTokens":2537,"cost":{"input":0.000125,"output":0.0003,"cacheRead":0.0025,"cacheWrite":0.003125,"total":0.00605}},"stopReason":"stop","timestamp":1737000300500}}
{"type":"turn_end","message":{"role":"assistant","content":[{"type":"text","text":"The file contains: Hello, World!"}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":25,"output":12,"cacheRead":2000,"cacheWrite":500,"totalTokens":2537,"cost":{"input":0.000125,"output":0.0003,"cacheRead":0.0025,"cacheWrite":0.003125,"total":0.00605}},"stopReason":"stop","timestamp":1737000300500},"toolResults":[]}
{"type":"agent_end","messages":[{"role":"user","content":[{"type":"text","text":"Read the file /tmp/hello.txt"}],"timestamp":1737000300000},{"role":"assistant","content":[{"type":"toolCall","id":"toolu_01ABC123","name":"read","arguments":{"path":"/tmp/hello.txt"}}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":5,"output":20,"cacheRead":0,"cacheWrite":2000,"totalTokens":2025,"cost":{"input":0.000025,"output":0.0005,"cacheRead":0,"cacheWrite":0.0125,"total":0.013025}},"stopReason":"stop","timestamp":1737000300100},{"role":"assistant","content":[{"type":"text","text":"The file contains: Hello, World!"}],"api":"anthropic-messages","provider":"anthropic-1m","model":"claude-sonnet-4-20250514","usage":{"input":25,"output":12,"cacheRead":2000,"cacheWrite":500,"totalTokens":2537,"cost":{"input":0.000125,"output":0.0003,"cacheRead":0.0025,"cacheWrite":0.003125,"total":0.00605}},"stopReason":"stop","timestamp":1737000300500}]}
Loading
Loading