diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ea3d548..6ad25f9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Rainforest CLI Changelog +## 3.8.0 - 2026-02-10 +- Add `generate` command for AI test generation + - (478f5968aa4291c090852b073202ae1f68d60ab8, @sebaherrera07) + ## 3.7.0 - 2026-02-09 - Update Go from 1.22.3 to 1.25.7 - (e0dc14e8c0c327daf6884a0e69e93482d156adae, @sebaherrera07) diff --git a/README.md b/README.md index aaa86239..dda5df2e 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,46 @@ Download specific tests based on their id on the Rainforest dashboard rainforest download 33445 11232 1337 ``` +#### Generating Tests with AI + +Generate a new test using AI based on a natural language prompt. + +```bash +rainforest generate "Log in with valid credentials and verify the dashboard loads" --start-uri /login +``` + +You can also use a full URL instead of a start URI. + +```bash +rainforest generate "Add an item to the shopping cart" --url https://example.com/shop +``` + +Specify a custom title for the generated test. + +```bash +rainforest generate "Complete the checkout process" --start-uri /cart --title "Checkout Flow Test" +``` + +Specify the platform for AI test generation. Note: AI test generation only supports one platform at a time, and the CLI accepts a single value for `--platform`. Commonly used platforms for AI generation include: `windows10_chrome`, `windows11_chrome`, and `windows11_chrome_fhd`, but other platforms may be available; unsupported values will be rejected by the Rainforest API. + +```bash +rainforest generate "Test responsive layout" --start-uri / --platform windows11_chrome_fhd +``` + +Provide credentials information for the AI to use during test generation. This is a free-form string passed to the AI model. + +```bash +rainforest generate "Log in as admin user" --start-uri /admin --credentials "username: admin, password: secret123" +``` + +Alternatively, use a login snippet for authentication. + +```bash +rainforest generate "Verify dashboard features" --start-uri /dashboard --login-snippet-id 54321 +``` + +Note: `--credentials` and `--login-snippet-id` are mutually exclusive. + #### Running Local RFML Tests Only If you want to run a local set of RFML files (for instance in a CI environment), use the `run -f` option: diff --git a/rainforest-cli.go b/rainforest-cli.go index 06f463f8..9e17e7fc 100644 --- a/rainforest-cli.go +++ b/rainforest-cli.go @@ -13,7 +13,7 @@ import ( const ( // Version of the app in SemVer - version = "3.7.0" + version = "3.8.0" // This is the default spec folder for RFML tests defaultSpecFolder = "./spec/rainforest" ) @@ -359,6 +359,49 @@ func main() { return newRFMLTest(c) }, }, + { + Name: "generate", + Aliases: []string{"gen", "ai"}, + Usage: "Generate a new test using AI", + OnUsageError: onCommandUsageErrorHandler("generate"), + ArgsUsage: "[prompt]", + Description: "Generate a new Rainforest test using AI based on a natural language prompt. " + + "The prompt should describe what the test should do. " + + "Example: rainforest generate \"Log in with valid credentials and verify the dashboard loads\"", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "title", + Usage: "Custom title for the generated test.", + }, + cli.StringFlag{ + Name: "start-uri", + Usage: "The starting `URI` path for the test (e.g., /login).", + }, + cli.StringFlag{ + Name: "url", + Usage: "The full starting `URL` for the test (alternative to --start-uri).", + }, + cli.StringFlag{ + Name: "platform", + Usage: "Specify the `PLATFORM` to use for AI generation (e.g., windows10_chrome, windows11_chrome).", + }, + cli.IntFlag{ + Name: "environment-id", + Usage: "The `ENVIRONMENT-ID` to use for the test.", + }, + cli.StringFlag{ + Name: "credentials", + Usage: "Free-form credentials information to pass to the AI for test generation (e.g., \"username: admin, password: secret123\"). This is an opaque string passed to the AI model. Mutually exclusive with --login-snippet-id.", + }, + cli.IntFlag{ + Name: "login-snippet-id", + Usage: "The `ID` of a snippet to use for login steps. Mutually exclusive with --credentials.", + }, + }, + Action: func(c *cli.Context) error { + return generateAITest(c, api) + }, + }, { Name: "validate", Usage: "Validate your RFML tests", diff --git a/rainforest/tests.go b/rainforest/tests.go index bc4b5ec8..d30090f3 100644 --- a/rainforest/tests.go +++ b/rainforest/tests.go @@ -533,3 +533,72 @@ func (c *Client) UpdateTest(test *RFTest, branchID int) error { } return nil } + +// AITestRequest represents the parameters needed to create a test using AI generation +type AITestRequest struct { + Title string `json:"title,omitempty"` + Type string `json:"type"` + StartURI string `json:"start_uri,omitempty"` + FullURL string `json:"full_url,omitempty"` + Prompt string `json:"prompt"` + Browsers []string `json:"browsers,omitempty"` + EnvironmentID int `json:"environment_id,omitempty"` + PromptCredentials string `json:"prompt_credentials,omitempty"` + LoginSnippetID int `json:"login_snippet_id,omitempty"` +} + +// AITestResponse represents the response from the AI test generation API +type AITestResponse struct { + TestID int `json:"id"` + RFMLID string `json:"rfml_id"` + Title string `json:"title"` + State string `json:"state"` +} + +// CreateTestWithAI creates a new test using AI generation +func (c *Client) CreateTestWithAI(request *AITestRequest) (*AITestResponse, error) { + // AI test generation only supports one browser at a time + if len(request.Browsers) > 1 { + return nil, errors.New("AI test generation only supports one browser at a time") + } + + // Ensure type is "test" + if request.Type != "test" { + return nil, errors.New("Type must be 'test' for AI test generation") + } + + // Ensure prompt is provided + if request.Prompt == "" { + return nil, errors.New("Prompt is required for AI test generation") + } + + // Ensure at least one of StartURI or FullURL is provided + if request.StartURI == "" && request.FullURL == "" { + return nil, errors.New("Either StartURI or FullURL must be provided for AI test generation") + } + + // Ensure StartURI and FullURL are mutually exclusive + if request.StartURI != "" && request.FullURL != "" { + return nil, errors.New("Only one of StartURI or FullURL may be provided for AI test generation") + } + + // Validate mutually exclusive login parameters + if request.PromptCredentials != "" && request.LoginSnippetID > 0 { + return nil, errors.New("Cannot specify both prompt_credentials and login_snippet_id. Choose one login method") + } + + // Prepare request + req, err := c.NewRequest("POST", "tests", request) + if err != nil { + return nil, err + } + + // Send request and process response + var response AITestResponse + _, err = c.Do(req, &response) + if err != nil { + return nil, err + } + + return &response, nil +} diff --git a/rainforest/tests_test.go b/rainforest/tests_test.go index 6ccba2d1..86725483 100644 --- a/rainforest/tests_test.go +++ b/rainforest/tests_test.go @@ -387,6 +387,111 @@ func TestHasUploadableFiles(t *testing.T) { } } +func TestCreateTestWithAI(t *testing.T) { + setup() + defer cleanup() + + mux.HandleFunc("/tests", func(w http.ResponseWriter, r *http.Request) { + if r.Method != "POST" { + t.Errorf("Expected POST, got %v", r.Method) + return + } + + body, _ := ioutil.ReadAll(r.Body) + + // Verify request serialization + var req AITestRequest + if err := json.Unmarshal(body, &req); err != nil { + t.Errorf("Failed to unmarshal request: %v", err) + return + } + if req.Title != "Test login flow" || req.Prompt != "Log in with valid credentials" || req.StartURI != "/login" { + t.Errorf("Unexpected request fields: %+v", req) + } + + json.NewEncoder(w).Encode(AITestResponse{TestID: 12345, RFMLID: "ai-test", Title: req.Title, State: "enabled"}) + }) + + response, err := client.CreateTestWithAI(&AITestRequest{ + Title: "Test login flow", Type: "test", StartURI: "/login", Prompt: "Log in with valid credentials", + Browsers: []string{"windows11_chrome"}, + }) + + if err != nil || response.TestID != 12345 || response.RFMLID != "ai-test" { + t.Errorf("Unexpected result: %+v, err: %v", response, err) + } +} + +func TestCreateTestWithAIValidation(t *testing.T) { + setup() + defer cleanup() + + validBrowser := []string{"windows11_chrome"} + + testCases := []struct { + name string + request *AITestRequest + expectedErr string + }{ + { + name: "multiple browsers", + request: &AITestRequest{ + Title: "Test", Type: "test", StartURI: "/", Prompt: "Test prompt", + Browsers: []string{"windows11_chrome", "windows10_chrome"}, + }, + expectedErr: "only supports one browser", + }, + { + name: "invalid type", + request: &AITestRequest{ + Title: "Test", Type: "snippet", StartURI: "/", Prompt: "Test prompt", Browsers: validBrowser, + }, + expectedErr: "Type must be 'test'", + }, + { + name: "missing prompt", + request: &AITestRequest{ + Title: "Test", Type: "test", StartURI: "/", Prompt: "", Browsers: validBrowser, + }, + expectedErr: "Prompt is required", + }, + { + name: "mutually exclusive credentials", + request: &AITestRequest{ + Title: "Test", Type: "test", StartURI: "/", Prompt: "Test prompt", + PromptCredentials: "user / pass", LoginSnippetID: 12345, Browsers: validBrowser, + }, + expectedErr: "Cannot specify both", + }, + { + name: "missing both StartURI and FullURL", + request: &AITestRequest{ + Title: "Test", Type: "test", Prompt: "Test prompt", Browsers: validBrowser, + }, + expectedErr: "Either StartURI or FullURL must be provided", + }, + { + name: "mutually exclusive StartURI and FullURL", + request: &AITestRequest{ + Title: "Test", Type: "test", StartURI: "/", FullURL: "https://example.com", + Prompt: "Test prompt", Browsers: validBrowser, + }, + expectedErr: "Only one of StartURI or FullURL", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := client.CreateTestWithAI(tc.request) + if err == nil { + t.Errorf("Expected error for %s", tc.name) + } else if !strings.Contains(err.Error(), tc.expectedErr) { + t.Errorf("Expected error containing %q, got: %v", tc.expectedErr, err.Error()) + } + }) + } +} + func TestUpdateTest(t *testing.T) { // Test just the required attributes rfTest := RFTest{ diff --git a/tests.go b/tests.go index e90693e7..9f8dea4f 100644 --- a/tests.go +++ b/tests.go @@ -306,6 +306,96 @@ func newRFMLTest(c cliContext) error { return nil } +// parseAndValidateBrowser parses the platform flag for AI test generation. +// Returns the browsers array with a single browser name, or nil if not specified. +func parseAndValidateBrowser(c cliContext) ([]string, error) { + platform := c.String("platform") + + if platform == "" { + return nil, nil + } + + // Validate that the platform name only contains valid characters + validPlatform := regexp.MustCompile(`^[a-z0-9_]+$`) + if !validPlatform.MatchString(platform) { + return nil, cli.NewExitError("Invalid platform name. Platform must only contain lowercase letters, numbers, and underscores", 1) + } + + return []string{platform}, nil +} + +// generateAITest generates a new test using AI based on a prompt +func generateAITest(c cliContext, api rfAPI) error { + prompt := c.Args().First() + if prompt == "" { + return cli.NewExitError("Please provide a prompt describing the test you want to generate", 1) + } + + title := c.String("title") + + startURI := c.String("start-uri") + fullURL := c.String("url") + + // Require either start_uri or full_url, but not both + if startURI == "" && fullURL == "" { + return cli.NewExitError("Please provide either --start-uri or --url for the test", 1) + } + if startURI != "" && fullURL != "" { + return cli.NewExitError("Please provide either --start-uri or --url, not both", 1) + } + + // Parse and validate browser + browsers, err := parseAndValidateBrowser(c) + if err != nil { + return err + } + + request := &rainforest.AITestRequest{ + Title: title, + Type: "test", + Prompt: prompt, + Browsers: browsers, + } + + if startURI != "" { + request.StartURI = startURI + } + if fullURL != "" { + request.FullURL = fullURL + } + + if envID := c.Int("environment-id"); envID > 0 { + request.EnvironmentID = envID + } + + creds := c.String("credentials") + loginSnippetID := c.Int("login-snippet-id") + + // Validate mutually exclusive login parameters + if creds != "" && loginSnippetID > 0 { + return cli.NewExitError("Cannot specify both --credentials and --login-snippet-id. Choose one login method", 1) + } + + if creds != "" { + request.PromptCredentials = creds + } + if loginSnippetID > 0 { + request.LoginSnippetID = loginSnippetID + } + + log.Printf("Generating test with AI...") + + response, err := api.CreateTestWithAI(request) + if err != nil { + return cli.NewExitError(err.Error(), 1) + } + + log.Printf("Test #%d created: %s", response.TestID, response.Title) + log.Printf("Note: The test is still being generated by AI and may take a few moments to complete.") + + return nil +} + func deleteRFML(c cliContext) error { filePath := c.Args().First() if !strings.Contains(filePath, ".rfml") { @@ -593,6 +683,7 @@ type rfAPI interface { UpdateTest(*rainforest.RFTest, int) error ParseEmbeddedFiles(*rainforest.RFTest) error ClientToken() string + CreateTestWithAI(*rainforest.AITestRequest) (*rainforest.AITestResponse, error) branchAPI } @@ -681,31 +772,9 @@ func downloadTests(c cliContext, client rfAPI) error { case err = <-errorsChan: return cli.NewExitError(err.Error(), 1) case test := <-testChan: - err = test.PrepareToWriteAsRFML(*testIDCollection, c.Bool("flatten-steps")) - if err != nil { - return cli.NewExitError(err.Error(), 1) - } - - paddedTestID := fmt.Sprintf("%010d", test.TestID) - sanitizedTitle := sanitizeTestTitle(test.Title) - fileName := fmt.Sprintf("%v_%v.rfml", paddedTestID, sanitizedTitle) - filePath := filepath.Join(absTestDirectory, fileName) - - var file *os.File - file, err = os.Create(filePath) - if err != nil { - return cli.NewExitError(err.Error(), 1) - } - - writer := rainforest.NewRFMLWriter(file) - err = writer.WriteRFMLTest(test) - - file.Close() - if err != nil { + if err := downloadTestAsRFML(test, *testIDCollection, absTestDirectory, c.Bool("flatten-steps")); err != nil { return cli.NewExitError(err.Error(), 1) } - - log.Printf("Downloaded test to %v", filePath) } } @@ -727,6 +796,33 @@ func downloadRFTestWorker(testInfoChan chan RFTestInfo, errorsChan chan error, t Helper Functions */ +// downloadTestAsRFML downloads a test, prepares it as RFML, and writes it to a file +func downloadTestAsRFML(test *rainforest.RFTest, testIDCollection rainforest.TestIDCollection, absTestDirectory string, flattenSteps bool) error { + err := test.PrepareToWriteAsRFML(testIDCollection, flattenSteps) + if err != nil { + return err + } + + paddedTestID := fmt.Sprintf("%010d", test.TestID) + sanitizedTitle := sanitizeTestTitle(test.Title) + fileName := fmt.Sprintf("%v_%v.rfml", paddedTestID, sanitizedTitle) + filePath := filepath.Join(absTestDirectory, fileName) + + file, err := os.Create(filePath) + if err != nil { + return err + } + defer file.Close() + + writer := rainforest.NewRFMLWriter(file) + if err := writer.WriteRFMLTest(test); err != nil { + return err + } + + log.Printf("Downloaded test to %v", filePath) + return nil +} + func prepareTestDirectory(testDir string) (string, error) { absTestDirectory, err := filepath.Abs(testDir) if err != nil { diff --git a/tests_test.go b/tests_test.go index 8cfdb695..7e4108cb 100644 --- a/tests_test.go +++ b/tests_test.go @@ -233,10 +233,11 @@ func TestNewRFMLTest(t *testing.T) { } type testRfAPI struct { - testIDs []rainforest.TestIDPair - tests []rainforest.RFTest - handleCreateTest func(*rainforest.RFTest) - handleUpdateTest func(*rainforest.RFTest, int) + testIDs []rainforest.TestIDPair + tests []rainforest.RFTest + handleCreateTest func(*rainforest.RFTest) + handleUpdateTest func(*rainforest.RFTest, int) + handleCreateTestAI func(*rainforest.AITestRequest) (*rainforest.AITestResponse, error) testBranchAPI } @@ -276,6 +277,18 @@ func (t *testRfAPI) ParseEmbeddedFiles(_ *rainforest.RFTest) error { return errStub } +func (t *testRfAPI) CreateTestWithAI(request *rainforest.AITestRequest) (*rainforest.AITestResponse, error) { + if t.handleCreateTestAI != nil { + return t.handleCreateTestAI(request) + } + return &rainforest.AITestResponse{ + TestID: 12345, + RFMLID: "ai-generated-test", + Title: request.Title, + State: "enabled", + }, nil +} + func createTestFolder(testFolderPath string) error { absTestFolderPath, err := filepath.Abs(testFolderPath) if err != nil { @@ -1234,3 +1247,125 @@ func setupTestRFMLDir() string { return dir } + +func TestGenerateAITest(t *testing.T) { + testAPI := new(testRfAPI) + var capturedRequest *rainforest.AITestRequest + testAPI.handleCreateTestAI = func(req *rainforest.AITestRequest) (*rainforest.AITestResponse, error) { + capturedRequest = req + return &rainforest.AITestResponse{TestID: 12345, RFMLID: "ai-test", Title: req.Title, State: "enabled"}, nil + } + + t.Run("validation errors", func(t *testing.T) { + validationTests := []struct { + name string + args cli.Args + mappings map[string]interface{} + expectedErr string + }{ + {"missing prompt", cli.Args{}, map[string]interface{}{}, "provide a prompt"}, + {"missing start-uri and url", cli.Args{"Test"}, map[string]interface{}{}, "either --start-uri or --url"}, + {"both start-uri and url", cli.Args{"Test"}, map[string]interface{}{"start-uri": "/", "url": "https://ex.com"}, "either --start-uri or --url"}, + {"invalid platform characters", cli.Args{"Test"}, map[string]interface{}{"start-uri": "/", "platform": "Windows Chrome!"}, "Invalid platform name"}, + {"both auth methods", cli.Args{"Test"}, map[string]interface{}{"start-uri": "/", "credentials": "u/p", "login-snippet-id": 123}, "Cannot specify both"}, + } + + for _, tt := range validationTests { + t.Run(tt.name, func(t *testing.T) { + ctx := newFakeContext(tt.mappings, tt.args) + err := generateAITest(ctx, testAPI) + if err == nil || !strings.Contains(err.Error(), tt.expectedErr) { + t.Errorf("Expected error containing %q, got: %v", tt.expectedErr, err) + } + }) + } + }) + + t.Run("successful generation", func(t *testing.T) { + successTests := []struct { + name string + mappings map[string]interface{} + validate func(*testing.T, *rainforest.AITestRequest) + }{ + { + name: "with start-uri", + mappings: map[string]interface{}{"start-uri": "/login", "title": "Login"}, + validate: func(t *testing.T, req *rainforest.AITestRequest) { + if req.StartURI != "/login" || req.Title != "Login" { + t.Errorf("Expected start_uri=/login, title=Login, got: %+v", req) + } + }, + }, + { + name: "with full URL", + mappings: map[string]interface{}{"url": "https://example.com"}, + validate: func(t *testing.T, req *rainforest.AITestRequest) { + if req.FullURL != "https://example.com" { + t.Errorf("Expected full_url=https://example.com, got: %v", req.FullURL) + } + }, + }, + { + name: "with custom platform", + mappings: map[string]interface{}{"start-uri": "/", "platform": "windows10_chrome"}, + validate: func(t *testing.T, req *rainforest.AITestRequest) { + if len(req.Browsers) != 1 || req.Browsers[0] != "windows10_chrome" { + t.Errorf("Expected browser=[windows10_chrome], got: %v", req.Browsers) + } + }, + }, + { + name: "without platform (backend defaults)", + mappings: map[string]interface{}{"start-uri": "/"}, + validate: func(t *testing.T, req *rainforest.AITestRequest) { + if len(req.Browsers) != 0 { + t.Errorf("Expected no browsers when platform not specified, got: %v", req.Browsers) + } + }, + }, + { + name: "with credentials", + mappings: map[string]interface{}{"start-uri": "/", "credentials": "user/pass", "environment-id": 999}, + validate: func(t *testing.T, req *rainforest.AITestRequest) { + if req.PromptCredentials != "user/pass" || req.EnvironmentID != 999 { + t.Errorf("Expected credentials and env_id set, got: %+v", req) + } + }, + }, + { + name: "with login snippet", + mappings: map[string]interface{}{"start-uri": "/", "login-snippet-id": 54321}, + validate: func(t *testing.T, req *rainforest.AITestRequest) { + if req.LoginSnippetID != 54321 { + t.Errorf("Expected login_snippet_id=54321, got: %v", req.LoginSnippetID) + } + }, + }, + } + + for _, tt := range successTests { + t.Run(tt.name, func(t *testing.T) { + capturedRequest = nil + ctx := newFakeContext(tt.mappings, cli.Args{"Test prompt"}) + if err := generateAITest(ctx, testAPI); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if capturedRequest == nil { + t.Fatal("Request not captured") + } + tt.validate(t, capturedRequest) + }) + } + }) + + t.Run("empty title passed to backend", func(t *testing.T) { + ctx := newFakeContext(map[string]interface{}{"start-uri": "/"}, cli.Args{"Some prompt"}) + capturedRequest = nil + if err := generateAITest(ctx, testAPI); err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if capturedRequest.Title != "" { + t.Errorf("Expected empty title when not specified, got: %v", capturedRequest.Title) + } + }) +}