feat: Implement web-search and url-fetch tools

Author: huytd

build.sbt

@@ -16,7 +16,9 @@ lazy val root = project
       "io.circe" %% "circe-generic" % "0.14.5",
       "io.circe" %% "circe-parser" % "0.14.5",
       "org.jline" % "jline" % "3.21.0",
-      "com.github.scopt" %% "scopt" % "4.1.0"
+      "com.github.scopt" %% "scopt" % "4.1.0",
+      "com.lihaoyi" %% "requests" % "0.8.0",
+      "com.lihaoyi" %% "upickle" % "3.1.2"
     )
   )
 

src/main/scala/com/supercoder/agents/CoderAgent.scala

@@ -2,7 +2,7 @@ package com.supercoder.agents
 
 import com.openai.models.FunctionDefinition
 import com.supercoder.base.{BaseChatAgent, ToolCallDescription}
-import com.supercoder.tools.{CodeEditTool, CommandExecutionTool, CodeSearchTool, FileReadTool, ProjectStructureTool}
+import com.supercoder.tools.{CodeEditTool, CodeSearchTool, CommandExecutionTool, FileReadTool, ProjectStructureTool, UrlFetchTool, WebSearchTool}
 
 val coderAgentPrompt = s"""
 You are a senior software engineer AI agent. Your task is to help the user with their coding needs.
@@ -14,9 +14,14 @@ You have access to the following tools:
 - ${FileReadTool.functionDefinition.name}: ${FileReadTool.functionDefinition.description}
 - ${CodeEditTool.functionDefinition.name}: ${CodeEditTool.functionDefinition.description}
 - ${CommandExecutionTool.functionDefinition.name}: ${CommandExecutionTool.functionDefinition.description}
+- ${UrlFetchTool.functionDefinition.name}: ${UrlFetchTool.functionDefinition.description}
+- ${WebSearchTool.functionDefinition.name}: ${WebSearchTool.functionDefinition.description}
 
 You can use these tools to help you with the user's request.
 
+When using the web-search tool, make sure you also use the url-fetch tool to read the content of the result URLs if needed.
+
+
 The discussion is about the code of the current project/folder. Always use the relevant tool to learn about the
 project if you are unsure before giving answer.
 """
@@ -29,7 +34,9 @@ class CoderAgent(additionalPrompt: String = "", model: String = "")
     ProjectStructureTool,
     FileReadTool,
     CodeEditTool,
-    CommandExecutionTool
+    CommandExecutionTool,
+    UrlFetchTool,
+    WebSearchTool
   )
 
   override def toolDefinitionList: List[FunctionDefinition] =

src/main/scala/com/supercoder/tools/UrlFetchTool.scala

@@ -0,0 +1,38 @@
+package com.supercoder.tools
+
+import com.openai.models.FunctionDefinition
+import com.supercoder.base.Tool
+import com.supercoder.lib.Console.green
+import io.circe.*
+import io.circe.generic.auto.*
+import io.circe.parser.*
+
+case class UrlFetchToolArguments(url: String)
+
+object UrlFetchTool extends Tool {
+  override val functionDefinition: FunctionDefinition = FunctionDefinition.builder()
+    .name("url-fetch")
+    .description("Fetch content from a specified URL. Arguments: {\"url\": \"<target-url>\"}")
+    .build()
+
+  override def execute(arguments: String): String = {
+    val parsedArguments = decode[UrlFetchToolArguments](arguments)
+    parsedArguments match {
+      case Right(args) =>
+        val url = args.url
+        println(green(s"\uD83D\uDD0D Fetching URL: $url"))
+        try {
+          val response = requests.get(
+            url,
+            connectTimeout = 5000,
+            readTimeout = 10000
+          )
+          response.text()
+        } catch {
+          case e: Exception => s"Error: ${e.getMessage}"
+        }
+      case Left(error) =>
+        s"Error: Invalid arguments - ${error.getMessage}"
+    }
+  }
+}

src/main/scala/com/supercoder/tools/WebSearchTool.scala

@@ -0,0 +1,44 @@
+package com.supercoder.tools
+
+import com.openai.models.FunctionDefinition
+import com.supercoder.base.Tool
+import com.supercoder.lib.Console.green
+import io.circe.*
+import io.circe.generic.auto.*
+import io.circe.parser.*
+
+import java.net.URLEncoder
+
+case class WebSearchToolArguments(query: String, limit: Int)
+
+object WebSearchTool extends Tool {
+  val searxngInstance: String = sys.env.getOrElse("SEARXNG_URL", "")
+
+  override val functionDefinition: FunctionDefinition = FunctionDefinition.builder()
+    .name("web-search")
+    .description("Perform web search using SearxNG. Use this when you need to find information that is not in the codebase or when you need to find a specific library or tool. Arguments: {\"query\": \"<search-query>\", \"limit\": <max-results>}")
+    .build()
+
+  override def execute(arguments: String): String = {
+    if (searxngInstance.isEmpty) {
+      return "Error: SearxNG instance URL is not set. Please set the SEARXNG_URL environment variable."
+    }
+    val parsedArguments = decode[WebSearchToolArguments](arguments)
+    parsedArguments match {
+      case Right(args) =>
+        val query = args.query
+        val encodedQuery = URLEncoder.encode(args.query, "UTF-8")
+        val limit = args.limit
+        println(green(s"🔍 Searching the web for: $query"))
+        try {
+          val searchUrl = s"$searxngInstance/search?q=$encodedQuery&format=json&limit=$limit"
+          val response = requests.get(searchUrl)
+          response.text()
+        } catch {
+          case e: Exception => s"Error: ${e.getMessage}"
+        }
+      case Left(error) =>
+        s"Error: Invalid arguments - ${error.getMessage}"
+    }
+  }
+}