use local llama

.gitignore

@@ -3,3 +3,4 @@ dist
 node_modules
 .vscode-test/
 *.vsix
+/.vs

README.md

@@ -32,6 +32,16 @@ Usage:
 
 For local model you can use that application https://github.com/LucienShui/huggingface-vscode-endpoint-server, and enter URL to that server in extension settings
 
+## Local running (Ollama)
+
+Or use settings for local Ollama, see https://ollama.ai/ (for sample use local ollama in WSL)
+
+    bearertoken = 1 
+    apiurl = http://localhost:11434/api/generate/
+    model = starcoder:7b
+    stream = false | true
+    is_local = true
+
 ## Notes for prompt
 
 Good prompt: 
@@ -60,6 +70,15 @@ This extension contributes the following settings:
 * `starcoderex.bearertoken`: Set bearer token for API https://huggingface.co
 * `starcoderex.apiurl`: Set custom API Url
 
+## Build
+
+cd < StarCodeExPath >
+
+npm install -g @vscode/vsce
+
+vsce package
+
+
 ## Release Notes
 
 ### 1.0.5

package.json

@@ -1,6 +1,6 @@
 {
   "name": "starcoderex",
-  "displayName": "StarCoderEx (AI code generator)",
+  "displayName": "StarCoderEx (AI code generator - local Ollama)",
   "description": "Extension for using alternative GitHub Copilot (StarCoder API) in VSCode",
   "repository": "https://github.com/Lisoveliy/starcoderex",
   "icon": "logo.png",
@@ -60,15 +60,27 @@
         "properties": {
           "starcoderex.countofrequests": {
             "type": "integer",
-            "default": 4
+            "default": 1
           },
           "starcoderex.bearertoken": {
             "type": "string",
-            "default": ""
+            "default": "1"
           },
-          "starcoderex.apiurl":{
+          "starcoderex.apiurl": {
             "type": "string",
-            "default": "https://api-inference.huggingface.co/models/bigcode/starcoder"
+            "default": "http://localhost:11434/api/generate/"
+          },
+          "starcoderex.model": {
+            "type": "string",
+            "default": "starcoder:7b"
+          },
+          "starcoderex.stream": {
+            "type": "string",
+            "default": "false"
+          },
+          "starcoderex.is_local": {
+            "type": "string",
+            "default": "true"
           }
         }
       }

src/request.ts

@@ -3,30 +3,60 @@ import * as vscode from "vscode";
 import updatetoken from "./updatetoken";
 
 export default async (input: string): Promise<string | null> =>{
-	//console.log(`Input: ${input}`);
+	console.log(`Input: ${input}`);
 	let promise: Promise<Response>;
-	try{
-	promise = fetch(vscode.workspace.getConfiguration("starcoderex").get("apiurl") as string,
+	try {
+		const config = vscode.workspace.getConfiguration("starcoderex");
+		const bearerToken = config.get("bearertoken");
+		const model = config.get("model");
+		const isStream = config.get("stream") === "true";
+		const isLocal = config.get("is_local") === "true";
+
+		if (isLocal) {
+			promise = fetch(vscode.workspace.getConfiguration("starcoderex").get("apiurl") as string,
+				{
+					// eslint-disable-next-line @typescript-eslint/naming-convention
+					headers: { "content-type": "application/json" },
+					method: "POST",
+					body: JSON.stringify({
+						model: model,
+						prompt: input,
+						stream: isStream
+					}),
+				});
+		} else {
+			promise = fetch(vscode.workspace.getConfiguration("starcoderex").get("apiurl") as string,
+				{
+					// eslint-disable-next-line @typescript-eslint/naming-convention
+					headers: { authorization: `Bearer ${bearerToken}`, "content-type": "application/json" },
+					method: "POST",
+					body: JSON.stringify({ inputs: input }),
+				});
+		}
+
+		let response = await promise;
+		if(response.status !== 200){
+			if(response.status === 400){
+			vscode.window.showErrorMessage("Bearer invalid!");
+			vscode.workspace.getConfiguration("starcoderex").update("bearertoken", "", vscode.ConfigurationTarget.Global);
+			updatetoken();
+			return null;
+			}else {
+				vscode.window.showWarningMessage("Service turned off right now. Try later!");
+			}
+		}
+
+		let output: string;
+		if (isLocal) {
+			output = ((await response.json()) as LocalResponseModel).response;
+		}
+		else
 		{
-			// eslint-disable-next-line @typescript-eslint/naming-convention
-			headers: { authorization: `Bearer ${vscode.workspace.getConfiguration("starcoderex").get("bearertoken")}`, "content-type": "application/json" },
-			method: "POST",
-			body: JSON.stringify({inputs: input}),
-		});
-	let response = await promise;
-	if(response.status !== 200){
-		if(response.status === 400){
-		vscode.window.showErrorMessage("Bearer invalid!");
-		vscode.workspace.getConfiguration("starcoderex").update("bearertoken", "", vscode.ConfigurationTarget.Global);
-		updatetoken();
-		return null;
-		}else {
-			vscode.window.showWarningMessage("Service turned off right now. Try later!");
+			output = ((await response.json()) as ResponseModel[])[0].generated_text;
 		}
-	}
-	let output = ((await response.json()) as ResponseModel[])[0].generated_text;
-	console.log(`Output: ${output.length}`);
-	return output;
+
+		console.log(`Output: ${output.length}`);
+		return output;
 	}catch(exception: any)
 	{
 		if(exception instanceof FetchError)

src/responsemodel.ts

@@ -1,7 +1,15 @@
-class ResponseModel{
+class LocalResponseModel {
+    // eslint-disable-next-line @typescript-eslint/naming-convention
+    response: string = "";
+    responseModel(response: string) {
+        this.response = response;
+    }
+}
+
+class ResponseModel {
     // eslint-disable-next-line @typescript-eslint/naming-convention
     generated_text: string = "";
-    responseModel(text: string){
+    responseModel(text: string) {
         this.generated_text = text;
     }
 }