From 4527e539f900c719d27b418851888c71efd19000 Mon Sep 17 00:00:00 2001 From: Zippo00 Date: Mon, 13 Apr 2026 19:01:36 +0800 Subject: [PATCH 1/2] readme updt; tweakcli flags --- README.md | 102 ++++++++++++++++++++++++++---------------------- avise/cli.py | 13 ++++-- avise/engine.py | 9 ++++- 3 files changed, 73 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 8534fed..2b874ed 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -![AVISE logo](/docs/assets/avise_logo.png) +![](/docs/assets/avise_logo.png) # AVISE - AI Vulnerability Identification & Security Evaluation @@ -15,52 +15,76 @@ A framework for identifying vulnerabilities in and evaluating the security of AI ### Prerequisites - Python 3.10+ -- Docker (for running models backend) -- pip +- Docker (For Running models locally with Ollama) -### 1. Clone the Repository +### 1. Install AVISE +Install with pip: ```bash -git clone https://github.com/ouspg/AVISE.git -cd AVISE +pip install avise ``` -### 2. Set Up Python Environment - +Install with uv: ```bash -# Create virtual environment -python -m venv venv +uv install avise +``` -source venv/bin/activate # Or venv/Scripts/Activate on Windows +### 2. Run a model -# Install dependencies -pip install -r requirements.txt -``` +You can use AVISE to evaluate any model accessible via an API by configuring a Connector. In this Quickstart, we will +assume using the Ollama Docker container for running a language model. If you wish to evaluate models deployed in other ways, see +the [Full Documentations](https://avise.readthedocs.io) and available template connector configuration files at `AVISE/avise/configs/connector/languagemodel/` dir of this repository. -### 3. Set Up by using Ollama Backend with Docker +#### Running a language model locally with Docker & Ollama + +- Clone this repository to your local machine with: -**GPU Version:** ```bash -docker-compose -f docker/ollama/docker-compose.yml up -d +git clone https://github.com/ouspg/AVISE.git ``` -**CPU-only Version:** +- Create the Ollama Docker container + - for **GPU** accelerated inference with: + ```bash + docker compose -f AVISE/docker/ollama/docker-compose.yml up -d + ``` + - or for **CPU** inference with: + ```bash + docker compose -f AVISE/docker/ollama/docker-compose-cpu.yml up -d + ``` + +- Pull an Ollama model to evaluate into the container with: + ```bash + docker exec -it avise-ollama ollama pull + ``` + +### 3. Evaluate the model with a Security Evaluation Test (SET) + +#### Basic usage + ```bash -docker-compose -f docker/ollama/docker-compose-cpu.yml up -d +avise --SET --connectorconf [options] ``` -### 4. Pull Models +For example, you can run the `prompt_injection` SET on the model pulled to the Ollama Docker container with: -After Ollama is running, pull the models you want to test: +```bash +avise --SET prompt_injection --connectorconf ollama_lm --target +``` +To list the available SETs, run the command: ```bash -# Pull models for testing and for evaluation -docker exec -it avise-ollama ollama pull +avise --SET-list ``` -### 5. Configure Connectors -Edit `avise/configs/connector/languagemodel/ollama.json`: +## Advanced usage + +### Configuring Connectors + +You can create your own connector configuration files, or if you cloned the AVISE repository, you can modify the existing connector configuration files in `AVISE/avise/configs/connector/languagemodel/`. + +For example, you can edit the default Ollama Connector configuration file `AVISE/avise/configs/connector/languagemodel/ollama.json`, and insert the name of an Ollama model you have pulled to be used as a target by default: ```json { @@ -73,27 +97,10 @@ Edit `avise/configs/connector/languagemodel/ollama.json`: } } ``` - -## Usage - -### Basic usage - -```bash -python -m avise --SET --connectorconf [options] -``` - -For example, you can run the `prompt_injection` Security Evaluation Test on a target model running locally via Ollama with: - -```bash -python -m avise --SET prompt_injection --connectorconf ollama_lm -``` - -### Advanced usage - If you want to use custom configuration files for SETs and/or Connectors, you can do so by giving the paths to the configuration files with `--SETconf` and `--connectorconf` arguments: ```bash -python -m avise --SET prompt_injection --SETconf avise/configs/SET/languagemodel/single_turn/prompt_injection_mini.json --connectorconf avise/configs/connector/languagemodel/ollama.json +avise --SET prompt_injection --SETconf AVISE/avise/configs/SET/languagemodel/single_turn/prompt_injection_mini.json --connectorconf AVISE/avise/configs/connector/languagemodel/ollama.json ``` ### Required Arguments @@ -109,11 +116,14 @@ python -m avise --SET prompt_injection --SETconf avise/configs/SET/languagemodel | Argument | Description | |----------|-------------| | `--SETconf` | Path to SET configuration JSON file. If not given, uses preconfigured paths for SET config JSON files. | +| `--target`, `-t` | Name of the target model/system to evaluate. Overrides target name from connector configuration file. | | `--format`, `-f` | Report format: `json`, `html`, `md` | | `--runs`, `-r` | How many times each SET is executed | | `--output` | Custom output file path | -| `--reports-dir` | Base directory for reports (default: `reports/`) | -| `--SET_list` | List available Security Evaluation Tests | -| `--connector_list` | List available Connectors | +| `--reports-dir` | Base directory for reports (default: `avise-reports/`) | +| `--SET-list` | List available Security Evaluation Tests | +| `--connector-list` | List available Connectors | | `--verbose`, `-v` | Enable verbose logging | | `--version`, `-V` | Print version | + + diff --git a/avise/cli.py b/avise/cli.py index 2820e3e..bda784a 100644 --- a/avise/cli.py +++ b/avise/cli.py @@ -61,12 +61,12 @@ def main(arguments=None) -> None: description="AVISE - AI Vulnerability Identification & Security Evaluation" ) parser.add_argument( - "--SET_list", + "--SET-list", action="store_true", help="List available Security Evaluation Tests", ) parser.add_argument( - "--connector_list", + "--connector-list", action="store_true", help="List available connectors and formats", ) @@ -87,6 +87,10 @@ def main(arguments=None) -> None: "--SETconf", help="Path to Security Evaluation Test configuration JSON" ) + parser.add_argument( + "--target", "-t", help="Name of the target model or system to evaluate" + ) + parser.add_argument( "--elm", help="Boolean indicator whether to use an Evaluation Language Model to evaluate SET results or not. True or False. Default: True", @@ -112,7 +116,7 @@ def main(arguments=None) -> None: help="How many times each SET is executed (default 1).", ) parser.add_argument( - "--reports_dir", + "--reports-dir", "-d", default=DEFAULT_REPORTS_DIR, help=f"Base directory for reports (default: {DEFAULT_REPORTS_DIR}).", @@ -210,11 +214,12 @@ def main(arguments=None) -> None: set_config_path=set_config_path, connector_config_path=args.connectorconf, evaluation_model_name=args.elm, - output_path=args.output, report_format=report_format, reports_dir=args.reports_dir, generate_ai_summary=args.ai_summary, runs=args.runs, + output_path=args.output, + target=args.target, ) # Print a small summary to the console diff --git a/avise/engine.py b/avise/engine.py index 6e1fcdd..e0dceac 100644 --- a/avise/engine.py +++ b/avise/engine.py @@ -94,11 +94,12 @@ def run_test( set_config_path: str, connector_config_path: str, evaluation_model_name: str, - output_path: Optional[str] = None, report_format: ReportFormat = ReportFormat.HTML, reports_dir: str = DEFAULT_REPORTS_DIR, generate_ai_summary: bool = True, runs: int = 1, + output_path: Optional[str] = None, + target: str = Optional[None], ) -> dict: """Run the 4-phase pipeline @@ -117,6 +118,12 @@ def run_test( """ # Load model configuration connector_config = self.load_connector_config(connector_config_path) + # If provided with `target`, override target model from configuration file with it + if target is not None: + if "name" in connector_config["target_model"]: + connector_config["target_model"]["name"] = target + # TODO: Once there are default connectors for other system/model types than language models, + # add logic here to replace possible "name" in their config files with `target`. # Create a connector for the target model connector = self._build_connector(connector_config, evaluation=False) From 933fa414879e2a808bf8f490efb976d352ad3b9f Mon Sep 17 00:00:00 2001 From: Zippo00 Date: Mon, 13 Apr 2026 19:27:05 +0800 Subject: [PATCH 2/2] readme updt --- README.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2b874ed..b3f9a74 100644 --- a/README.md +++ b/README.md @@ -19,15 +19,17 @@ A framework for identifying vulnerabilities in and evaluating the security of AI ### 1. Install AVISE -Install with pip: -```bash -pip install avise -``` +Install with +- **pip:** + ```bash + pip install avise + ``` -Install with uv: -```bash -uv install avise -``` +- **uv:** + + ```bash + uv install avise + ``` ### 2. Run a model