Import lissa from jamieklein (squashed): replace upstream lissa with …

.gitignore

@@ -50,3 +50,6 @@ build/
 /transitive-trace*.csv
 /config.json
 package-info.java
+jamieklein/lissa/cache/
+jamieklein/lissa/configs/
+jamieklein/lissa/datasets/

src/main/java/edu/kit/kastel/sdq/lissa/cli/MainCLI.java

@@ -2,8 +2,10 @@
 package edu.kit.kastel.sdq.lissa.cli;
 
 import java.nio.file.Path;
+import java.util.Arrays;
 
 import edu.kit.kastel.sdq.lissa.cli.command.EvaluateCommand;
+import edu.kit.kastel.sdq.lissa.cli.command.OptimizeCommand;
 import edu.kit.kastel.sdq.lissa.cli.command.TransitiveTraceCommand;
 
 import picocli.CommandLine;
@@ -16,11 +18,11 @@
  *     <li>{@link EvaluateCommand} - Evaluates trace link analysis configurations</li>
  *     <li>{@link TransitiveTraceCommand} - Performs transitive trace link analysis</li>
  * </ul>
- *
+ * <p>
  * The CLI supports various command-line options and provides help information
  * through the standard help options (--help, -h).
  */
-@CommandLine.Command(subcommands = {EvaluateCommand.class, TransitiveTraceCommand.class})
+@CommandLine.Command(subcommands = {EvaluateCommand.class, TransitiveTraceCommand.class, OptimizeCommand.class})
 public final class MainCLI {
 
     /**
@@ -34,6 +36,18 @@ private MainCLI() {}
      * @param args Command line arguments to be processed
      */
     public static void main(String[] args) {
+        if (args.length == 0) {
+            args = new String[] {
+                "optimize",
+                "-c",
+                "configs\\req2req\\CM1-NASA_simple_gpt_gpt-4o-mini-2024-07-18.json",
+                "--max-iter",
+                "3",
+                "--target-f1",
+                "0.40"
+            };
+        }
+        System.out.println("ARGS: " + Arrays.toString(args));
         new CommandLine(new MainCLI()).registerConverter(Path.class, Path::of).execute(args);
     }
 }

src/main/java/edu/kit/kastel/sdq/lissa/cli/command/OptimizeCommand.java

@@ -0,0 +1,72 @@
+/* Licensed under MIT 2025. */
+package edu.kit.kastel.sdq.lissa.cli.command;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Path;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import edu.kit.kastel.sdq.lissa.ratlr.cache.CacheManager;
+import edu.kit.kastel.sdq.lissa.ratlr.configuration.Configuration;
+import edu.kit.kastel.sdq.lissa.ratlr.optimizer.PromptOptimizer;
+
+import picocli.CommandLine;
+
+/**
+ * CLI subcommand for running LiSSA with a prompt-optimization loop.
+ * This command accepts a configuration file, a maximum number of optimization iterations,
+ * and a target F1 score. It delegates the optimization process to the PromptOptimizer.
+ */
+@CommandLine.Command(name = "optimize", description = "Runs LiSSA with a prompt-optimization loop")
+public final class OptimizeCommand implements Runnable {
+
+    private static final Logger logger = LoggerFactory.getLogger(OptimizeCommand.class);
+
+    @CommandLine.Option(
+            names = {"-c", "--config"},
+            required = true,
+            description = "Path to the LiSSA JSON configuration")
+    private Path config;
+
+    @CommandLine.Option(
+            names = {"--max-iter"},
+            defaultValue = "1",
+            description = "Maximum optimization iterations (≥1)")
+    private int maxIterations;
+
+    @CommandLine.Option(
+            names = {"--target-f1"},
+            defaultValue = "0.40",
+            description = "Target F1 score to reach before stopping")
+    private double targetF1;
+
+    /**
+     * Executes the optimization process with the provided configuration.
+     * This method initializes the PromptOptimizer and starts the optimization loop.
+     */
+    @Override
+    public void run() {
+        logger.info("Starting optimization for {}", config);
+        try {
+            Configuration fullConfig = new ObjectMapper().readValue(config.toFile(), Configuration.class);
+            String cacheDirectoryString = fullConfig.cacheDir();
+            if (cacheDirectoryString != null) {
+                CacheManager.setCacheDir(cacheDirectoryString);
+                logger.info("Cache directory set to {}", cacheDirectoryString);
+            } else {
+                logger.warn("No cache directory found in configuration, caching may be disabled.");
+            }
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+        try {
+            new PromptOptimizer(config, maxIterations, targetF1).optimize();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

src/main/java/edu/kit/kastel/sdq/lissa/ratlr/Evaluation.java

@@ -3,8 +3,10 @@
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Objects;
 import java.util.Set;
+import java.util.SortedMap;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -15,10 +17,12 @@
 import edu.kit.kastel.sdq.lissa.ratlr.cache.CacheManager;
 import edu.kit.kastel.sdq.lissa.ratlr.classifier.Classifier;
 import edu.kit.kastel.sdq.lissa.ratlr.configuration.Configuration;
-import edu.kit.kastel.sdq.lissa.ratlr.context.ContextStore;
+import edu.kit.kastel.sdq.lissa.ratlr.configuration.GoldStandardConfiguration;
 import edu.kit.kastel.sdq.lissa.ratlr.elementstore.ElementStore;
 import edu.kit.kastel.sdq.lissa.ratlr.embeddingcreator.EmbeddingCreator;
+import edu.kit.kastel.sdq.lissa.ratlr.knowledge.Element;
 import edu.kit.kastel.sdq.lissa.ratlr.knowledge.TraceLink;
+import edu.kit.kastel.sdq.lissa.ratlr.optimizer.ClassificationResultsManager;
 import edu.kit.kastel.sdq.lissa.ratlr.postprocessor.TraceLinkIdPostprocessor;
 import edu.kit.kastel.sdq.lissa.ratlr.preprocessor.Preprocessor;
 import edu.kit.kastel.sdq.lissa.ratlr.resultaggregator.ResultAggregator;
@@ -36,10 +40,6 @@
  *     <li>Statistics generation and result storage</li>
  * </ul>
  * <p>
- * The pipeline uses a {@link edu.kit.kastel.sdq.lissa.ratlr.context.ContextStore} to share context objects
- * between components such as artifact providers, preprocessors, embedding creators, classifiers, and aggregators.
- * </p>
- *
  * The pipeline follows these steps:
  * <ol>
  *     <li>Load artifacts from configured providers</li>
@@ -59,25 +59,45 @@ public class Evaluation {
 
     private Configuration configuration;
 
-    /** Provider for source artifacts */
+    /**
+     * Provider for source artifacts
+     */
     private ArtifactProvider sourceArtifactProvider;
-    /** Provider for target artifacts */
+    /**
+     * Provider for target artifacts
+     */
     private ArtifactProvider targetArtifactProvider;
-    /** Preprocessor for source artifacts */
+    /**
+     * Preprocessor for source artifacts
+     */
     private Preprocessor sourcePreprocessor;
-    /** Preprocessor for target artifacts */
+    /**
+     * Preprocessor for target artifacts
+     */
     private Preprocessor targetPreprocessor;
-    /** Creator for element embeddings */
+    /**
+     * Creator for element embeddings
+     */
     private EmbeddingCreator embeddingCreator;
-    /** Store for source elements */
+    /**
+     * Store for source elements
+     */
     private ElementStore sourceStore;
-    /** Store for target elements */
+    /**
+     * Store for target elements
+     */
     private ElementStore targetStore;
-    /** Classifier for trace link analysis */
+    /**
+     * Classifier for trace link analysis
+     */
     private Classifier classifier;
-    /** Aggregator for classification results */
+    /**
+     * Aggregator for classification results
+     */
     private ResultAggregator aggregator;
-    /** Postprocessor for trace link IDs */
+    /**
+     * Postprocessor for trace link IDs
+     */
     private TraceLinkIdPostprocessor traceLinkIdPostProcessor;
 
     /**
@@ -86,11 +106,11 @@ public class Evaluation {
      * <ol>
      *     <li>Validates the configuration file path</li>
      *     <li>Loads and initializes the configuration</li>
-     *     <li>Sets up all required components for the pipeline, sharing a {@link ContextStore}</li>
+     *     <li>Sets up all required components for the pipeline</li>
      * </ol>
      *
      * @param configFile Path to the configuration file
-     * @throws IOException If there are issues reading the configuration file
+     * @throws IOException          If there are issues reading the configuration file
      * @throws NullPointerException If configFile is null
      */
     public Evaluation(Path configFile) throws IOException {
@@ -119,25 +139,21 @@ private void setup() throws IOException {
         configuration = new ObjectMapper().readValue(configFile.toFile(), Configuration.class);
         CacheManager.setCacheDir(configuration.cacheDir());
 
-        ContextStore contextStore = new ContextStore();
-
-        sourceArtifactProvider =
-                ArtifactProvider.createArtifactProvider(configuration.sourceArtifactProvider(), contextStore);
-        targetArtifactProvider =
-                ArtifactProvider.createArtifactProvider(configuration.targetArtifactProvider(), contextStore);
+        sourceArtifactProvider = ArtifactProvider.createArtifactProvider(configuration.sourceArtifactProvider());
+        targetArtifactProvider = ArtifactProvider.createArtifactProvider(configuration.targetArtifactProvider());
 
-        sourcePreprocessor = Preprocessor.createPreprocessor(configuration.sourcePreprocessor(), contextStore);
-        targetPreprocessor = Preprocessor.createPreprocessor(configuration.targetPreprocessor(), contextStore);
+        sourcePreprocessor = Preprocessor.createPreprocessor(configuration.sourcePreprocessor());
+        targetPreprocessor = Preprocessor.createPreprocessor(configuration.targetPreprocessor());
 
-        embeddingCreator = EmbeddingCreator.createEmbeddingCreator(configuration.embeddingCreator(), contextStore);
+        embeddingCreator = EmbeddingCreator.createEmbeddingCreator(configuration.embeddingCreator());
         sourceStore = new ElementStore(configuration.sourceStore(), false);
         targetStore = new ElementStore(configuration.targetStore(), true);
 
-        classifier = configuration.createClassifier(contextStore);
-        aggregator = ResultAggregator.createResultAggregator(configuration.resultAggregator(), contextStore);
+        classifier = configuration.createClassifier();
+        aggregator = ResultAggregator.createResultAggregator(configuration.resultAggregator());
 
-        traceLinkIdPostProcessor = TraceLinkIdPostprocessor.createTraceLinkIdPostprocessor(
-                configuration.traceLinkIdPostprocessor(), contextStore);
+        traceLinkIdPostProcessor =
+                TraceLinkIdPostprocessor.createTraceLinkIdPostprocessor(configuration.traceLinkIdPostprocessor());
 
         configuration.serializeAndDestroyConfiguration();
     }
@@ -197,7 +213,25 @@ public Set<TraceLink> run() {
                 traceLinks, configFile.toFile(), configuration, sourceArtifacts.size(), targetArtifacts.size());
         Statistics.saveTraceLinks(traceLinks, configFile.toFile(), configuration);
 
-        CacheManager.getDefaultInstance().flush();
+        /**
+         * Saves detailed classification results for trace links to disk.
+         * This includes generating JSON results and merging source/target elements
+         * to produce TP, FP, FN categorizations for further analysis.
+         */
+        try {
+            Path resultDir = Paths.get("logs");
+            Path jsonFile = ClassificationResultsManager.defaultResultFile(resultDir);
+
+            ClassificationResultsManager classificationResultsManager = new ClassificationResultsManager(jsonFile);
+
+            SortedMap<String, Element> allElements =
+                    ClassificationResultsManager.mergeElements(sourceElements, targetElements);
+            GoldStandardConfiguration goldConfig = configuration.goldStandardConfiguration();
+            classificationResultsManager.saveDetailedResults(traceLinks, allElements, goldConfig);
+
+        } catch (IOException ex) {
+            logger.error("Failed to write detailed results file", ex);
+        }
 
         return traceLinks;
     }
@@ -212,6 +246,7 @@ public int getSourceArtifactCount() {
     }
 
     /**
+     *
      * Gets the number of target artifacts in this evaluation.
      *
      * @return Number of target artifacts

src/main/java/edu/kit/kastel/sdq/lissa/ratlr/Main.java

@@ -13,7 +13,6 @@
 import edu.kit.kastel.sdq.lissa.ratlr.cache.CacheManager;
 import edu.kit.kastel.sdq.lissa.ratlr.classifier.Classifier;
 import edu.kit.kastel.sdq.lissa.ratlr.configuration.Configuration;
-import edu.kit.kastel.sdq.lissa.ratlr.context.ContextStore;
 import edu.kit.kastel.sdq.lissa.ratlr.elementstore.ElementStore;
 import edu.kit.kastel.sdq.lissa.ratlr.embeddingcreator.EmbeddingCreator;
 import edu.kit.kastel.sdq.lissa.ratlr.postprocessor.TraceLinkIdPostprocessor;
@@ -32,10 +31,6 @@
  *     <li>Generating statistics</li>
  * </ul>
  * <p>
- * The pipeline uses a {@link edu.kit.kastel.sdq.lissa.ratlr.context.ContextStore} to share context objects
- * between components such as artifact providers, preprocessors, embedding creators, classifiers, and aggregators.
- * </p>
- *
  * The pipeline follows these steps:
  * <ol>
  *     <li>Load configuration from file</li>
@@ -58,7 +53,7 @@ public class Main {
      * This method:
      * <ol>
      *     <li>Loads configuration from file (defaults to "config.json" if no file specified)</li>
-     *     <li>Initializes all required components, sharing a {@link ContextStore}</li>
+     *     <li>Initializes all required components</li>
      *     <li>Executes the complete trace link analysis pipeline</li>
      *     <li>Generates and saves results</li>
      * </ol>
@@ -72,29 +67,23 @@ public static void main(String[] args) throws IOException {
         Configuration configuration = new ObjectMapper().readValue(configFile, Configuration.class);
         CacheManager.setCacheDir(configuration.cacheDir());
 
-        ContextStore contextStore = new ContextStore();
-
         ArtifactProvider sourceArtifactProvider =
-                ArtifactProvider.createArtifactProvider(configuration.sourceArtifactProvider(), contextStore);
+                ArtifactProvider.createArtifactProvider(configuration.sourceArtifactProvider());
         ArtifactProvider targetArtifactProvider =
-                ArtifactProvider.createArtifactProvider(configuration.targetArtifactProvider(), contextStore);
+                ArtifactProvider.createArtifactProvider(configuration.targetArtifactProvider());
 
-        Preprocessor sourcePreprocessor =
-                Preprocessor.createPreprocessor(configuration.sourcePreprocessor(), contextStore);
-        Preprocessor targetPreprocessor =
-                Preprocessor.createPreprocessor(configuration.targetPreprocessor(), contextStore);
+        Preprocessor sourcePreprocessor = Preprocessor.createPreprocessor(configuration.sourcePreprocessor());
+        Preprocessor targetPreprocessor = Preprocessor.createPreprocessor(configuration.targetPreprocessor());
 
-        EmbeddingCreator embeddingCreator =
-                EmbeddingCreator.createEmbeddingCreator(configuration.embeddingCreator(), contextStore);
+        EmbeddingCreator embeddingCreator = EmbeddingCreator.createEmbeddingCreator(configuration.embeddingCreator());
         ElementStore sourceStore = new ElementStore(configuration.sourceStore(), false);
         ElementStore targetStore = new ElementStore(configuration.targetStore(), true);
 
-        Classifier classifier = configuration.createClassifier(contextStore);
-        ResultAggregator aggregator =
-                ResultAggregator.createResultAggregator(configuration.resultAggregator(), contextStore);
+        Classifier classifier = configuration.createClassifier();
+        ResultAggregator aggregator = ResultAggregator.createResultAggregator(configuration.resultAggregator());
 
-        TraceLinkIdPostprocessor traceLinkIdPostProcessor = TraceLinkIdPostprocessor.createTraceLinkIdPostprocessor(
-                configuration.traceLinkIdPostprocessor(), contextStore);
+        TraceLinkIdPostprocessor traceLinkIdPostProcessor =
+                TraceLinkIdPostprocessor.createTraceLinkIdPostprocessor(configuration.traceLinkIdPostprocessor());
 
         configuration.serializeAndDestroyConfiguration();