JsonCompatibilitySummary.java

package jdk.sandbox.compatibility;

import jdk.sandbox.java.util.json.Json;
import jdk.sandbox.java.util.json.JsonArray;
import jdk.sandbox.java.util.json.JsonObject;
import jdk.sandbox.java.util.json.JsonString;
import jdk.sandbox.java.util.json.JsonNumber;
import jdk.sandbox.java.util.json.JsonParseException;

import java.nio.charset.MalformedInputException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Logger;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

/// Generates a conformance summary report.
/// Run with: mvn exec:java -pl json-compatibility-suite
/// Test data location: see src/test/resources/json-test-suite-data.zip
public class JsonCompatibilitySummary {

    private static final Logger LOGGER = Logger.getLogger(JsonCompatibilitySummary.class.getName());
    private static final Path ZIP_FILE = findZipFile();
    private static final Path TARGET_TEST_DIR = Paths.get("target/test-data/json-test-suite/test_parsing");

    private static Path findZipFile() {
        // Try different possible locations for the ZIP file
        Path[] candidates = {
            Paths.get("src/test/resources/json-test-suite-data.zip"),
            Paths.get("json-compatibility-suite/src/test/resources/json-test-suite-data.zip"),
            Paths.get("../json-compatibility-suite/src/test/resources/json-test-suite-data.zip")
        };
        
        for (Path candidate : candidates) {
            if (Files.exists(candidate)) {
                return candidate;
            }
        }
        
        // If none found, return the first candidate and let it fail with a clear message
        return candidates[0];
    }

    public static void main(String[] args) throws Exception {
        boolean jsonOutput = args.length > 0 && "--json".equals(args[0]);
        JsonCompatibilitySummary summary = new JsonCompatibilitySummary();
        summary.extractTestData();
        if (jsonOutput) {
            summary.generateJsonReport();
        } else {
            summary.generateConformanceReport();
        }
    }

    void extractTestData() throws IOException {
        if (!Files.exists(ZIP_FILE)) {
            throw new RuntimeException("Test data ZIP file not found: " + ZIP_FILE.toAbsolutePath());
        }
        
        // Create target directory
        Files.createDirectories(TARGET_TEST_DIR.getParent());
        
        // Extract ZIP file
        try (ZipInputStream zis = new ZipInputStream(new FileInputStream(ZIP_FILE.toFile()))) {
            ZipEntry entry;
            while ((entry = zis.getNextEntry()) != null) {
                if (!entry.isDirectory() && entry.getName().startsWith("test_parsing/")) {
                    Path outputPath = TARGET_TEST_DIR.getParent().resolve(entry.getName());
                    Files.createDirectories(outputPath.getParent());
                    Files.copy(zis, outputPath, java.nio.file.StandardCopyOption.REPLACE_EXISTING);
                }
                zis.closeEntry();
            }
        }
    }

    void generateConformanceReport() throws Exception {
        LOGGER.fine(() -> "Starting conformance report generation");
        TestResults results = runTests();
        
        System.out.println("\n=== JSON Test Suite Conformance Report ===");
        System.out.println("Repository: java.util.json backport");
        System.out.printf("Test files analyzed: %d%n", results.totalFiles);
        System.out.printf("Files skipped (could not read): %d%n%n", results.skippedFiles);
        
        System.out.println("Valid JSON (y_ files):");
        System.out.printf("  Passed: %d%n", results.yPass);
        System.out.printf("  Failed: %d%n", results.yFail);
        System.out.printf("  Success rate: %.1f%%%n%n", 100.0 * results.yPass / (results.yPass + results.yFail));
        
        System.out.println("Invalid JSON (n_ files):");
        System.out.printf("  Correctly rejected: %d%n", results.nPass);
        System.out.printf("  Incorrectly accepted: %d%n", results.nFail);
        System.out.printf("  Success rate: %.1f%%%n%n", 100.0 * results.nPass / (results.nPass + results.nFail));
        
        System.out.println("Implementation-defined (i_ files):");
        System.out.printf("  Accepted: %d%n", results.iAccept);
        System.out.printf("  Rejected: %d%n%n", results.iReject);
        
        double conformance = 100.0 * (results.yPass + results.nPass) / (results.yPass + results.yFail + results.nPass + results.nFail);
        System.out.printf("Overall Conformance: %.1f%%%n", conformance);
        
        if (!results.shouldPassButFailed.isEmpty()) {
            LOGGER.fine(() -> "Valid JSON that failed to parse count=" + results.shouldPassButFailed.size());
            System.out.println("\n⚠️  Valid JSON that failed to parse:");
            results.shouldPassButFailed.forEach(f -> System.out.println("  - " + f));
        }

        if (!results.shouldFailButPassed.isEmpty()) {
            LOGGER.fine(() -> "Invalid JSON that was incorrectly accepted count=" + results.shouldFailButPassed.size());
            System.out.println("\n⚠️  Invalid JSON that was incorrectly accepted:");
            results.shouldFailButPassed.forEach(f -> System.out.println("  - " + f));
        }
        
        if (results.shouldPassButFailed.isEmpty() && results.shouldFailButPassed.isEmpty()) {
            System.out.println("\n✅ Perfect conformance!");
        }
    }

    void generateJsonReport() throws Exception {
        LOGGER.fine(() -> "Starting JSON report generation");
        TestResults results = runTests();
        JsonObject report = createJsonReport(results);
        System.out.println(Json.toDisplayString(report, 2));
    }

    private TestResults runTests() throws Exception {
        LOGGER.fine(() -> "Walking test files under: " + TARGET_TEST_DIR.toAbsolutePath());
        if (!Files.exists(TARGET_TEST_DIR)) {
            throw new RuntimeException("Test data not extracted. Run extractTestData() first.");
        }
        
        List<String> shouldPassButFailed = new ArrayList<>();
        List<String> shouldFailButPassed = new ArrayList<>();
        List<String> skippedFiles = new ArrayList<>();
        
        int yPass = 0, yFail = 0;
        int nPass = 0, nFail = 0;
        int iAccept = 0, iReject = 0;
        
        List<Path> files;
        try (var stream = Files.walk(TARGET_TEST_DIR)) {
            files = stream
                .filter(p -> p.toString().endsWith(".json"))
                .sorted()
                .toList();
        }
        LOGGER.fine(() -> "Discovered JSON test files: " + files.size());
            
        for (Path file : files) {
            String filename = file.getFileName().toString();
            String content;
            char[] charContent;
            final Path filePathForLog = file;
            LOGGER.fine(() -> "Processing file: " + filePathForLog);
            
            try {
                content = Files.readString(file, StandardCharsets.UTF_8);
                charContent = content.toCharArray();
            } catch (MalformedInputException e) {
                LOGGER.finer(()->"UTF-8 failed for " + filename + ", using robust encoding detection");
                try {
                    byte[] rawBytes = Files.readAllBytes(file);
                    charContent = RobustCharDecoder.decodeToChars(rawBytes, filename);
                } catch (Exception ex) {
                    skippedFiles.add(filename);
                    LOGGER.fine(() -> "Skipping unreadable file: " + filename + " due to: " + ex.getClass().getSimpleName() + ": " + ex.getMessage());
                    continue;
                }
            }
            
            // Test with char[] API (always available)
            boolean parseSucceeded;
            try {
                Json.parse(charContent);
                parseSucceeded = true;
            } catch (JsonParseException e) {
                parseSucceeded = false;
            } catch (StackOverflowError e) {
                LOGGER.warning("StackOverflowError on file: " + filename);
                parseSucceeded = false; // Treat as parse failure
            }
            final boolean parseResultForLog = parseSucceeded;
            LOGGER.fine(() -> "Parsed " + filename + ": " + (parseResultForLog ? "SUCCESS" : "FAIL"));
            
            // Update counters based on results
            if (parseSucceeded) {
                if (filename.startsWith("y_")) {
                    yPass++;
                } else if (filename.startsWith("n_")) {
                    nFail++;
                    shouldFailButPassed.add(filename);
                } else if (filename.startsWith("i_")) {
                    iAccept++;
                }
            } else {
                if (filename.startsWith("y_")) {
                    yFail++;
                    shouldPassButFailed.add(filename);
                } else if (filename.startsWith("n_")) {
                    nPass++;
                } else if (filename.startsWith("i_")) {
                    iReject++;
                }
            }
        }
        final int yPassF = yPass;
        final int yFailF = yFail;
        final int nPassF = nPass;
        final int nFailF = nFail;
        final int iAcceptF = iAccept;
        final int iRejectF = iReject;
        LOGGER.fine(() -> "Finished processing files. yPass=" + yPassF + ", yFail=" + yFailF + ", nPass=" + nPassF + ", nFail=" + nFailF + ", iAccept=" + iAcceptF + ", iReject=" + iRejectF);
        return new TestResults(files.size(), skippedFiles.size(), 
            yPass, yFail, nPass, nFail, iAccept, iReject,
            shouldPassButFailed, shouldFailButPassed, skippedFiles);
    }

    private JsonObject createJsonReport(TestResults results) {
        double ySuccessRate = 100.0 * results.yPass / (results.yPass + results.yFail);
        double nSuccessRate = 100.0 * results.nPass / (results.nPass + results.nFail);
        double conformance = 100.0 * (results.yPass + results.nPass) / (results.yPass + results.yFail + results.nPass + results.nFail);

        return JsonObject.of(java.util.Map.of(
            "repository", JsonString.of("java.util.json backport"),
            "filesAnalyzed", JsonNumber.of(results.totalFiles),
            "filesSkipped", JsonNumber.of(results.skippedFiles),
            "validJson", JsonObject.of(java.util.Map.of(
                "passed", JsonNumber.of(results.yPass),
                "failed", JsonNumber.of(results.yFail),
                "successRate", JsonNumber.of(Math.round(ySuccessRate * 10) / 10.0)
            )),
            "invalidJson", JsonObject.of(java.util.Map.of(
                "correctlyRejected", JsonNumber.of(results.nPass),
                "incorrectlyAccepted", JsonNumber.of(results.nFail),
                "successRate", JsonNumber.of(Math.round(nSuccessRate * 10) / 10.0)
            )),
            "implementationDefined", JsonObject.of(java.util.Map.of(
                "accepted", JsonNumber.of(results.iAccept),
                "rejected", JsonNumber.of(results.iReject)
            )),
            "overallConformance", JsonNumber.of(Math.round(conformance * 10) / 10.0),
            "shouldPassButFailed", JsonArray.of(results.shouldPassButFailed.stream()
                .map(JsonString::of)
                .toList()),
            "shouldFailButPassed", JsonArray.of(results.shouldFailButPassed.stream()
                .map(JsonString::of)
                .toList())
        ));
    }

    private record TestResults(
        int totalFiles, int skippedFiles,
        int yPass, int yFail, int nPass, int nFail, int iAccept, int iReject,
        List<String> shouldPassButFailed, List<String> shouldFailButPassed, List<String> skippedFiles2
    ) {}
}