test_deterministic.py

#!/usr/bin/env python3
"""
Test script to verify deterministic behavior of the stylometric analysis.
"""

import os
import pandas as pd
import numpy as np
from stylometric_analyzer import StylometricAnalyzer

def test_deterministic_analysis():
    """Test that running the same analysis multiple times produces identical results."""
    
    # Create test articles
    test_articles = {
        "article1": "This is a test article about technology and innovation. It discusses various aspects of modern computing and artificial intelligence.",
        "article2": "Another test article focusing on science and research. It covers topics related to machine learning and data analysis.",
        "article3": "A third article about business and economics. It examines market trends and financial analysis."
    }
    
    print("Testing deterministic behavior...")
    print("Running analysis 3 times with the same input...")
    
    results = []
    
    for i in range(3):
        print(f"\n--- Run {i+1} ---")
        
        # Create a fresh analyzer instance
        analyzer = StylometricAnalyzer()
        
        # Perform pairwise analysis
        df = analyzer.pairwise_analysis(test_articles)
        
        # Store results for comparison
        results.append(df)
        
        print(f"Run {i+1} completed. Shape: {df.shape}")
        print(f"First row semantic score: {df.iloc[0]['semantic_avg_score']:.6f}")
    
    # Compare results
    print("\n=== COMPARISON ===")
    
    # Check if all DataFrames are identical
    all_identical = True
    for i in range(1, len(results)):
        if not results[0].equals(results[i]):
            all_identical = False
            print(f"Results {i} differs from results 0")
        else:
            print(f"Results {i} is identical to results 0")
    
    if all_identical:
        print("\n✓ SUCCESS: All runs produced identical results!")
        print("The analysis is deterministic.")
    else:
        print("\n✗ FAILURE: Results differ between runs!")
        print("The analysis is not deterministic.")
        
        # Show differences
        for i in range(1, len(results)):
            print(f"\nDifferences between run 0 and run {i}:")
            diff_mask = results[0] != results[i]
            if diff_mask.any().any():
                for col in results[0].columns:
                    if diff_mask[col].any():
                        print(f"  Column '{col}' has differences")
                        for idx in diff_mask[diff_mask[col]].index:
                            print(f"    Row {idx}: {results[0].loc[idx, col]} vs {results[i].loc[idx, col]}")
    
    return all_identical

def test_clustering_deterministic():
    """Test that clustering produces deterministic results."""
    
    from heatmap_visualizer import create_similarity_heatmap
    
    # Create test data
    test_data = pd.DataFrame({
        'pair': ['article1 vs article2', 'article1 vs article3', 'article2 vs article3'],
        'semantic_avg_score': [0.75, 0.65, 0.80],
        'semantic_median_score': [0.74, 0.64, 0.79],
        'semantic_max_score': [0.85, 0.75, 0.90],
        'semantic_z_score': [1.5, 0.5, 2.0],
        'semantic_baseline_mean': [0.6, 0.6, 0.6],
        'semantic_baseline_std': [0.1, 0.1, 0.1]
    })
    
    print("\nTesting clustering deterministic behavior...")
    
    results = []
    
    for i in range(3):
        print(f"\n--- Clustering Run {i+1} ---")
        
        # Create heatmap (this will trigger clustering)
        fig = create_similarity_heatmap(test_data, metric='semantic')
        
        # Extract the z data (similarity matrix) from the figure
        z_data = fig.data[0].z
        results.append(z_data)
        
        print(f"Clustering run {i+1} completed.")
        print(f"Matrix shape: {z_data.shape}")
        print(f"First element: {z_data[0, 0]:.6f}")
    
    # Compare clustering results
    print("\n=== CLUSTERING COMPARISON ===")
    
    all_identical = True
    for i in range(1, len(results)):
        if not np.array_equal(results[0], results[i]):
            all_identical = False
            print(f"Clustering results {i} differs from results 0")
        else:
            print(f"Clustering results {i} is identical to results 0")
    
    if all_identical:
        print("\n✓ SUCCESS: All clustering runs produced identical results!")
        print("The clustering is deterministic.")
    else:
        print("\n✗ FAILURE: Clustering results differ between runs!")
        print("The clustering is not deterministic.")
    
    return all_identical

if __name__ == "__main__":
    print("Deterministic Analysis Test")
    print("=" * 40)
    
    # Test main analysis
    analysis_deterministic = test_deterministic_analysis()
    
    # Test clustering
    clustering_deterministic = test_clustering_deterministic()
    
    print("\n" + "=" * 40)
    print("FINAL RESULTS:")
    print(f"Analysis deterministic: {'✓' if analysis_deterministic else '✗'}")
    print(f"Clustering deterministic: {'✓' if clustering_deterministic else '✗'}")
    
    if analysis_deterministic and clustering_deterministic:
        print("\n🎉 ALL TESTS PASSED! The analysis is fully deterministic.")
    else:
        print("\n⚠️  Some tests failed. The analysis may not be fully deterministic.")