pre-commit passed

Author: sidmohan0

.pre-commit-config.yaml

@@ -25,4 +25,5 @@ repos:
     rev: v4.0.0-alpha.8
     hooks:
       - id: prettier
+        files: '.*\.(md|markdown|json|yaml|yml|js|jsx|css|html)$'
         exclude: .venv

datafog/services/spark_service.py

@@ -23,51 +23,59 @@ class SparkService:
 
     def __init__(self, master=None):
         self.master = master
-        
+
         # Ensure pyspark is installed first
         self.ensure_installed("pyspark")
-        
+
         # Now import necessary modules after ensuring pyspark is installed
         try:
             from pyspark.sql import DataFrame, SparkSession
             from pyspark.sql.functions import udf
             from pyspark.sql.types import ArrayType, StringType
-    
+
             # Assign fields
             self.SparkSession = SparkSession
             self.DataFrame = DataFrame
             self.udf = udf
             self.ArrayType = ArrayType
             self.StringType = StringType
-            
+
             # Create the spark session
             self.spark = self.create_spark_session()
         except ImportError as e:
-            raise ImportError(f"Failed to import PySpark modules: {e}. "
-                             f"Make sure PySpark is installed correctly.")
+            raise ImportError(
+                f"Failed to import PySpark modules: {e}. "
+                f"Make sure PySpark is installed correctly."
+            )
 
     def create_spark_session(self):
         # Check if we're running in a test environment
-        in_test_env = 'PYTEST_CURRENT_TEST' in os.environ or 'TOX_ENV_NAME' in os.environ
-        
+        in_test_env = (
+            "PYTEST_CURRENT_TEST" in os.environ or "TOX_ENV_NAME" in os.environ
+        )
+
         # Set Java system properties to handle security manager issues
         # This is needed for newer Java versions
-        os.environ['PYSPARK_SUBMIT_ARGS'] = '--conf spark.driver.allowMultipleContexts=true pyspark-shell'
-        
+        os.environ["PYSPARK_SUBMIT_ARGS"] = (
+            "--conf spark.driver.allowMultipleContexts=true pyspark-shell"
+        )
+
         # Create a builder with the app name
         builder = self.SparkSession.builder.appName("datafog")
-        
+
         # Add configuration to work around security manager issues
         builder = builder.config("spark.driver.allowMultipleContexts", "true")
-        builder = builder.config("spark.driver.extraJavaOptions", "-Djava.security.manager=allow")
-        
+        builder = builder.config(
+            "spark.driver.extraJavaOptions", "-Djava.security.manager=allow"
+        )
+
         # If master is specified, use it
         if self.master:
             builder = builder.master(self.master)
         # Otherwise, if we're in a test environment, use local mode
         elif in_test_env:
             builder = builder.master("local[1]")
-            
+
         # Create and return the session
         return builder.getOrCreate()
 
@@ -86,6 +94,7 @@ def ensure_installed(self, package_name):
                 print(f"{package_name} installed successfully.")
             except subprocess.CalledProcessError as e:
                 print(f"Failed to install {package_name}: {e}")
-                raise ImportError(f"Could not install {package_name}. "
-                                 f"Please install it manually with 'pip install {package_name}'.")
-
+                raise ImportError(
+                    f"Could not install {package_name}. "
+                    f"Please install it manually with 'pip install {package_name}'."
+                )

notes/story-1.7-tkt.md

@@ -24,4 +24,4 @@ Or directly with pytest:
 
 ```bash
 pytest -m "integration"
-```
+```

tests/test_spark_integration.py

@@ -14,11 +14,11 @@ def spark_service():
     """Create a shared SparkService instance for all tests."""
     # Initialize SparkService with explicit local mode
     service = SparkService(master="local[1]")
-    
+
     yield service
-    
+
     # Clean up after all tests
-    if hasattr(service, 'spark') and service.spark is not None:
+    if hasattr(service, "spark") and service.spark is not None:
         service.spark.stop()
 
 
@@ -30,15 +30,15 @@ def sample_json_data():
         {"name": "Jane Smith", "email": "jane.smith@example.com", "age": 25},
         {"name": "Bob Johnson", "email": "bob.johnson@example.com", "age": 40},
     ]
-    
+
     # Create a temporary file
     with tempfile.NamedTemporaryFile(suffix=".json", delete=False, mode="w") as f:
         for item in data:
             f.write(json.dumps(item) + "\n")
         temp_file = f.name
-    
+
     yield temp_file
-    
+
     # Clean up the temporary file after the test
     if os.path.exists(temp_file):
         os.remove(temp_file)
@@ -51,7 +51,7 @@ def test_spark_service_initialization(spark_service):
     assert spark_service.spark is not None
     assert spark_service.spark.sparkContext.appName == "datafog"
     assert spark_service.spark.sparkContext.master.startswith("local")
-    
+
     # Verify that the necessary Spark classes are available
     assert spark_service.DataFrame is not None
     assert spark_service.SparkSession is not None
@@ -63,16 +63,16 @@ def test_spark_read_json(spark_service, sample_json_data):
     """Test that SparkService can read JSON data in local mode."""
     # Read the JSON data
     result = spark_service.read_json(sample_json_data)
-    
+
     # Verify the result
     assert len(result) == 3, f"Expected 3 rows, got {len(result)}"
-    
+
     # PySpark Row objects have a __contains__ method and can be accessed like dictionaries
     # but they're not actually dictionaries
     assert all(hasattr(item, "name") for item in result), "Missing 'name' field"
     assert all(hasattr(item, "email") for item in result), "Missing 'email' field"
     assert all(hasattr(item, "age") for item in result), "Missing 'age' field"
-    
+
     # Verify specific values
     names = [item.name for item in result]
     assert "John Doe" in names, f"Expected 'John Doe' in {names}"