Fix: raise soft ulimit for open files in Python runtime startup

hack/test-python.sh

@@ -66,4 +66,32 @@ PYTHON_PATH=$(get_python_path)
 # Cleanup
 rm -rf .venv
 
+echo "✓ Python CloudEvents template tests passed"
+
+# Test HTTP scaffolding ulimit helper
+cd "${PROJECT_ROOT}/templates/python/scaffolding/instanced-http"
+
+python3 -m venv .venv || python -m venv .venv
+PYTHON_PATH=$(get_python_path)
+
+"${PYTHON_PATH}" -m pip install -q --upgrade pip
+"${PYTHON_PATH}" -m pip install -q pytest
+"${PYTHON_PATH}" -m pytest tests/test_ulimit.py -v
+
+rm -rf .venv .pytest_cache service/__pycache__ tests/__pycache__
+
+echo "✓ Python HTTP scaffolding tests passed"
+
+# Test CloudEvents scaffolding ulimit helper
+cd "${PROJECT_ROOT}/templates/python/scaffolding/instanced-cloudevents"
+
+python3 -m venv .venv || python -m venv .venv
+PYTHON_PATH=$(get_python_path)
+
+"${PYTHON_PATH}" -m pip install -q --upgrade pip
+"${PYTHON_PATH}" -m pip install -q pytest
+"${PYTHON_PATH}" -m pytest tests/test_ulimit.py -v
+
+rm -rf .venv .pytest_cache service/__pycache__ tests/__pycache__
+
 echo "✓ All Python template tests completed successfully"

templates/python/scaffolding/instanced-cloudevents/.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+.pytest_cache/
+.venv/
+*.pyc

templates/python/scaffolding/instanced-cloudevents/service/_ulimit.py

@@ -0,0 +1,34 @@
+"""
+Utility to raise the process soft open-file limit to match the hard limit.
+
+Platforms such as some container runtimes default the soft limit to 1024,
+which causes Python functions to fail under load.  Go and Java runtimes do
+this automatically; this module replicates that behaviour for Python.
+"""
+import logging
+
+# Safe maximum used when the hard limit is RLIM_INFINITY (no hard cap set).
+_MAX_NOFILE = 65536
+
+
+def configure():
+    """Raise the soft open-file limit toward the hard limit, if possible."""
+    try:
+        import resource
+    except ImportError:
+        # resource module is Unix-only; skip on non-Unix platforms.
+        return
+    try:
+        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+        if soft < hard:
+            # RLIM_INFINITY means no hard cap is set by the kernel.
+            # Passing it directly to setrlimit raises an OSError on most
+            # systems, so cap the target at a known-safe value instead.
+            if hard == resource.RLIM_INFINITY:
+                target = _MAX_NOFILE
+            else:
+                target = min(hard, _MAX_NOFILE)
+            resource.setrlimit(resource.RLIMIT_NOFILE, (target, hard))
+            logging.info("Raised open-file limit from %d to %d", soft, target)
+    except (ValueError, OSError) as e:
+        logging.warning("Could not raise open-file limit: %s", e)

templates/python/scaffolding/instanced-cloudevents/service/main.py

@@ -5,6 +5,7 @@
 final container.
 """
 import logging
+from service._ulimit import configure as _configure_ulimit
 from func_python.cloudevent import serve
 
 logging.basicConfig(level=logging.INFO)
@@ -19,5 +20,6 @@
         raise
 
 if __name__ == "__main__":
+    _configure_ulimit()
     logging.info("Functions middleware invoking user function")
     serve(handler)

templates/python/scaffolding/instanced-cloudevents/tests/test_ulimit.py

@@ -0,0 +1,106 @@
+"""
+Tests for the open-file limit helper in the CloudEvents scaffolding.
+"""
+import sys
+import importlib
+import logging
+from unittest.mock import patch, MagicMock
+
+# Ensure the service package is importable from the scaffolding root.
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from service._ulimit import configure
+
+
+def test_raises_soft_limit_to_hard():
+    """configure() should raise the soft limit to the hard limit when soft < hard."""
+    mock_resource = MagicMock()
+    mock_resource.getrlimit.return_value = (1024, 65536)
+    mock_resource.RLIMIT_NOFILE = 7  # canonical Linux value
+    mock_resource.RLIM_INFINITY = 9223372036854775807  # Linux RLIM_INFINITY
+
+    with patch.dict(sys.modules, {"resource": mock_resource}):
+        importlib.reload(sys.modules["service._ulimit"])
+        from service._ulimit import configure as _configure
+        _configure()
+
+    mock_resource.setrlimit.assert_called_once_with(
+        mock_resource.RLIMIT_NOFILE, (65536, 65536)
+    )
+
+
+def test_no_change_when_soft_equals_hard():
+    """configure() should not call setrlimit when soft == hard."""
+    mock_resource = MagicMock()
+    mock_resource.getrlimit.return_value = (65536, 65536)
+    mock_resource.RLIMIT_NOFILE = 7
+    mock_resource.RLIM_INFINITY = 9223372036854775807  # Linux RLIM_INFINITY
+
+    with patch.dict(sys.modules, {"resource": mock_resource}):
+        importlib.reload(sys.modules["service._ulimit"])
+        from service._ulimit import configure as _configure
+        _configure()
+
+    mock_resource.setrlimit.assert_not_called()
+
+
+def test_rlim_infinity_capped_at_max():
+    """configure() should cap the target at _MAX_NOFILE when hard == RLIM_INFINITY."""
+    mock_resource = MagicMock()
+    mock_resource.RLIMIT_NOFILE = 7
+    mock_resource.RLIM_INFINITY = 9223372036854775807  # Linux RLIM_INFINITY
+    # Soft is below the safe max; hard is unlimited.
+    mock_resource.getrlimit.return_value = (1024, mock_resource.RLIM_INFINITY)
+
+    with patch.dict(sys.modules, {"resource": mock_resource}):
+        importlib.reload(sys.modules["service._ulimit"])
+        from service._ulimit import configure as _configure
+        _configure()
+
+    mock_resource.setrlimit.assert_called_once_with(
+        mock_resource.RLIMIT_NOFILE, (65536, mock_resource.RLIM_INFINITY)
+    )
+
+
+def test_import_error_is_silently_skipped():
+    """configure() should do nothing when resource module is unavailable (non-Unix)."""
+    with patch.dict(sys.modules, {"resource": None}):
+        importlib.reload(sys.modules["service._ulimit"])
+        from service._ulimit import configure as _configure
+        # Must not raise.
+        _configure()
+
+
+def test_os_error_logs_warning(caplog):
+    """configure() should log a warning and not propagate an OSError."""
+    mock_resource = MagicMock()
+    mock_resource.getrlimit.return_value = (1024, 65536)
+    mock_resource.RLIMIT_NOFILE = 7
+    mock_resource.RLIM_INFINITY = 9223372036854775807  # Linux RLIM_INFINITY
+    mock_resource.setrlimit.side_effect = OSError("operation not permitted")
+
+    with patch.dict(sys.modules, {"resource": mock_resource}):
+        importlib.reload(sys.modules["service._ulimit"])
+        from service._ulimit import configure as _configure
+        with caplog.at_level(logging.WARNING):
+            _configure()  # must not raise
+
+    assert "Could not raise open-file limit" in caplog.text
+
+
+def test_value_error_logs_warning(caplog):
+    """configure() should log a warning and not propagate a ValueError."""
+    mock_resource = MagicMock()
+    mock_resource.getrlimit.return_value = (1024, 65536)
+    mock_resource.RLIMIT_NOFILE = 7
+    mock_resource.RLIM_INFINITY = 9223372036854775807  # Linux RLIM_INFINITY
+    mock_resource.setrlimit.side_effect = ValueError("invalid argument")
+
+    with patch.dict(sys.modules, {"resource": mock_resource}):
+        importlib.reload(sys.modules["service._ulimit"])
+        from service._ulimit import configure as _configure
+        with caplog.at_level(logging.WARNING):
+            _configure()  # must not raise
+
+    assert "Could not raise open-file limit" in caplog.text

templates/python/scaffolding/instanced-http/.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+.pytest_cache/
+.venv/
+*.pyc

templates/python/scaffolding/instanced-http/service/_ulimit.py

@@ -0,0 +1,34 @@
+"""
+Utility to raise the process soft open-file limit to match the hard limit.
+
+Platforms such as some container runtimes default the soft limit to 1024,
+which causes Python functions to fail under load.  Go and Java runtimes do
+this automatically; this module replicates that behaviour for Python.
+"""
+import logging
+
+# Safe maximum used when the hard limit is RLIM_INFINITY (no hard cap set).
+_MAX_NOFILE = 65536
+
+
+def configure():
+    """Raise the soft open-file limit toward the hard limit, if possible."""
+    try:
+        import resource
+    except ImportError:
+        # resource module is Unix-only; skip on non-Unix platforms.
+        return
+    try:
+        soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+        if soft < hard:
+            # RLIM_INFINITY means no hard cap is set by the kernel.
+            # Passing it directly to setrlimit raises an OSError on most
+            # systems, so cap the target at a known-safe value instead.
+            if hard == resource.RLIM_INFINITY:
+                target = _MAX_NOFILE
+            else:
+                target = min(hard, _MAX_NOFILE)
+            resource.setrlimit(resource.RLIMIT_NOFILE, (target, hard))
+            logging.info("Raised open-file limit from %d to %d", soft, target)
+    except (ValueError, OSError) as e:
+        logging.warning("Could not raise open-file limit: %s", e)

templates/python/scaffolding/instanced-http/service/main.py

@@ -5,6 +5,7 @@
 final container.
 """
 import logging
+from service._ulimit import configure as _configure_ulimit
 from func_python.http import serve
 
 logging.basicConfig(level=logging.INFO)
@@ -19,5 +20,6 @@
         raise
 
 if __name__ == "__main__":
+    _configure_ulimit()
     logging.info("Functions middleware invoking user function")
     serve(handler)

templates/python/scaffolding/instanced-http/tests/test_ulimit.py

@@ -0,0 +1,106 @@
+"""
+Tests for the open-file limit helper in the HTTP scaffolding.
+"""
+import sys
+import importlib
+import logging
+from unittest.mock import patch, MagicMock
+
+# Ensure the service package is importable from the scaffolding root.
+import os
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from service._ulimit import configure
+
+
+def test_raises_soft_limit_to_hard():
+    """configure() should raise the soft limit to the hard limit when soft < hard."""
+    mock_resource = MagicMock()
+    mock_resource.getrlimit.return_value = (1024, 65536)
+    mock_resource.RLIMIT_NOFILE = 7  # canonical Linux value
+    mock_resource.RLIM_INFINITY = 9223372036854775807  # Linux RLIM_INFINITY
+
+    with patch.dict(sys.modules, {"resource": mock_resource}):
+        importlib.reload(sys.modules["service._ulimit"])
+        from service._ulimit import configure as _configure
+        _configure()
+
+    mock_resource.setrlimit.assert_called_once_with(
+        mock_resource.RLIMIT_NOFILE, (65536, 65536)
+    )
+
+
+def test_no_change_when_soft_equals_hard():
+    """configure() should not call setrlimit when soft == hard."""
+    mock_resource = MagicMock()
+    mock_resource.getrlimit.return_value = (65536, 65536)
+    mock_resource.RLIMIT_NOFILE = 7
+    mock_resource.RLIM_INFINITY = 9223372036854775807  # Linux RLIM_INFINITY
+
+    with patch.dict(sys.modules, {"resource": mock_resource}):
+        importlib.reload(sys.modules["service._ulimit"])
+        from service._ulimit import configure as _configure
+        _configure()
+
+    mock_resource.setrlimit.assert_not_called()
+
+
+def test_rlim_infinity_capped_at_max():
+    """configure() should cap the target at _MAX_NOFILE when hard == RLIM_INFINITY."""
+    mock_resource = MagicMock()
+    mock_resource.RLIMIT_NOFILE = 7
+    mock_resource.RLIM_INFINITY = 9223372036854775807  # Linux RLIM_INFINITY
+    # Soft is below the safe max; hard is unlimited.
+    mock_resource.getrlimit.return_value = (1024, mock_resource.RLIM_INFINITY)
+
+    with patch.dict(sys.modules, {"resource": mock_resource}):
+        importlib.reload(sys.modules["service._ulimit"])
+        from service._ulimit import configure as _configure
+        _configure()
+
+    mock_resource.setrlimit.assert_called_once_with(
+        mock_resource.RLIMIT_NOFILE, (65536, mock_resource.RLIM_INFINITY)
+    )
+
+
+def test_import_error_is_silently_skipped():
+    """configure() should do nothing when resource module is unavailable (non-Unix)."""
+    with patch.dict(sys.modules, {"resource": None}):
+        importlib.reload(sys.modules["service._ulimit"])
+        from service._ulimit import configure as _configure
+        # Must not raise.
+        _configure()
+
+
+def test_os_error_logs_warning(caplog):
+    """configure() should log a warning and not propagate an OSError."""
+    mock_resource = MagicMock()
+    mock_resource.getrlimit.return_value = (1024, 65536)
+    mock_resource.RLIMIT_NOFILE = 7
+    mock_resource.RLIM_INFINITY = 9223372036854775807  # Linux RLIM_INFINITY
+    mock_resource.setrlimit.side_effect = OSError("operation not permitted")
+
+    with patch.dict(sys.modules, {"resource": mock_resource}):
+        importlib.reload(sys.modules["service._ulimit"])
+        from service._ulimit import configure as _configure
+        with caplog.at_level(logging.WARNING):
+            _configure()  # must not raise
+
+    assert "Could not raise open-file limit" in caplog.text
+
+
+def test_value_error_logs_warning(caplog):
+    """configure() should log a warning and not propagate a ValueError."""
+    mock_resource = MagicMock()
+    mock_resource.getrlimit.return_value = (1024, 65536)
+    mock_resource.RLIMIT_NOFILE = 7
+    mock_resource.RLIM_INFINITY = 9223372036854775807  # Linux RLIM_INFINITY
+    mock_resource.setrlimit.side_effect = ValueError("invalid argument")
+
+    with patch.dict(sys.modules, {"resource": mock_resource}):
+        importlib.reload(sys.modules["service._ulimit"])
+        from service._ulimit import configure as _configure
+        with caplog.at_level(logging.WARNING):
+            _configure()  # must not raise
+
+    assert "Could not raise open-file limit" in caplog.text