Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions integration_tests/s3/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ def put_object_into_storage(storage, path, contents):
"s3": put_aws_object,
"minio": put_minio_object,
}
if storage not in put_object_methods:
raise ValueError(f"Storage type '{storage}' unsupported in tests")
return put_object_methods[storage](path, contents)


Expand All @@ -87,6 +89,8 @@ def delete_object_from_storage(storage, path):
"s3": delete_aws_object,
"minio": delete_minio_object,
}
if storage not in delete_object_methods:
raise ValueError(f"Storage type '{storage}' unsupported in tests")
return delete_object_methods[storage](path)


Expand Down
15 changes: 14 additions & 1 deletion integration_tests/s3/test_s3_generic.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,13 @@
from pathway.internals.parse_graph import G
from pathway.tests.utils import get_aws_s3_settings, write_lines

from .base import create_jsonlines, put_aws_object, read_jsonlines_fields
from .base import (
create_jsonlines,
delete_object_from_storage,
put_aws_object,
put_object_into_storage,
read_jsonlines_fields,
)


@pytest.mark.parametrize(
Expand Down Expand Up @@ -375,6 +381,13 @@ def test_s3_objects_filter(tmp_path: pathlib.Path, s3_path: str):
)


def test_unsupported_storage_type_raises_valueerror():
with pytest.raises(ValueError, match="Storage type 'unsupported' unsupported in tests"):
put_object_into_storage("unsupported", "path", "contents")
with pytest.raises(ValueError, match="Storage type 'unsupported' unsupported in tests"):
delete_object_from_storage("unsupported", "path")


def test_s3_objects_filter_complex_path(tmp_path: pathlib.Path, s3_path: str):
input_s3_path_csv = f"{s3_path}/one/two/three/input.csv"
input_s3_path_json = f"{s3_path}/one/three/input.csv"
Expand Down