diff --git a/.gitignore b/.gitignore index 9aa5b22..c005a0b 100644 --- a/.gitignore +++ b/.gitignore @@ -93,3 +93,9 @@ ENV/ # PyCharm .idea/ + +# VSCode +.vscode/ + +# mypy +.mypy_cache/ diff --git a/setup.py b/setup.py index 8df52bb..37c2004 100755 --- a/setup.py +++ b/setup.py @@ -15,7 +15,8 @@ 'simplejson==3.11.1', 'python-dateutil>=2.6.0', 'backoff==1.8.0', - 'ciso8601', + 'ciso8601', + 'typing-extensions' ], extras_require={ 'dev': [ diff --git a/singer/__init__.py b/singer/__init__.py index d3b2c87..76d456a 100644 --- a/singer/__init__.py +++ b/singer/__init__.py @@ -61,17 +61,7 @@ ) from singer.schema import Schema -from singer.bookmarks import ( - write_bookmark, - get_bookmark, - clear_bookmark, - reset_stream, - set_offset, - clear_offset, - get_offset, - set_currently_syncing, - get_currently_syncing, -) +from singer.bookmarks import State if __name__ == "__main__": import doctest diff --git a/singer/bookmarks.py b/singer/bookmarks.py index fc6d7ca..28ac502 100644 --- a/singer/bookmarks.py +++ b/singer/bookmarks.py @@ -1,46 +1,84 @@ -def ensure_bookmark_path(state, path): - submap = state - for path_component in path: - if submap.get(path_component) is None: - submap[path_component] = {} - - submap = submap[path_component] - return state - -def write_bookmark(state, tap_stream_id, key, val): - state = ensure_bookmark_path(state, ['bookmarks', tap_stream_id]) - state['bookmarks'][tap_stream_id][key] = val - return state - -def clear_bookmark(state, tap_stream_id, key): - state = ensure_bookmark_path(state, ['bookmarks', tap_stream_id]) - state['bookmarks'][tap_stream_id].pop(key, None) - return state - -def reset_stream(state, tap_stream_id): - state = ensure_bookmark_path(state, ['bookmarks', tap_stream_id]) - state['bookmarks'][tap_stream_id] = {} - return state - -def get_bookmark(state, tap_stream_id, key, default=None): - return state.get('bookmarks', {}).get(tap_stream_id, {}).get(key, default) - -def set_offset(state, tap_stream_id, offset_key, offset_value): - state = ensure_bookmark_path(state, ['bookmarks', tap_stream_id, "offset", offset_key]) - state['bookmarks'][tap_stream_id]["offset"][offset_key] = offset_value - return state - -def clear_offset(state, tap_stream_id): - state = ensure_bookmark_path(state, ['bookmarks', tap_stream_id, "offset"]) - state['bookmarks'][tap_stream_id]["offset"] = {} - return state - -def get_offset(state, tap_stream_id, default=None): - return state.get('bookmarks', {}).get(tap_stream_id, {}).get("offset", default) - -def set_currently_syncing(state, tap_stream_id): - state['currently_syncing'] = tap_stream_id - return state - -def get_currently_syncing(state, default=None): - return state.get('currently_syncing', default) +from typing import Any, Dict, Optional, Sequence, Union +from .logger import get_logger + + +LOGGER = get_logger() + + +class State: + def __init__( + self, bookmarks: Optional[Dict] = None, currently_syncing: Optional[str] = None # pylint: disable=bad-continuation + ) -> None: + self._bookmarks = bookmarks or {} + self._currently_syncing = currently_syncing + + def __str__(self) -> str: + return str(self.__dict__) + + def __eq__(self, other: Any) -> bool: + return self.__dict__ == other.__dict__ + + @property + def bookmarks(self) -> Dict: + return self._bookmarks + + @classmethod + def from_dict(cls, data: Dict) -> "State": + return State( + bookmarks=data.get("bookmarks"), + currently_syncing=data.get("currently_syncing"), + ) + + def to_dict(self) -> Dict: + state = {"bookmarks": self.bookmarks} # type: Dict[str, Any] + if self.get_currently_syncing(): + state["currently_syncing"] = self.get_currently_syncing() + return state + + def _ensure_bookmark_path(self, path: Sequence) -> None: + submap = self.bookmarks + for path_component in path: + if submap.get(path_component) is None: + submap[path_component] = {} + + submap = submap[path_component] + + def write_bookmark(self, tap_stream_id: str, key: str, val: Any) -> None: + self._ensure_bookmark_path((tap_stream_id,)) + self.bookmarks[tap_stream_id][key] = val + + def clear_bookmark(self, tap_stream_id: str, key: str) -> None: + self._ensure_bookmark_path((tap_stream_id,)) + self.bookmarks[tap_stream_id].pop(key, None) + + def reset_stream(self, tap_stream_id: str) -> None: + self._ensure_bookmark_path((tap_stream_id,)) + self.bookmarks[tap_stream_id] = {} + + def get_bookmark(self, tap_stream_id: str, key: str, default: Any = None) -> Any: + return self.bookmarks.get(tap_stream_id, {}).get(key, default) + + def set_offset( + self, tap_stream_id: str, offset_key: str, offset_value: Any # pylint: disable=bad-continuation + ) -> None: + self._ensure_bookmark_path((tap_stream_id, "offset", offset_key)) + self.bookmarks[tap_stream_id]["offset"][offset_key] = offset_value + + def clear_offset(self, tap_stream_id: str) -> None: + self._ensure_bookmark_path((tap_stream_id, "offset")) + self.bookmarks[tap_stream_id]["offset"] = {} + + def get_offset( + self, tap_stream_id: str, offset_key: str, default: Any = None # pylint: disable=bad-continuation + ) -> Any: + return ( + self.bookmarks.get(tap_stream_id, {}) + .get("offset", {}) + .get(offset_key, default) + ) + + def get_currently_syncing(self, default: Optional[str] = None) -> Optional[str]: + return self._currently_syncing or default + + def set_currently_syncing(self, value: Union[str, None]) -> None: + self._currently_syncing = value diff --git a/singer/catalog.py b/singer/catalog.py index 1767ff1..47afa0b 100644 --- a/singer/catalog.py +++ b/singer/catalog.py @@ -3,7 +3,6 @@ import sys from . import metadata as metadata_module -from .bookmarks import get_currently_syncing from .logger import get_logger from .schema import Schema @@ -132,7 +131,7 @@ def get_stream(self, tap_stream_id): return None def _shuffle_streams(self, state): - currently_syncing = get_currently_syncing(state) + currently_syncing = state.get_currently_syncing() if currently_syncing is None: return self.streams diff --git a/singer/utils.py b/singer/utils.py index 85f3d39..61b150c 100644 --- a/singer/utils.py +++ b/singer/utils.py @@ -10,6 +10,7 @@ import pytz import backoff as backoff_module +from singer.bookmarks import State from singer.catalog import Catalog DATETIME_PARSE = "%Y-%m-%dT%H:%M:%SZ" @@ -169,9 +170,9 @@ def parse_args(required_config_keys): args.config = load_json(args.config) if args.state: setattr(args, 'state_path', args.state) - args.state = load_json(args.state) + args.state = State.from_dict(load_json(args.state)) else: - args.state = {} + args.state = State() if args.properties: setattr(args, 'properties_path', args.properties) args.properties = load_json(args.properties) diff --git a/tests/test_bookmarks.py b/tests/test_bookmarks.py index 4902105..59846f5 100644 --- a/tests/test_bookmarks.py +++ b/tests/test_bookmarks.py @@ -1,166 +1,463 @@ +from copy import copy import unittest -from singer import bookmarks +from singer.bookmarks import State + class TestGetBookmark(unittest.TestCase): def test_empty_state(self): - empty_state = {} + empty_state = State() # Case with no value to fall back on - self.assertIsNone(bookmarks.get_bookmark(empty_state, 'some_stream', 'my_key')) + self.assertIsNone(empty_state.get_bookmark("some_stream", "my_key")) # Case with a given default - self.assertEqual(bookmarks.get_bookmark(empty_state, 'some_stream', 'my_key', 'default_value'), - 'default_value') + self.assertEqual( + empty_state.get_bookmark("some_stream", "my_key", "default_value"), + "default_value", + ) def test_empty_bookmark(self): - empty_bookmark = {'bookmarks':{}} + empty_bookmark = State(bookmarks={}) # Case with no value to fall back on - self.assertIsNone(bookmarks.get_bookmark(empty_bookmark, 'some_stream', 'my_key')) + self.assertIsNone(empty_bookmark.get_bookmark("some_stream", "my_key")) # Case with a given default - self.assertEqual(bookmarks.get_bookmark(empty_bookmark, 'some_stream', 'my_key', 'default_value'), - 'default_value') + self.assertEqual( + empty_bookmark.get_bookmark("some_stream", "my_key", "default_value"), + "default_value", + ) def test_non_empty_state(self): - stream_id_1 = 'customers' - bookmark_key_1 = 'datetime' + stream_id_1 = "customers" + bookmark_key_1 = "datetime" bookmark_val_1 = 123456789 + bookmarks = {stream_id_1: {bookmark_key_1: bookmark_val_1}} - non_empty_state = { - 'bookmarks' : { - stream_id_1 : { - bookmark_key_1 : bookmark_val_1 - } - } - } + non_empty_state = State(bookmarks=bookmarks) # # Cases with no value to fall back on # # Bad stream, bad key - self.assertIsNone(bookmarks.get_bookmark(non_empty_state, 'some_stream', 'my_key')) + self.assertIsNone(non_empty_state.get_bookmark("some_stream", "my_key")) # Good stream, bad key - self.assertIsNone(bookmarks.get_bookmark(non_empty_state, stream_id_1, 'my_key')) + self.assertIsNone(non_empty_state.get_bookmark(stream_id_1, "my_key")) # Good stream, good key - self.assertEqual(bookmarks.get_bookmark(non_empty_state, stream_id_1, bookmark_key_1), - bookmark_val_1) + self.assertEqual( + non_empty_state.get_bookmark(stream_id_1, bookmark_key_1), bookmark_val_1 + ) # # Cases with a given default # # Bad stream, bad key - self.assertEqual(bookmarks.get_bookmark(non_empty_state, 'some_stream', 'my_key', 'default_value'), - 'default_value') + self.assertEqual( + non_empty_state.get_bookmark("some_stream", "my_key", "default_value"), + "default_value", + ) # Bad stream, good key - self.assertEqual(bookmarks.get_bookmark(non_empty_state, 'some_stream', bookmark_key_1, 'default_value'), - 'default_value') + self.assertEqual( + non_empty_state.get_bookmark( + "some_stream", bookmark_key_1, "default_value" + ), + "default_value", + ) + + # Good stream, bad key + self.assertEqual( + non_empty_state.get_bookmark(stream_id_1, "my_key", "default_value"), + "default_value", + ) + + # Good stream, good key + self.assertEqual( + non_empty_state.get_bookmark(stream_id_1, bookmark_key_1, "default_value"), + bookmark_val_1, + ) + + +class TestWriteBookmark(unittest.TestCase): + def test_empty_state(self): + empty_state = State() + self.assertIsNone(empty_state.get_bookmark("some_stream", "my_key")) + empty_state.write_bookmark("some_stream", "my_key", "val") + self.assertEqual(empty_state.get_bookmark("some_stream", "my_key"), "val") + + def test_empty_bookmark(self): + empty_bookmark = State(bookmarks={}) + self.assertIsNone(empty_bookmark.get_bookmark("some_stream", "my_key")) + empty_bookmark.write_bookmark("some_stream", "my_key", "val") + self.assertEqual(empty_bookmark.get_bookmark("some_stream", "my_key"), "val") + + def test_non_empty_state(self): + stream_id_1 = "customers" + bookmark_key_1 = "datetime" + bookmark_val_1 = 123456789 + bookmark_val_2 = 0 + bookmarks = {stream_id_1: {bookmark_key_1: bookmark_val_1}} + + non_empty_state = State(bookmarks=bookmarks) + + self.assertEqual( + non_empty_state.get_bookmark(stream_id_1, bookmark_key_1), bookmark_val_1 + ) + non_empty_state.write_bookmark(stream_id_1, bookmark_key_1, bookmark_val_2) + self.assertEqual( + non_empty_state.get_bookmark(stream_id_1, bookmark_key_1), bookmark_val_2 + ) + + +class TestClearBookmark(unittest.TestCase): + def test_empty_state(self): + empty_state = State() + empty_state.clear_bookmark("some_stream", "key") + self.assertIsNone(empty_state.get_bookmark("some_stream", "my_key")) + + def test_empty_bookmark(self): + empty_bookmark = State(bookmarks={}) + empty_bookmark.clear_bookmark("some_stream", "key") + self.assertIsNone(empty_bookmark.get_bookmark("some_stream", "my_key")) + + def test_non_empty_state(self): + stream_id_1 = "customers" + bookmark_key_1 = "datetime" + bookmark_val_1 = 123456789 + bookmarks = {stream_id_1: {bookmark_key_1: bookmark_val_1}} + + # + # Cases with no value to fall back on + # + + # Bad stream, bad key + non_empty_state = State(bookmarks=bookmarks) + self.assertEqual( + non_empty_state.get_bookmark(stream_id_1, bookmark_key_1), bookmark_val_1 + ) + non_empty_state.clear_bookmark("some_stream", "some_key") + self.assertEqual( + non_empty_state.get_bookmark(stream_id_1, bookmark_key_1), bookmark_val_1 + ) # Good stream, bad key - self.assertEqual(bookmarks.get_bookmark(non_empty_state, stream_id_1, 'my_key', 'default_value'), - 'default_value') + non_empty_state = State(bookmarks=bookmarks) + self.assertEqual( + non_empty_state.get_bookmark(stream_id_1, bookmark_key_1), bookmark_val_1 + ) + non_empty_state.clear_bookmark(stream_id_1, "some_key") + self.assertEqual( + non_empty_state.get_bookmark(stream_id_1, bookmark_key_1), bookmark_val_1 + ) # Good stream, good key - self.assertEqual(bookmarks.get_bookmark(non_empty_state, stream_id_1, bookmark_key_1, 'default_value'), - bookmark_val_1) + non_empty_state = State(bookmarks=bookmarks) + self.assertEqual( + non_empty_state.get_bookmark(stream_id_1, bookmark_key_1), bookmark_val_1 + ) + non_empty_state.clear_bookmark(stream_id_1, bookmark_key_1) + self.assertIsNone(non_empty_state.get_bookmark(stream_id_1, bookmark_key_1)) + + +class TestClearStream(unittest.TestCase): + def test_empty_state(self): + empty_state = State() + self.assertFalse("some_stream" in empty_state.bookmarks) + empty_state.reset_stream("some_stream") + self.assertTrue("some_stream" in empty_state.bookmarks) + self.assertIsNone(empty_state.bookmarks["some_stream"] or None) + + def test_empty_bookmark(self): + empty_bookmark = State(bookmarks={}) + self.assertFalse("some_stream" in empty_bookmark.bookmarks) + empty_bookmark.reset_stream("some_stream") + self.assertTrue("some_stream" in empty_bookmark.bookmarks) + self.assertIsNone(empty_bookmark.bookmarks["some_stream"] or None) + + def test_non_empty_state(self): + stream_id_1 = "customers" + bookmark_key_1 = "datetime" + bookmark_val_1 = 123456789 + bookmarks = {stream_id_1: {bookmark_key_1: bookmark_val_1}} + + non_empty_state = State(bookmarks=bookmarks) + + self.assertEqual( + non_empty_state.get_bookmark(stream_id_1, bookmark_key_1), bookmark_val_1 + ) + non_empty_state.reset_stream(stream_id_1) + self.assertTrue(stream_id_1 in non_empty_state.bookmarks) + self.assertIsNone(non_empty_state.get_bookmark(stream_id_1, bookmark_key_1)) class TestGetOffset(unittest.TestCase): def test_empty_state(self): - empty_state = {} + empty_state = State() # Case with no value to fall back on - self.assertIsNone(bookmarks.get_offset(empty_state, 'some_stream')) + self.assertIsNone(empty_state.get_offset("some_stream", "offset_key")) # Case with a given default - self.assertEqual(bookmarks.get_offset(empty_state, 'some_stream', 'default_value'), - 'default_value') + self.assertEqual( + empty_state.get_offset("some_stream", "offset_key", "default_value"), + "default_value", + ) def test_empty_bookmark(self): - empty_bookmark = {'bookmarks':{}} + empty_bookmark = State(bookmarks={}) # Case with no value to fall back on - self.assertIsNone(bookmarks.get_offset(empty_bookmark, 'some_stream')) + self.assertIsNone(empty_bookmark.get_offset("some_stream", "offset_key")) # Case with a given default - self.assertEqual(bookmarks.get_offset(empty_bookmark, 'some_stream', 'default_value'), - 'default_value') + self.assertEqual( + empty_bookmark.get_offset("some_stream", "offset_key", "default_value"), + "default_value", + ) def test_non_empty_state(self): - stream_id_1 = 'customers' - bookmark_key_1 = 'datetime' + stream_id_1 = "customers" + bookmark_key_1 = "datetime" bookmark_val_1 = 123456789 - offset_val = 'fizzy water' - - non_empty_state = { - 'bookmarks' : { - stream_id_1 : { - bookmark_key_1 : bookmark_val_1, - 'offset' : offset_val - } + offset_key = "key" + offset_val = "fizzy water" + + bookmarks = { + stream_id_1: { + bookmark_key_1: bookmark_val_1, + "offset": {offset_key: offset_val}, } } + non_empty_state = State(bookmarks=bookmarks) + # # Cases with no value to fall back on # # Bad stream - self.assertIsNone(bookmarks.get_offset(non_empty_state, 'some_stream')) + self.assertIsNone(non_empty_state.get_offset("some_stream", offset_key)) # Good stream - self.assertEqual(bookmarks.get_offset(non_empty_state, stream_id_1), - offset_val) + self.assertEqual( + non_empty_state.get_offset(stream_id_1, offset_key), offset_val + ) # # Case with a given default # # Bad stream - self.assertEqual(bookmarks.get_offset(non_empty_state, 'some_stream', 'default_value'), - 'default_value') + self.assertEqual( + non_empty_state.get_offset("some_stream", offset_key, "default_value"), + "default_value", + ) + + # Good stream + self.assertEqual( + non_empty_state.get_offset(stream_id_1, offset_key, "default_value"), + offset_val, + ) + + +class TestClearOffset(unittest.TestCase): + def test_empty_state(self): + empty_state = State() + empty_state.clear_offset("some_stream") + self.assertIsNone(empty_state.get_offset("some_stream", "offset_key")) + + def test_empty_bookmark(self): + empty_bookmark = State(bookmarks={}) + empty_bookmark.clear_offset("some_stream") + self.assertIsNone(empty_bookmark.get_offset("some_stream", "offset_key")) + + def test_non_empty_state(self): + stream_id_1 = "customers" + bookmark_key_1 = "datetime" + bookmark_val_1 = 123456789 + offset_key = "key" + offset_val = "fizzy water" + + bookmarks = { + stream_id_1: { + bookmark_key_1: bookmark_val_1, + "offset": {offset_key: offset_val}, + } + } + + non_empty_state = State(bookmarks=bookmarks) + + # Bad stream + self.assertEqual( + non_empty_state.get_offset(stream_id_1, offset_key), offset_val + ) + non_empty_state.clear_offset("some_stream") + self.assertEqual( + non_empty_state.get_offset(stream_id_1, offset_key), offset_val + ) # Good stream - self.assertEqual(bookmarks.get_offset(non_empty_state, stream_id_1, 'default_value'), - offset_val) + self.assertEqual( + non_empty_state.get_offset(stream_id_1, offset_key), offset_val + ) + non_empty_state.clear_offset(stream_id_1) + self.assertIsNone(non_empty_state.get_offset(stream_id_1, offset_key)) + + +class TestSetOffset(unittest.TestCase): + def test_empty_state(self): + empty_state = State() + empty_state.set_offset("some_stream", "offset_key", "offset_value") + self.assertEqual( + empty_state.get_offset("some_stream", "offset_key"), "offset_value" + ) + + def test_empty_bookmark(self): + empty_bookmark = State(bookmarks={}) + empty_bookmark.set_offset("some_stream", "offset_key", "offset_value") + self.assertEqual( + empty_bookmark.get_offset("some_stream", "offset_key"), "offset_value" + ) + + def test_non_empty_state(self): + stream_id_1 = "customers" + bookmark_key_1 = "datetime" + bookmark_val_1 = 123456789 + offset_key_1 = "offset_key_1" + offset_key_2 = "offset_key_2" + offset_val_1 = "fizzy water" + offset_val_2 = "still water" + + bookmarks = { + stream_id_1: { + bookmark_key_1: bookmark_val_1, + "offset": {"offset_key_1": offset_val_1,}, + } + } + + non_empty_state = State(bookmarks=bookmarks) + + # Test setting new key + non_empty_state.set_offset(stream_id_1, offset_key_2, offset_val_2) + self.assertEqual( + non_empty_state.get_offset(stream_id_1, offset_key_1), offset_val_1 + ) + self.assertEqual( + non_empty_state.get_offset(stream_id_1, offset_key_2), offset_val_2 + ) + + # Test overwriting key + non_empty_state.set_offset(stream_id_1, offset_key_1, offset_val_2) + self.assertEqual( + non_empty_state.get_offset(stream_id_1, offset_key_1), offset_val_2 + ) + self.assertEqual( + non_empty_state.get_offset(stream_id_1, offset_key_2), offset_val_2 + ) class TestGetCurrentlySyncing(unittest.TestCase): def test_empty_state(self): - empty_state = {} + empty_state = State() # Case with no value to fall back on - self.assertIsNone(bookmarks.get_currently_syncing(empty_state)) + self.assertIsNone(empty_state.get_currently_syncing()) # Case with a given default - self.assertEqual(bookmarks.get_currently_syncing(empty_state, 'default_value'), - 'default_value') + self.assertEqual( + empty_state.get_currently_syncing("default_value"), "default_value" + ) def test_non_empty_state(self): - stream_id_1 = 'customers' - bookmark_key_1 = 'datetime' + stream_id_1 = "customers" + bookmark_key_1 = "datetime" bookmark_val_1 = 123456789 - offset_val = 'fizzy water' - - non_empty_state = { - 'bookmarks' : { - stream_id_1 : { - bookmark_key_1 : bookmark_val_1, - 'offset' : offset_val - } - }, - 'currently_syncing' : stream_id_1 + offset_val = "fizzy water" + + bookmarks = { + stream_id_1: {bookmark_key_1: bookmark_val_1, "offset": offset_val} } + non_empty_state = State(bookmarks=bookmarks, currently_syncing=stream_id_1) + # Case with no value to fall back on - self.assertEqual(bookmarks.get_currently_syncing(non_empty_state), - stream_id_1) + self.assertEqual(non_empty_state.get_currently_syncing(), stream_id_1) - # Case with a given default - self.assertEqual(bookmarks.get_currently_syncing(non_empty_state, 'default_value'), - stream_id_1) + # Case with no value to fall back on + self.assertEqual( + non_empty_state.get_currently_syncing("default_value"), stream_id_1 + ) + + +class TestSetCurrentlySyncing(unittest.TestCase): + def test_empty_state(self): + empty_state = State() + + self.assertIsNone(empty_state.get_currently_syncing()) + empty_state.set_currently_syncing("some_stream") + self.assertEqual(empty_state.get_currently_syncing(), "some_stream") + + def test_non_empty_state(self): + stream_id_1 = "customers" + bookmark_key_1 = "datetime" + bookmark_val_1 = 123456789 + offset_key = "key" + offset_val = "fizzy water" + + bookmarks = { + stream_id_1: { + bookmark_key_1: bookmark_val_1, + "offset": {offset_key: offset_val}, + } + } + + non_empty_state = State(bookmarks=bookmarks, currently_syncing=stream_id_1) + + self.assertEqual(non_empty_state.get_currently_syncing(), stream_id_1) + + non_empty_state.set_currently_syncing("some_stream") + self.assertEqual(non_empty_state.get_currently_syncing(), "some_stream") + + +class TestToDictAndFromDict(unittest.TestCase): + + bookmarks = { + "stream_1": {"stream_1_key_1": 1, "stream_1_key_2": "2019-02-01T00:00:00Z"}, + "stream_2": { + "stream_2_key_1": 2, + "stream_2_key_2": "2019-03-01T00:00:00Z", + "offset": {"offset_1": 1}, + }, + } + + dict_form = {"bookmarks": bookmarks, "currently_syncing": "stream_1"} + + obj_form = State(bookmarks=bookmarks, currently_syncing="stream_1") + + def test_from_dict(self): + dict_form = self.dict_form.copy() + obj_form = copy(self.obj_form) + + # With currently_syncing + self.assertEqual(obj_form, State.from_dict(dict_form)) + + # # Without currently_syncing + del dict_form["currently_syncing"] + obj_form.set_currently_syncing(None) + self.assertEqual(obj_form, State.from_dict(dict_form)) + + def test_to_dict(self): + dict_form = self.dict_form.copy() + obj_form = copy(self.obj_form) + + # With currently_syncing + self.assertEqual(self.dict_form, self.obj_form.to_dict()) + + # # Without currently_syncing + del dict_form["currently_syncing"] + obj_form.set_currently_syncing(None) + self.assertEqual(self.dict_form, self.obj_form.to_dict()) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index cd6dc50..48746f3 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -1,5 +1,6 @@ import unittest +from singer.bookmarks import State from singer.schema import Schema from singer.catalog import Catalog, CatalogEntry, write_catalog @@ -23,7 +24,7 @@ def test_one_selected_stream(self): [selected_entry, CatalogEntry(tap_stream_id='b',schema=Schema(),metadata=[]), CatalogEntry(tap_stream_id='c',schema=Schema(),metadata=[])]) - state = {} + state = State() selected_streams = catalog.get_selected_streams(state) self.assertEquals([e for e in selected_streams],[selected_entry]) @@ -42,7 +43,7 @@ def test_resumes_currently_syncing_stream(self): [selected_entry_a, CatalogEntry(tap_stream_id='b',schema=Schema(),metadata=[]), selected_entry_c]) - state = {'currently_syncing': 'c'} + state = State(currently_syncing='c') selected_streams = catalog.get_selected_streams(state) self.assertEquals([e for e in selected_streams][0],selected_entry_c)