From 7f2bf9e2dc5078f0257171e27cc32f92a2c904dc Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 5 Sep 2024 12:51:11 +0200 Subject: [PATCH 1/6] either get mp as MerginProject or string with path to it --- dbsync.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbsync.py b/dbsync.py index d36fd6a..6c3ea66 100644 --- a/dbsync.py +++ b/dbsync.py @@ -18,6 +18,7 @@ import re import pathlib import logging +import typing import psycopg2 import psycopg2.extensions @@ -546,8 +547,10 @@ def _get_project_version(work_path) -> str: return mp.version() -def _get_project_id(mp: MerginProject): +def _get_project_id(mp: typing.Union[MerginProject, str]): """Returns the project ID""" + if isinstance(mp, str): + mp = _get_mergin_project(mp) try: project_id = uuid.UUID(mp.project_id()) except ( From 3d28eab9dc1211dbaeff9bb684f29c45d734ef46 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 5 Sep 2024 12:51:34 +0200 Subject: [PATCH 2/6] explicitly convert to string --- dbsync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbsync.py b/dbsync.py index 6c3ea66..5f2cb0e 100644 --- a/dbsync.py +++ b/dbsync.py @@ -577,7 +577,7 @@ def _set_db_project_comment( "version": version, } if project_id: - comment["project_id"] = project_id + comment["project_id"] = str(project_id) if error: comment["error"] = error cur = conn.cursor() From bf101883b2f134fcc52e8a14c3fc609be79d7e04 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 5 Sep 2024 12:52:30 +0200 Subject: [PATCH 3/6] always store project id in db_comment --- dbsync.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/dbsync.py b/dbsync.py index 5f2cb0e..6c505af 100644 --- a/dbsync.py +++ b/dbsync.py @@ -862,12 +862,12 @@ def pull(conn_cfg, mc): os.remove(gpkg_basefile_old) conn = psycopg2.connect(conn_cfg.conn_info) - version = _get_project_version(work_dir) _set_db_project_comment( conn, conn_cfg.base, conn_cfg.mergin_project, - version, + version=_get_project_version(work_dir), + project_id=_get_project_id(work_dir), ) @@ -1072,7 +1072,13 @@ def push(conn_cfg, mc): # update base schema in the DB logging.debug("Updating DB base schema...") _geodiff_apply_changeset(conn_cfg.driver, conn_cfg.conn_info, conn_cfg.base, tmp_changeset_file, ignored_tables) - _set_db_project_comment(conn, conn_cfg.base, conn_cfg.mergin_project, version) + _set_db_project_comment( + conn, + conn_cfg.base, + conn_cfg.mergin_project, + version, + project_id=_get_project_id(work_dir), + ) def init( @@ -1320,6 +1326,7 @@ def init( conn_cfg.base, conn_cfg.mergin_project, local_version, + project_id=_get_project_id(work_dir), ) else: if not modified_schema_exists: @@ -1434,12 +1441,12 @@ def init( mc.push_project(work_dir) # mark project version into db schema - version = _get_project_version(work_dir) _set_db_project_comment( conn, conn_cfg.base, conn_cfg.mergin_project, - version, + version=_get_project_version(work_dir), + project_id=_get_project_id(work_dir), ) From bc55030200fac990e94d119d1ad84ce5ef6d59cf Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 5 Sep 2024 12:53:16 +0200 Subject: [PATCH 4/6] check that project id in DB and project ID are the same, otherwise raise error how to handle situation --- dbsync.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dbsync.py b/dbsync.py index 6c505af..5ef67db 100644 --- a/dbsync.py +++ b/dbsync.py @@ -1157,6 +1157,14 @@ def init( f"Downloading version {db_proj_info['version']} of Mergin Maps project {conn_cfg.mergin_project} " f"to {work_dir}" ) + project_info = mc.project_info(conn_cfg.mergin_project) + if db_proj_info["project_id"] != project_info["id"]: + raise DbSyncError( + "Mergin Maps project ID doesn't match Mergin Maps project ID stored in the database. " + "Did you change configuration from one Mergin Maps project to another? " + f"You either need to remove schema `{conn_cfg.base}` from Database or use `--force-init` option. " + f"{FORCE_INIT_MESSAGE}" + ) mc.download_project(conn_cfg.mergin_project, work_dir, db_proj_info["version"]) else: # Get project ID from DB if available From 373c2c38cc24764d1b494f63fdad693050f6b410 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 5 Sep 2024 12:54:41 +0200 Subject: [PATCH 5/6] better cleanup prior and after tests --- test/conftest.py | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 49f9af5..414d9d1 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -18,7 +18,7 @@ API_USER = os.environ.get("TEST_API_USERNAME") USER_PWD = os.environ.get("TEST_API_PASSWORD") WORKSPACE = os.environ.get("TEST_API_WORKSPACE") -TMP_DIR = tempfile.gettempdir() +TMP_DIR = os.path.join(tempfile.gettempdir(), "dbsync_test") TEST_DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "test_data", @@ -167,7 +167,7 @@ def init_sync_from_geopackage(mc, project_name, source_gpkg_path, ignored_tables dbsync_init(mc) -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def mc(): assert SERVER_URL and API_USER and USER_PWD # assert SERVER_URL and SERVER_URL.rstrip('/') != 'https://app.merginmaps.com/' and API_USER and USER_PWD @@ -299,3 +299,37 @@ def init_sync_from_db(mc: MerginClient, project_name: str, path_sql_file: str, i ) dbsync_init(mc) + + +def _clean_workspace(mc: MerginClient, workspace: str) -> None: + """Immediately delete all projects in the test workspace.""" + + projects = mc.projects_list(only_namespace=workspace) + + for project in projects: + mc.delete_project_now(f"{workspace}/{project['name']}") + + +@pytest.fixture(autouse=True, scope="session") +def clean_workspace(mc: MerginClient): + """Delete all projects in the test workspace prior and after test session.""" + + _clean_workspace(mc, WORKSPACE) + yield + _clean_workspace(mc, WORKSPACE) + + +def _remove_dir(dir_path: str) -> None: + """Remove directory if it exists.""" + + if os.path.exists(dir_path): + shutil.rmtree(dir_path) + + +@pytest.fixture(autouse=True, scope="session") +def clean_test_dir(): + """Remove and recreate temporary directory for test files. Remove it after test session.""" + _remove_dir(TMP_DIR) + os.makedirs(TMP_DIR) + yield + _remove_dir(TMP_DIR) From 2ead57ec5fde006b7ff879e7965d6fdd7c877d26 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 5 Sep 2024 12:56:11 +0200 Subject: [PATCH 6/6] test that changing project raises correct error --- test/test_init_db.py | 64 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/test/test_init_db.py b/test/test_init_db.py index bf784f2..d395f54 100644 --- a/test/test_init_db.py +++ b/test/test_init_db.py @@ -13,7 +13,7 @@ MerginClient, ) -from dbsync import dbsync_pull, dbsync_push, config, DbSyncError +from dbsync import dbsync_pull, dbsync_push, config, DbSyncError, dbsync_init from .conftest import ( GEODIFF_EXE, @@ -148,3 +148,65 @@ def test_missing_table(mc: MerginClient): init_sync_from_db(mc, project_name, path_test_data("create_another_schema.sql")) assert "The 'modified' schema does not exist" in str(err.value) + + +def test_mm_project_change(mc: MerginClient, db_connection): + """Test that after init and local changes the changes are correctly pushed to database""" + project_name = "test_project_change" + project_full_name = complete_project_name(project_name) + project_dir = name_project_dir(project_name) + db_schema_main = "test_init_from_db_main" + db_schema_base = "test_init_from_db_base" + + path_synced_gpkg = project_dir + "/" + filename_sync_gpkg() + + init_sync_from_db(mc, project_name, path_test_data("create_base.sql")) + + cur = db_connection.cursor() + + # check that there are 3 features prior to changes + cur.execute(f'SELECT COUNT(*) from {db_schema_main}."simple"') + assert cur.fetchone()[0] == 3 + + mc.download_project(project_full_name, project_dir) + + # make changes in GPKG to create new version of the project + shutil.copy(path_test_data("inserted_point_from_db.gpkg"), path_synced_gpkg) + + # push project + mc.push_project(project_dir) + + # run sync + dbsync_pull(mc) + dbsync_push(mc) + + # check that new feature was added + cur.execute(f'SELECT COUNT(*) from {db_schema_main}."simple"') + assert cur.fetchone()[0] == 4 + + project_name = "test_project_change_2" + project_full_name = complete_project_name(project_name) + project_dir = name_project_dir(project_name) + mc.create_project_and_push(project_full_name, project_dir) + + # change config to new project + config.update( + { + "CONNECTIONS": [ + { + "driver": "postgres", + "conn_info": DB_CONNINFO, + "modified": db_schema_main, + "base": db_schema_base, + "mergin_project": project_full_name, + "sync_file": filename_sync_gpkg(), + } + ] + } + ) + + # run init + with pytest.raises( + DbSyncError, match="Mergin Maps project ID doesn't match Mergin Maps project ID stored in the database" + ): + dbsync_init(mc)