From 2d657c9eac71cd0181b637b58b0c55a5de4a7b76 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 10 Jul 2024 10:02:32 +0200 Subject: [PATCH 01/18] feat(migrate): add extra migrations and flyway maven plugin to upgrade from 4.20 to 6.3 database --- pom.xml | 31 +++++++++++++++++++ .../V5.0.0.0__fzj-drop-uri-fieldtype.sql | 2 ++ .../db/extra/V5.1.1.0__6919-prepare-table.sql | 21 +++++++++++++ .../db/extra/V5.4.1.0__7400-prepare-table.sql | 15 +++++++++ .../db/extra/V5.9.0.0__7440-prepare-table.sql | 21 +++++++++++++ .../extra/V5.9.0.2__7440-reinject-cc-by.sql | 8 +++++ .../db/extra/V6.0.0.0__9763-prepare-table.sql | 17 ++++++++++ .../db/extra/V6.1.0.0__9983-prepare-table.sql | 12 +++++++ 8 files changed, 127 insertions(+) create mode 100644 src/main/resources/db/extra/V5.0.0.0__fzj-drop-uri-fieldtype.sql create mode 100644 src/main/resources/db/extra/V5.1.1.0__6919-prepare-table.sql create mode 100644 src/main/resources/db/extra/V5.4.1.0__7400-prepare-table.sql create mode 100644 src/main/resources/db/extra/V5.9.0.0__7440-prepare-table.sql create mode 100644 src/main/resources/db/extra/V5.9.0.2__7440-reinject-cc-by.sql create mode 100644 src/main/resources/db/extra/V6.0.0.0__9763-prepare-table.sql create mode 100644 src/main/resources/db/extra/V6.1.0.0__9983-prepare-table.sql diff --git a/pom.xml b/pom.xml index c8971fb6587..d04ea658da8 100644 --- a/pom.xml +++ b/pom.xml @@ -850,6 +850,37 @@ + + + org.flywaydb + flyway-maven-plugin + ${flyway.version} + + + org.postgresql + postgresql + ${postgresql.version} + + + + + jdbc:postgresql://localhost:54321/dataverse + dataverse + secret + __ + + filesystem:src/main/resources/db/migration + filesystem:src/main/resources/db/extra + + + + org.apache.maven.plugins maven-compiler-plugin diff --git a/src/main/resources/db/extra/V5.0.0.0__fzj-drop-uri-fieldtype.sql b/src/main/resources/db/extra/V5.0.0.0__fzj-drop-uri-fieldtype.sql new file mode 100644 index 00000000000..f97f76424f4 --- /dev/null +++ b/src/main/resources/db/extra/V5.0.0.0__fzj-drop-uri-fieldtype.sql @@ -0,0 +1,2 @@ +-- Replace the URI_NO field type with TEXT to be upstream compatible again +UPDATE datasetfieldtype SET fieldtype = 'TEXT' WHERE name = 'storage_location'; \ No newline at end of file diff --git a/src/main/resources/db/extra/V5.1.1.0__6919-prepare-table.sql b/src/main/resources/db/extra/V5.1.1.0__6919-prepare-table.sql new file mode 100644 index 00000000000..f6161ddfdd7 --- /dev/null +++ b/src/main/resources/db/extra/V5.1.1.0__6919-prepare-table.sql @@ -0,0 +1,21 @@ +-- This is a workaround for the missing DDL statements in migration V5.1.1.2 + +CREATE TABLE IF NOT EXISTS externaltooltype +( + id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, + type VARCHAR(255) NOT NULL, + externalTool_id BIGINT NOT NULL, + CONSTRAINT pk_externaltooltype PRIMARY KEY (id) +); + +ALTER TABLE externaltooltype + ADD CONSTRAINT fk_externaltooltype_externaltool_id FOREIGN KEY (externalTool_id) REFERENCES externaltool (id); + +CREATE INDEX IF NOT EXISTS index_actionlogrecord_actiontype + ON externaltooltype (id); + +CREATE INDEX IF NOT EXISTS index_actionlogrecord_useridentifier + ON externaltooltype (externaltool_id); + +CREATE INDEX IF NOT EXISTS index_externaltooltype_externaltool_id + ON externaltooltype (externaltool_id); diff --git a/src/main/resources/db/extra/V5.4.1.0__7400-prepare-table.sql b/src/main/resources/db/extra/V5.4.1.0__7400-prepare-table.sql new file mode 100644 index 00000000000..13a4e78705a --- /dev/null +++ b/src/main/resources/db/extra/V5.4.1.0__7400-prepare-table.sql @@ -0,0 +1,15 @@ +-- This is a workaround for the missing auxiliaryfile table + +create table if not exists auxiliaryfile +( + id serial primary key, + checksum varchar(255), + contenttype varchar(255), + filesize bigint, + formattag varchar(255), + formatversion varchar(255), + ispublic boolean, + origin varchar(255), + type varchar(255), + datafile_id bigint not null constraint fk_auxiliaryfile_datafile_id references dvobject +); diff --git a/src/main/resources/db/extra/V5.9.0.0__7440-prepare-table.sql b/src/main/resources/db/extra/V5.9.0.0__7440-prepare-table.sql new file mode 100644 index 00000000000..8ea09111c8c --- /dev/null +++ b/src/main/resources/db/extra/V5.9.0.0__7440-prepare-table.sql @@ -0,0 +1,21 @@ +-- This is a workaround for the missing license table + +create table if not exists license +( + id serial primary key, + active boolean not null, + iconurl text, + isdefault boolean not null, + name text constraint unq_license_0 unique, + shortdescription text, + sortorder bigint default 0 not null, + uri text constraint unq_license_1 unique +); + +create index if not exists license_sortorder_id on license (sortorder, id); + +-- Save information about CCBY license to temporary table (to be used in a migration after the main license migration) +CREATE TEMPORARY TABLE ccby( + id serial primary key +); +INSERT INTO ccby (id) SELECT id FROM termsofuseandaccess WHERE license = 'CCBY'; diff --git a/src/main/resources/db/extra/V5.9.0.2__7440-reinject-cc-by.sql b/src/main/resources/db/extra/V5.9.0.2__7440-reinject-cc-by.sql new file mode 100644 index 00000000000..826d60ec3b8 --- /dev/null +++ b/src/main/resources/db/extra/V5.9.0.2__7440-reinject-cc-by.sql @@ -0,0 +1,8 @@ +-- Insert the CC-BY license and add references to it from the termsofuseandacccess table + +INSERT INTO license (id, uri, name, shortdescription, active, isDefault, iconurl, sortorder) +VALUES (2, 'http://creativecommons.org/licenses/by/4.0', 'CC BY 4.0', 'Creative Commons Attribution 4.0 International License.', true, false, 'https://licensebuttons.net/l/by/4.0/88x31.png', 2) +ON CONFLICT DO NOTHING; + +UPDATE termsofuseandaccess SET license_id = (SELECT license.id FROM license WHERE license.name = 'CC BY 4.0') +WHERE id IN (SELECT id FROM ccby); diff --git a/src/main/resources/db/extra/V6.0.0.0__9763-prepare-table.sql b/src/main/resources/db/extra/V6.0.0.0__9763-prepare-table.sql new file mode 100644 index 00000000000..f0933da16bb --- /dev/null +++ b/src/main/resources/db/extra/V6.0.0.0__9763-prepare-table.sql @@ -0,0 +1,17 @@ +-- This is a workaround for the missing embargo table +create table if not exists embargo +( + id serial primary key, + dateavailable date not null, + reason text +); + +-- This is a workaround for the missing storageuse table +create table if not exists storageuse +( + id serial primary key, + sizeinbytes bigint, + dvobjectcontainer_id bigint not null constraint fk_storageuse_dvobjectcontainer_id references dvobject +); +create index if not exists index_storageuse_dvobjectcontainer_id on storageuse (dvobjectcontainer_id); + diff --git a/src/main/resources/db/extra/V6.1.0.0__9983-prepare-table.sql b/src/main/resources/db/extra/V6.1.0.0__9983-prepare-table.sql new file mode 100644 index 00000000000..47b0d2a27af --- /dev/null +++ b/src/main/resources/db/extra/V6.1.0.0__9983-prepare-table.sql @@ -0,0 +1,12 @@ +-- This is a workaround for the missing externalvocabularyvalue table +create table if not exists externalvocabularyvalue +( + id serial primary key, + lastupdatedate timestamp, + uri text constraint externalvocabularvalue_uri_key unique, + value text +); + +create index if not exists index_externalvocabularyvalue_uri on externalvocabularyvalue (uri); + + From eccc0fec1401f073a55cedcf0cf2ae0de76f4de7 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 22 May 2025 18:27:03 +0200 Subject: [PATCH 02/18] feat(migrate): add Dataverse database migration module with (optional) Docker and Flyway This commit introduces a new Maven module, `dataverse-migrate-db`, for database migration. It utilizes Docker for setting up a PostgreSQL environment and Flyway for managing schema migrations. The configuration supports customizable properties and includes execution phases for setting up, dumping, migrating, and cleaning up the database. The setup of Postgres in Docker can be skipped to run the migrations against some other remote database. It is up to the user to ensure no harm comes from that. --- modules/dataverse-migrate-db/pom.xml | 266 +++++++++++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 modules/dataverse-migrate-db/pom.xml diff --git a/modules/dataverse-migrate-db/pom.xml b/modules/dataverse-migrate-db/pom.xml new file mode 100644 index 00000000000..21585e2c381 --- /dev/null +++ b/modules/dataverse-migrate-db/pom.xml @@ -0,0 +1,266 @@ + + + 4.0.0 + + + edu.harvard.iq + dataverse-parent + ${revision} + ../dataverse-parent + + + migrate-db + pom + migrate-db + + + + true + true + true + true + true + true + + true + postgres,pg_dump + false + ${docker.skip} + ${docker.skip} + ${docker.skip} + true + true + + localhost + 15432 + dataverse + ${postgresql.db} + supersecret + ${project.basedir}/db_dump.sql + sql + ${project.build.directory} + + 30 + + + + + + io.fabric8 + docker-maven-plugin + + true + + + + postgres + postgres:${postgresql.server.version} + + true + + + + custom + migration + migrationdb + + + localhost:${postgresql.port}:5432 + + + ${postgresql.username} + ${postgresql.password} + ${postgresql.db} + + + + ${postgresql.dump.file}:/docker-entrypoint-initdb.d/dumb.${postgresql.dump.file.ext} + + + + + + + + + 5432 + + + + + + + + pg_dump + postgres:${postgresql.server.version} + + true + + + + custom + migration + + + ${postgresql.username} + ${postgresql.password} + + + + ${postgresql.dump.target}:/dumptarget + + + + + + ${postgresql.password} + + + + + sh + -c + pg_dump -h migrationdb -p 5432 -U ${postgresql.username} -v -c -C -f /dumptarget/migrated_db_dump.sql ${postgresql.db} + + + + + + + + + + + + cleanup + clean + + stop + + + + start + prepare-package + + start + + + ${docker.skipStart} + + + postgres + + false + + + + pg_dump + + true + + + + + + + dump + verify + + start + + + ${docker.skipDump} + + + postgres + + true + + + + pg_dump + + false + + + + + + + stop + install + + stop + + + ${docker.skipStop} + + + + + + org.flywaydb + flyway-maven-plugin + ${flyway.version} + + + org.flywaydb + flyway-core + ${flyway.version} + + + org.flywaydb + flyway-database-postgresql + ${flyway.version} + + + org.postgresql + postgresql + ${postgresql.version} + + + + jdbc:postgresql://${postgresql.host}:${postgresql.port}/${postgresql.db} + ${postgresql.username} + ${postgresql.password} + __ + + + + migrate + package + + migrate + + + true + + filesystem:../../src/main/resources/db/migration + filesystem:extra-migrations + + + + + cleanup + package + + repair + + + + filesystem:../../src/main/resources/db/migration + + + + + + + + From a08bc15781f21bf1ab01e0d1350868c1e8ea9bc5 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 22 May 2025 18:31:45 +0200 Subject: [PATCH 03/18] feat(migrate): migrate extra SQL table scripts to dedicated module Moved SQL definitions for missing tables from `db/extra` to the new `dataverse-migrate-db/extra-migrations` module. This improves organization and separates migration scripts from other database resources for easier maintenance. --- .../extra-migrations/V5.1.1.0__prepare-table-exttooltype.sql | 0 .../extra-migrations/V5.4.1.0__prepare-table-auxfile.sql | 0 .../extra-migrations/V5.9.0.0__prepare-table-license.sql | 0 .../V6.0.0.0__prepare-tables-embargo-storageuse.sql | 0 .../extra-migrations/V6.1.0.0__prepare-table-extvocab.sql | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/extra/V5.1.1.0__6919-prepare-table.sql => modules/dataverse-migrate-db/extra-migrations/V5.1.1.0__prepare-table-exttooltype.sql (100%) rename src/main/resources/db/extra/V5.4.1.0__7400-prepare-table.sql => modules/dataverse-migrate-db/extra-migrations/V5.4.1.0__prepare-table-auxfile.sql (100%) rename src/main/resources/db/extra/V5.9.0.0__7440-prepare-table.sql => modules/dataverse-migrate-db/extra-migrations/V5.9.0.0__prepare-table-license.sql (100%) rename src/main/resources/db/extra/V6.0.0.0__9763-prepare-table.sql => modules/dataverse-migrate-db/extra-migrations/V6.0.0.0__prepare-tables-embargo-storageuse.sql (100%) rename src/main/resources/db/extra/V6.1.0.0__9983-prepare-table.sql => modules/dataverse-migrate-db/extra-migrations/V6.1.0.0__prepare-table-extvocab.sql (100%) diff --git a/src/main/resources/db/extra/V5.1.1.0__6919-prepare-table.sql b/modules/dataverse-migrate-db/extra-migrations/V5.1.1.0__prepare-table-exttooltype.sql similarity index 100% rename from src/main/resources/db/extra/V5.1.1.0__6919-prepare-table.sql rename to modules/dataverse-migrate-db/extra-migrations/V5.1.1.0__prepare-table-exttooltype.sql diff --git a/src/main/resources/db/extra/V5.4.1.0__7400-prepare-table.sql b/modules/dataverse-migrate-db/extra-migrations/V5.4.1.0__prepare-table-auxfile.sql similarity index 100% rename from src/main/resources/db/extra/V5.4.1.0__7400-prepare-table.sql rename to modules/dataverse-migrate-db/extra-migrations/V5.4.1.0__prepare-table-auxfile.sql diff --git a/src/main/resources/db/extra/V5.9.0.0__7440-prepare-table.sql b/modules/dataverse-migrate-db/extra-migrations/V5.9.0.0__prepare-table-license.sql similarity index 100% rename from src/main/resources/db/extra/V5.9.0.0__7440-prepare-table.sql rename to modules/dataverse-migrate-db/extra-migrations/V5.9.0.0__prepare-table-license.sql diff --git a/src/main/resources/db/extra/V6.0.0.0__9763-prepare-table.sql b/modules/dataverse-migrate-db/extra-migrations/V6.0.0.0__prepare-tables-embargo-storageuse.sql similarity index 100% rename from src/main/resources/db/extra/V6.0.0.0__9763-prepare-table.sql rename to modules/dataverse-migrate-db/extra-migrations/V6.0.0.0__prepare-tables-embargo-storageuse.sql diff --git a/src/main/resources/db/extra/V6.1.0.0__9983-prepare-table.sql b/modules/dataverse-migrate-db/extra-migrations/V6.1.0.0__prepare-table-extvocab.sql similarity index 100% rename from src/main/resources/db/extra/V6.1.0.0__9983-prepare-table.sql rename to modules/dataverse-migrate-db/extra-migrations/V6.1.0.0__prepare-table-extvocab.sql From b72a3b97818b58b0d096fa5267696d56dbf98782 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 22 May 2025 18:32:27 +0200 Subject: [PATCH 04/18] feat(migrate): remove FZJ specific SQL migration scripts for URI field and CC-BY. Deleted legacy database migration scripts no longer relevant due to upstream compatibility adjustments and resolved licensing updates. This cleanup helps maintain a more streamlined and accurate codebase. --- .../db/extra/V5.0.0.0__fzj-drop-uri-fieldtype.sql | 2 -- .../resources/db/extra/V5.9.0.2__7440-reinject-cc-by.sql | 8 -------- 2 files changed, 10 deletions(-) delete mode 100644 src/main/resources/db/extra/V5.0.0.0__fzj-drop-uri-fieldtype.sql delete mode 100644 src/main/resources/db/extra/V5.9.0.2__7440-reinject-cc-by.sql diff --git a/src/main/resources/db/extra/V5.0.0.0__fzj-drop-uri-fieldtype.sql b/src/main/resources/db/extra/V5.0.0.0__fzj-drop-uri-fieldtype.sql deleted file mode 100644 index f97f76424f4..00000000000 --- a/src/main/resources/db/extra/V5.0.0.0__fzj-drop-uri-fieldtype.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Replace the URI_NO field type with TEXT to be upstream compatible again -UPDATE datasetfieldtype SET fieldtype = 'TEXT' WHERE name = 'storage_location'; \ No newline at end of file diff --git a/src/main/resources/db/extra/V5.9.0.2__7440-reinject-cc-by.sql b/src/main/resources/db/extra/V5.9.0.2__7440-reinject-cc-by.sql deleted file mode 100644 index 826d60ec3b8..00000000000 --- a/src/main/resources/db/extra/V5.9.0.2__7440-reinject-cc-by.sql +++ /dev/null @@ -1,8 +0,0 @@ --- Insert the CC-BY license and add references to it from the termsofuseandacccess table - -INSERT INTO license (id, uri, name, shortdescription, active, isDefault, iconurl, sortorder) -VALUES (2, 'http://creativecommons.org/licenses/by/4.0', 'CC BY 4.0', 'Creative Commons Attribution 4.0 International License.', true, false, 'https://licensebuttons.net/l/by/4.0/88x31.png', 2) -ON CONFLICT DO NOTHING; - -UPDATE termsofuseandaccess SET license_id = (SELECT license.id FROM license WHERE license.name = 'CC BY 4.0') -WHERE id IN (SELECT id FROM ccby); From 02a6c365e77c3c07e09dcc20dc2bac89d928916a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 22 May 2025 18:33:18 +0200 Subject: [PATCH 05/18] feat(migrate): add database migration scripts for new tables Introduces migrations for `DatasetType` and `MakeDataCountProcessState` tables. These include schema definitions, relationships, constraints, and an index to support new functionalities. The initial extra migrations were done when 6.2 was released, now adding the others. --- .../V6.3.0.0__prepare-table-datasettype.sql | 33 +++++++++++++++++++ ...V6.5.0.0__prepare-table-mdc-proc-state.sql | 11 +++++++ 2 files changed, 44 insertions(+) create mode 100644 modules/dataverse-migrate-db/extra-migrations/V6.3.0.0__prepare-table-datasettype.sql create mode 100644 modules/dataverse-migrate-db/extra-migrations/V6.5.0.0__prepare-table-mdc-proc-state.sql diff --git a/modules/dataverse-migrate-db/extra-migrations/V6.3.0.0__prepare-table-datasettype.sql b/modules/dataverse-migrate-db/extra-migrations/V6.3.0.0__prepare-table-datasettype.sql new file mode 100644 index 00000000000..331842666b7 --- /dev/null +++ b/modules/dataverse-migrate-db/extra-migrations/V6.3.0.0__prepare-table-datasettype.sql @@ -0,0 +1,33 @@ +CREATE TABLE IF NOT EXISTS DatasetType +( + id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, + name VARCHAR(255) NOT NULL, + CONSTRAINT pk_datasettype PRIMARY KEY (id) +); + +CREATE TABLE IF NOT EXISTS DatasetType_licenses +( + DatasetType_id BIGINT NOT NULL, + licenses_id BIGINT NOT NULL +); + +CREATE TABLE IF NOT EXISTS DatasetType_metadataBlocks +( + DatasetType_id BIGINT NOT NULL, + metadataBlocks_id BIGINT NOT NULL +); + +ALTER TABLE DatasetType + ADD CONSTRAINT uc_828171b214f7369ee92a28781 UNIQUE (name); + +ALTER TABLE DatasetType_licenses + ADD CONSTRAINT fk_datlic_on_dataset_type FOREIGN KEY (DatasetType_id) REFERENCES DatasetType (id); + +ALTER TABLE DatasetType_licenses + ADD CONSTRAINT fk_datlic_on_license FOREIGN KEY (licenses_id) REFERENCES License (id); + +ALTER TABLE DatasetType_metadataBlocks + ADD CONSTRAINT fk_datmet_on_dataset_type FOREIGN KEY (DatasetType_id) REFERENCES DatasetType (id); + +ALTER TABLE DatasetType_metadataBlocks + ADD CONSTRAINT fk_datmet_on_metadata_block FOREIGN KEY (metadataBlocks_id) REFERENCES MetadataBlock (id); \ No newline at end of file diff --git a/modules/dataverse-migrate-db/extra-migrations/V6.5.0.0__prepare-table-mdc-proc-state.sql b/modules/dataverse-migrate-db/extra-migrations/V6.5.0.0__prepare-table-mdc-proc-state.sql new file mode 100644 index 00000000000..23353351e15 --- /dev/null +++ b/modules/dataverse-migrate-db/extra-migrations/V6.5.0.0__prepare-table-mdc-proc-state.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS MakeDataCountProcessState +( + id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, + yearMonth VARCHAR(255) NOT NULL, + state INTEGER NOT NULL, + stateChangeTimestamp TIMESTAMP WITHOUT TIME ZONE, + server VARCHAR(255), + CONSTRAINT pk_makedatacountprocessstate PRIMARY KEY (id) +); + +CREATE INDEX IF NOT EXISTS idx_18d578c22760ad21e31cc873d ON MakeDataCountProcessState (yearMonth); \ No newline at end of file From b6065b723194c5a4dfc2a69de11b8f86400ad536 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 22 May 2025 18:36:06 +0200 Subject: [PATCH 06/18] feat(migrate,ct): update dependency versions and relocate configuration Moved `flyway.version` and `postgresql.server.version` to the `dataverse-parent` POM for better configuration centralization. Removed their redundant declarations from the root POM to maintain consistency. Enables reusing the values in the new `dataverse-migrate-db` module Downgraded the PostgreSQL server version from 17 to the one specified in the installation guide (16) to be in sync with the official recommendations. --- modules/dataverse-parent/pom.xml | 2 ++ pom.xml | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index df1b31181bd..45bc90c4c9b 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -151,6 +151,8 @@ 6.2025.3 42.7.7 + 16 + 10.19.0 9.8.0 2.31.3 26.30.0 diff --git a/pom.xml b/pom.xml index d04ea658da8..76f9a4a0727 100644 --- a/pom.xml +++ b/pom.xml @@ -27,7 +27,6 @@ war 1.2.18.4 - 10.19.0 1.1.1 1.20.1 5.4.0 @@ -1141,7 +1140,6 @@ true docker-build - 17 gdcc/dataverse:${app.image.tag} - - - jdbc:postgresql://localhost:54321/dataverse - dataverse - secret - __ - - filesystem:src/main/resources/db/migration - filesystem:src/main/resources/db/extra - - - - org.apache.maven.plugins maven-compiler-plugin From bdd637fb1ba3f7d0addaf8830cc58425559489eb Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 23 May 2025 18:52:19 +0200 Subject: [PATCH 08/18] chore(migrate): add migration for keywordTermUri and placeholder configuration #10288 Introduces a new migration script to update `keywordValue` fields containing HTTP values to `keywordTermUri` (as seen in the release notes). Updates the POM to include a placeholder for toggling this migration, ensuring flexibility during upgrades. Using a Flyway placeholder, we can inject a Maven property into the SQL script and conditionally execute the migration. Defaulting to false (don't execute). See also: https://documentation.red-gate.com/fd/migration-placeholders-275218550.html --- .../V6.2.0.2__migrate_keywordTermUri.sql | 17 +++++++++++++++++ modules/dataverse-migrate-db/pom.xml | 7 +++++++ 2 files changed, 24 insertions(+) create mode 100644 modules/dataverse-migrate-db/extra-migrations/V6.2.0.2__migrate_keywordTermUri.sql diff --git a/modules/dataverse-migrate-db/extra-migrations/V6.2.0.2__migrate_keywordTermUri.sql b/modules/dataverse-migrate-db/extra-migrations/V6.2.0.2__migrate_keywordTermUri.sql new file mode 100644 index 00000000000..1b7851b1ad5 --- /dev/null +++ b/modules/dataverse-migrate-db/extra-migrations/V6.2.0.2__migrate_keywordTermUri.sql @@ -0,0 +1,17 @@ +-- +DO +$$ + DECLARE + should_migrate BOOLEAN := CASE WHEN UPPER('${V6_3_migrate_keywordTermUri}') = 'TRUE' THEN TRUE ELSE FALSE END; + BEGIN + IF should_migrate THEN + RAISE NOTICE 'Migrating keywordValue fields with http... to keywordTermUri as requested'; + UPDATE datasetfield df + SET datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordTermURI') + FROM datasetfieldvalue dfv + WHERE dfv.datasetfield_id = df.id + AND df.datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordValue') + AND dfv.value ILIKE 'http%'; + END IF; + END +$$; \ No newline at end of file diff --git a/modules/dataverse-migrate-db/pom.xml b/modules/dataverse-migrate-db/pom.xml index 21585e2c381..8973d647aea 100644 --- a/modules/dataverse-migrate-db/pom.xml +++ b/modules/dataverse-migrate-db/pom.xml @@ -42,6 +42,10 @@ ${project.build.directory} 30 + + + + false @@ -245,6 +249,9 @@ filesystem:../../src/main/resources/db/migration filesystem:extra-migrations + + ${migrate.keywordTermUri} + From 7c286d86658ccb29d82db353d1b061ffa9628207 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 27 May 2025 16:21:05 +0200 Subject: [PATCH 09/18] refactor(migrate): refine keyword migration logic with row count and hints Added row count check and more detailed notices for the `keywordValue` to `keywordTermUri` migration process. Added guidance for re-execution if migration is skipped. --- .../V6.2.0.2__migrate_keywordTermUri.sql | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/modules/dataverse-migrate-db/extra-migrations/V6.2.0.2__migrate_keywordTermUri.sql b/modules/dataverse-migrate-db/extra-migrations/V6.2.0.2__migrate_keywordTermUri.sql index 1b7851b1ad5..ac84a35c3ba 100644 --- a/modules/dataverse-migrate-db/extra-migrations/V6.2.0.2__migrate_keywordTermUri.sql +++ b/modules/dataverse-migrate-db/extra-migrations/V6.2.0.2__migrate_keywordTermUri.sql @@ -3,15 +3,35 @@ DO $$ DECLARE should_migrate BOOLEAN := CASE WHEN UPPER('${V6_3_migrate_keywordTermUri}') = 'TRUE' THEN TRUE ELSE FALSE END; + keyword_count INTEGER; BEGIN - IF should_migrate THEN - RAISE NOTICE 'Migrating keywordValue fields with http... to keywordTermUri as requested'; - UPDATE datasetfield df - SET datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordTermURI') - FROM datasetfieldvalue dfv - WHERE dfv.datasetfield_id = df.id - AND df.datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordValue') - AND dfv.value ILIKE 'http%'; + -- Get the count of rows that match the criteria + SELECT COUNT(*) INTO keyword_count + FROM datasetfieldvalue dfv INNER JOIN datasetfield df ON df.id = dfv.datasetfield_id + WHERE df.datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordValue') + AND value ILIKE 'http%'; + + -- Only show hint if count is greater than 0 + IF keyword_count > 0 THEN + RAISE NOTICE '--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---'; + RAISE NOTICE 'Found % keywordValue metadata fields starting with "http"', keyword_count; + + IF should_migrate THEN + RAISE NOTICE 'Migrating keywordValue fields with http... to keywordTermUri as requested by -Dmigrate.keywordTermUri'; + UPDATE datasetfield df + SET datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordTermURI') + FROM datasetfieldvalue dfv + WHERE dfv.datasetfield_id = df.id + AND df.datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordValue') + AND dfv.value ILIKE 'http%'; + + GET DIAGNOSTICS keyword_count = ROW_COUNT; + RAISE NOTICE 'Updated % rows', keyword_count; + + ELSE + RAISE NOTICE 'Auto-migrate these into keywordTermUri using `mvn -Dmigrate.keywordTermUri ...` (re-execute migrations)'; + END IF; + RAISE NOTICE '--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---'; END IF; END $$; \ No newline at end of file From 64ec444c7653b043b856ec7d946c6ce29cc9dfe5 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 27 May 2025 16:22:03 +0200 Subject: [PATCH 10/18] feat(migrate): add migration to clean up saved searches and linked objects This migration identifies and optionally deletes problematic saved searches, dataset links, and collection links related to issue #7398. A new Maven property, `migrate.cleanupSavedSearches`, has been added to enable or disable the cleanup process during migrations. --- ..._7398-cleanup-saved-searches-and-links.sql | 93 +++++++++++++++++++ modules/dataverse-migrate-db/pom.xml | 2 + 2 files changed, 95 insertions(+) create mode 100644 modules/dataverse-migrate-db/extra-migrations/V5.3.0.7__7398-cleanup-saved-searches-and-links.sql diff --git a/modules/dataverse-migrate-db/extra-migrations/V5.3.0.7__7398-cleanup-saved-searches-and-links.sql b/modules/dataverse-migrate-db/extra-migrations/V5.3.0.7__7398-cleanup-saved-searches-and-links.sql new file mode 100644 index 00000000000..ac3eab95679 --- /dev/null +++ b/modules/dataverse-migrate-db/extra-migrations/V5.3.0.7__7398-cleanup-saved-searches-and-links.sql @@ -0,0 +1,93 @@ +-- +DO +$$ + DECLARE + should_delete BOOLEAN := CASE WHEN UPPER('${V5_4_cleanup_searches_and_links}') = 'TRUE' THEN TRUE ELSE FALSE END; + affected_searches_count INTEGER; + affected_linked_datasets_count INTEGER; + affected_linked_collections_count INTEGER; + row_count INTEGER; + BEGIN + -- Get the count of rows that match the criteria + SELECT COUNT(*) INTO affected_searches_count + from savedsearch ss, savedsearchfilterquery ssfq, dataverselinkingdataverse dld + where ss.id = ssfq.savedsearch_id + and ss.definitionpoint_id = dld.linkingdataverse_id + and dld.dataverse_id = rtrim(reverse(split_part(reverse(ssfq.filterquery),'/',1)),'"')::integer + and ss.query='*' + and ssfq.filterquery like 'subtreePaths%'; + + select COUNT(*) INTO affected_linked_datasets_count + from datasetlinkingdataverse dld, dvobject dvo, dataverselinkingdataverse dvld + where dld.dataset_id = dvo.id + and dld.linkingdataverse_id = dvld.linkingdataverse_id + and dvo.owner_id = dvld.dataverse_id; + + select COUNT(*) INTO affected_linked_collections_count + from dataverselinkingdataverse dld, dvobject dvo, dataverselinkingdataverse dvld + where dld.dataverse_id = dvo.id + and dld.linkingdataverse_id = dvld.linkingdataverse_id + and dvo.owner_id = dvld.dataverse_id; + + -- Only show hint if count is greater than 0 + IF affected_searches_count > 0 OR affected_linked_datasets_count > 0 OR affected_linked_collections_count > 0 THEN + RAISE NOTICE '--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---'; + RAISE NOTICE 'Found % saved searches affected by issue #7398', affected_searches_count; + RAISE NOTICE 'Found % dataset links affected by issue #7398', affected_linked_datasets_count; + RAISE NOTICE 'Found % collections links affected by issue #7398', affected_linked_collections_count; + + IF should_delete THEN + RAISE NOTICE 'Cleaning up Saved Searches and Linked Datasets as requested by -Dmigrate.cleanupSavedSearches'; + + -- delete the saved searches identified using the ss_for_deletion query + create temporary table delete_ss on commit drop as ( + Select ss.id + from savedsearch ss, savedsearchfilterquery ssfq, dataverselinkingdataverse dld + where ss.id = ssfq.savedsearch_id + and ss.definitionpoint_id = dld.linkingdataverse_id + and dld.dataverse_id = rtrim(reverse(split_part(reverse(ssfq.filterquery),'/',1)),'"')::integer + and ss.query='*' + and ssfq.filterquery like 'subtreePaths%' + ); + + GET DIAGNOSTICS row_count = ROW_COUNT; + + delete from savedsearchfilterquery where savedsearch_id in (select id from delete_ss); + delete from savedsearch where id in (select id from delete_ss); + + RAISE NOTICE 'Deleted % saved searches', row_count; + + COMMIT; + + -- delete linked objects identified using the query in dld_for_deletion + delete from datasetlinkingdataverse where id in ( + select dld.id + from datasetlinkingdataverse dld, dvobject dvo, dataverselinkingdataverse dvld + where dld.dataset_id = dvo.id + and dld.linkingdataverse_id = dvld.linkingdataverse_id + and dvo.owner_id = dvld.dataverse_id + ); + + GET DIAGNOSTICS row_count = ROW_COUNT; + RAISE NOTICE 'Deleted % linked datasets', row_count; + + delete from dataverselinkingdataverse where id in ( + select dld.id + from dataverselinkingdataverse dld, dvobject dvo, dataverselinkingdataverse dvld + where dld.dataverse_id = dvo.id + and dld.linkingdataverse_id = dvld.linkingdataverse_id + and dvo.owner_id = dvld.dataverse_id + ); + + GET DIAGNOSTICS row_count = ROW_COUNT; + RAISE NOTICE 'Deleted % linked collections', row_count; + + COMMIT; + + ELSE + RAISE NOTICE 'Auto-migrate these using `mvn -Dmigrate.cleanupSavedSearches ...` (re-execute migrations)'; + END IF; + RAISE NOTICE '--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---'; + END IF; + END +$$; diff --git a/modules/dataverse-migrate-db/pom.xml b/modules/dataverse-migrate-db/pom.xml index 8973d647aea..0aa2f850528 100644 --- a/modules/dataverse-migrate-db/pom.xml +++ b/modules/dataverse-migrate-db/pom.xml @@ -46,6 +46,7 @@ false + false @@ -251,6 +252,7 @@ ${migrate.keywordTermUri} + ${migrate.cleanupSavedSearches} From c405bb2af76164806ef1260c097cd4178053dd78 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 27 May 2025 16:22:29 +0200 Subject: [PATCH 11/18] chore(migrate): add SQL script to detect issues with file access flags This script identifies datasets with mismatched file access request flags caused by the curate command. It provides a manual fixing hint for affected cases, referencing issue #7687. The future possibility of an automated migration is noted. --- ....0.8__7687-file-access-flag-update-bug.sql | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 modules/dataverse-migrate-db/extra-migrations/V5.3.0.8__7687-file-access-flag-update-bug.sql diff --git a/modules/dataverse-migrate-db/extra-migrations/V5.3.0.8__7687-file-access-flag-update-bug.sql b/modules/dataverse-migrate-db/extra-migrations/V5.3.0.8__7687-file-access-flag-update-bug.sql new file mode 100644 index 00000000000..60f87ac80fc --- /dev/null +++ b/modules/dataverse-migrate-db/extra-migrations/V5.3.0.8__7687-file-access-flag-update-bug.sql @@ -0,0 +1,30 @@ +-- this query will identify datasets where a superuser has run the Curate command and the update included a change to +-- the fileaccessrequest flag, resulting in the file access request updates not being reflected in the published version +DO +$$ + DECLARE + -- should_migrate BOOLEAN := CASE WHEN UPPER('...') = 'TRUE' THEN TRUE ELSE FALSE END; + affected_count INTEGER; + BEGIN + -- Get the count of rows that match the criteria + SELECT COUNT(*) INTO affected_count + from datasetversion dv, termsofuseandaccess ta, dataset da + where dv.dataset_id=da.id + and dv.termsofuseandaccess_id=ta.id + and ta.fileaccessrequest != da.fileaccessrequest + and dv.versionstate='RELEASED' + and dv.releasetime in (select max(releasetime) + from datasetversion + where dataset_id=da.id); + + -- Only show hint if count is greater than 0 + IF affected_count > 0 THEN + RAISE NOTICE '--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---'; + RAISE NOTICE 'Found % datasets affected by issue #7687', affected_count; + RAISE NOTICE 'For now, please fix these manually. See Dataverse v5.4 release notes about #7687.'; + RAISE NOTICE '--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---'; + END IF; + + -- TODO: an opt-in migration to fix them all would be nice! + END; +$$; \ No newline at end of file From ad9dfa20b067abbaf789444f5384e68c4ea50f48 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 27 May 2025 16:22:49 +0200 Subject: [PATCH 12/18] feat(migrate): add migration script to delete orphan templates This script identifies and removes templates without associated dataverses, addressing issue #8600. It also updates dataverses using such templates as defaults before deleting the orphan templates to ensure data integrity. --- .../V5.10.0.0__8600-orphaned-templates.sql | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 modules/dataverse-migrate-db/extra-migrations/V5.10.0.0__8600-orphaned-templates.sql diff --git a/modules/dataverse-migrate-db/extra-migrations/V5.10.0.0__8600-orphaned-templates.sql b/modules/dataverse-migrate-db/extra-migrations/V5.10.0.0__8600-orphaned-templates.sql new file mode 100644 index 00000000000..d10e0e37ac7 --- /dev/null +++ b/modules/dataverse-migrate-db/extra-migrations/V5.10.0.0__8600-orphaned-templates.sql @@ -0,0 +1,115 @@ +-- Migration script to delete orphan templates (templates with no associated dataverse), see also issue #8600 + +DO +$$ + DECLARE + orphan_templates_count INTEGER; + affected_collections_count INTEGER; + row_count INTEGER; + BEGIN + -- Get the count of orphan templates + SELECT COUNT(t.id) INTO orphan_templates_count + FROM template t + WHERE dataverse_id IS NULL; + + -- Count dataverse collections that use orphan templates as default + SELECT COUNT(*) INTO affected_collections_count + FROM dataverse d + WHERE d.defaulttemplate_id IN ( + SELECT t.id FROM template t WHERE dataverse_id IS NULL + ); + + -- Only execute queries if the affected count is greater than 0 + IF orphan_templates_count > 0 THEN + RAISE NOTICE '--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---'; + RAISE NOTICE 'Found % orphan templates (templates with no associated dataverse)', orphan_templates_count; + RAISE NOTICE 'Found % dataverses using orphan templates as their default template', affected_collections_count; + + -- Please note: The below is quite a bunch of things we need to execute. When using the Admin API call + -- to delete the template, all the cascading is handled by JPA for us. We don't have that... + + -- First, update all dataverses that use orphan templates as default template + UPDATE dataverse + SET defaulttemplate_id = NULL + WHERE defaulttemplate_id IN ( + SELECT t.id FROM template t WHERE dataverse_id IS NULL + ); + + GET DIAGNOSTICS row_count = ROW_COUNT; + RAISE NOTICE 'Updated % collections to remove orphan templates set as default', row_count; + + -- Create a temporary table to keep track of datasetfields to delete + CREATE TEMPORARY TABLE temp_datasetfields_to_delete AS + SELECT id FROM datasetfield + WHERE template_id IN ( + SELECT t.id FROM template t WHERE dataverse_id IS NULL + ); + + -- Create a temporary table to keep track of compound values to delete + CREATE TEMPORARY TABLE temp_compoundvalues_to_delete AS + SELECT cv.id + FROM datasetfieldcompoundvalue cv + WHERE cv.parentdatasetfield_id IN ( + SELECT id FROM temp_datasetfields_to_delete + ); + + -- Delete mappings between datasetfield and controlledvocabularyvalues + DELETE FROM datasetfield_controlledvocabularyvalue + WHERE datasetfield_id IN ( + SELECT id FROM temp_datasetfields_to_delete + ); + + GET DIAGNOSTICS row_count = ROW_COUNT; + RAISE NOTICE 'Deleted % vocabulary mappings associated with orphan templates', row_count; + + -- Delete datasetfieldvalue records that reference the datasetfields we're going to delete + DELETE FROM datasetfieldvalue + WHERE datasetfield_id IN ( + SELECT id FROM temp_datasetfields_to_delete + ); + + GET DIAGNOSTICS row_count = ROW_COUNT; + RAISE NOTICE 'Deleted % datasetfieldvalues associated with orphan templates', row_count; + + -- Break the circular reference by setting parentdatasetfieldcompoundvalue_id to NULL + UPDATE datasetfield + SET parentdatasetfieldcompoundvalue_id = NULL + WHERE parentdatasetfieldcompoundvalue_id IN ( + SELECT id FROM temp_compoundvalues_to_delete + ); + + GET DIAGNOSTICS row_count = ROW_COUNT; + RAISE NOTICE 'Updated % datasetfields to remove references to compound values', row_count; + + -- Now we can safely delete the compound values + DELETE FROM datasetfieldcompoundvalue + WHERE id IN ( + SELECT id FROM temp_compoundvalues_to_delete + ); + + GET DIAGNOSTICS row_count = ROW_COUNT; + RAISE NOTICE 'Deleted % datasetfieldcompoundvalues associated with orphan templates', row_count; + + -- Delete datasetfields that reference orphan templates + DELETE FROM datasetfield + WHERE id IN ( + SELECT id FROM temp_datasetfields_to_delete + ); + + GET DIAGNOSTICS row_count = ROW_COUNT; + RAISE NOTICE 'Deleted % datasetfields referencing orphan templates', row_count; + + -- Clean up temporary tables + DROP TABLE temp_datasetfields_to_delete; + DROP TABLE temp_compoundvalues_to_delete; + + -- Then finally delete all orphan templates + DELETE FROM template + WHERE dataverse_id IS NULL; + + GET DIAGNOSTICS row_count = ROW_COUNT; + RAISE NOTICE 'Deleted % orphan templates', row_count; + RAISE NOTICE '--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---'; + END IF; + END +$$; \ No newline at end of file From ce251e91e87c57b6e8c92c41b5936e79da4cf863 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 27 May 2025 16:23:05 +0200 Subject: [PATCH 13/18] chore(migrate): remove obsolete EJB timer table with SQL migration This commit adds a new SQL migration script to drop the EJB__TIMER__TBL table if it exists. This helps clean up unused database artifacts and improves maintainability. --- .../extra-migrations/V4.20.0.6__remove-ejb-timer.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 modules/dataverse-migrate-db/extra-migrations/V4.20.0.6__remove-ejb-timer.sql diff --git a/modules/dataverse-migrate-db/extra-migrations/V4.20.0.6__remove-ejb-timer.sql b/modules/dataverse-migrate-db/extra-migrations/V4.20.0.6__remove-ejb-timer.sql new file mode 100644 index 00000000000..de29aa0da4a --- /dev/null +++ b/modules/dataverse-migrate-db/extra-migrations/V4.20.0.6__remove-ejb-timer.sql @@ -0,0 +1,2 @@ +-- +DROP TABLE IF EXISTS EJB__TIMER__TBL; From ff1bf17b7588b5a36f161859369c9f5baad6f6e1 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 27 May 2025 16:24:09 +0200 Subject: [PATCH 14/18] feat(migrate): add migration script to identify datasets with access issues This script identifies datasets with restricted files lacking terms of access or request access settings. It logs affected datasets with owner details and provides references for further information in the Dataverse 5.11 release notes and issue #8191. If matching files/datasets are found, the admin needs to take manual action to resolve the situation (we cannot decide here which would be the way to go for any file). --- ...te__2-8191-check-restricted-file-terms.sql | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 modules/dataverse-migrate-db/extra-migrations/afterMigrate__2-8191-check-restricted-file-terms.sql diff --git a/modules/dataverse-migrate-db/extra-migrations/afterMigrate__2-8191-check-restricted-file-terms.sql b/modules/dataverse-migrate-db/extra-migrations/afterMigrate__2-8191-check-restricted-file-terms.sql new file mode 100644 index 00000000000..62e39ace8b2 --- /dev/null +++ b/modules/dataverse-migrate-db/extra-migrations/afterMigrate__2-8191-check-restricted-file-terms.sql @@ -0,0 +1,69 @@ +-- this query will identify datasets where at least one file does not have either terms of access or +-- request access enabled, and will include owner information for those datasets + +DO +$$ + DECLARE + affected_dataset RECORD; + row_count INTEGER := 0; + BEGIN + -- Create a temporary table to store the results + CREATE TEMPORARY TABLE IF NOT EXISTS affected_dataset_results ( + email TEXT, + name TEXT, + dataset_url TEXT + ); + + -- Insert the query results into the temporary table + INSERT INTO affected_dataset_results + select au.email, + concat(au.firstname, ' ', au.lastname) as name, + concat('dx.doi.org/' , dvo.authority , '/' , dvo.identifier) as dataset_url + from roleassignment ra, dataverserole dvr, + authenticateduser au, dvobject dvo + where + au.useridentifier = rtrim(substring(ra.assigneeidentifier, 2, 100)) + and dvo.id = ra.definitionpoint_id + and + ra.role_id = dvr.id and + dvr.alias in ( + 'fullContributor', + 'dsContributor', + 'contributor', + 'admin', + 'curator' + ) and + ra.definitionpoint_id in ( + select dvo.id from datasetversion v + join termsofuseandaccess ua on ua.id = v.termsofuseandaccess_id + join filemetadata fm on v.id = fm.datasetversion_id + join datafile f on f.id = fm.datafile_id + join dvobject dvo on v.dataset_id = dvo.id + where ua.fileaccessrequest = false and ua.termsofaccess isnull + and f.restricted = true + ); + + -- Get the number of affected rows + GET DIAGNOSTICS row_count = ROW_COUNT; + + -- Print notice if there are affected datasets + IF row_count > 0 THEN + RAISE NOTICE '--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---'; + RAISE NOTICE 'Found % dataset(s) with files lacking proper access settings.', row_count; + RAISE NOTICE 'For details see Dataverse 5.11 release notes and issue 8191.'; + + -- Loop through affected datasets and print details + FOR affected_dataset IN SELECT * FROM affected_dataset_results LOOP + RAISE NOTICE 'Dataset %, Owner % (%)', + affected_dataset.dataset_url, + affected_dataset.name, + affected_dataset.email; + END LOOP; + + RAISE NOTICE '--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---'; + END IF; + + -- Clean up temporary table + DROP TABLE IF EXISTS affected_dataset_results; + END; +$$; From 868a9eb56b615bb7f143f4c57e5140afd1eedabb Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 2 Jun 2025 09:04:41 +0200 Subject: [PATCH 15/18] chore(migrate): update migration script for keywordTermUri handling Added explanatory comments and references to Dataverse 6.3 release notes to clarify migration. Included guidance for manual loading of the keywordTermURI field and revised notices for better user direction. --- .../V6.2.0.2__migrate_keywordTermUri.sql | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/modules/dataverse-migrate-db/extra-migrations/V6.2.0.2__migrate_keywordTermUri.sql b/modules/dataverse-migrate-db/extra-migrations/V6.2.0.2__migrate_keywordTermUri.sql index ac84a35c3ba..eaefa85c7e3 100644 --- a/modules/dataverse-migrate-db/extra-migrations/V6.2.0.2__migrate_keywordTermUri.sql +++ b/modules/dataverse-migrate-db/extra-migrations/V6.2.0.2__migrate_keywordTermUri.sql @@ -1,4 +1,4 @@ --- +-- See also DO $$ DECLARE @@ -15,8 +15,13 @@ $$ IF keyword_count > 0 THEN RAISE NOTICE '--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---'; RAISE NOTICE 'Found % keywordValue metadata fields starting with "http"', keyword_count; + RAISE NOTICE 'See release notes of Dataverse 6.3 to learn more about migrating these to the keywordTermURI field.'; - IF should_migrate THEN + /* + To execute this migration, the keywordTermURI metadata field needs to be loaded. + So far, we never added metadata fields using a migration. Maybe it's fine to keep it a manual task for now. + + IF should_migrate THEN RAISE NOTICE 'Migrating keywordValue fields with http... to keywordTermUri as requested by -Dmigrate.keywordTermUri'; UPDATE datasetfield df SET datasetfieldtype_id = (SELECT id FROM datasetfieldtype WHERE name = 'keywordTermURI') @@ -31,6 +36,8 @@ $$ ELSE RAISE NOTICE 'Auto-migrate these into keywordTermUri using `mvn -Dmigrate.keywordTermUri ...` (re-execute migrations)'; END IF; + */ + RAISE NOTICE '--- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- --- ---'; END IF; END From 2c82cae36763fffaeb33d60f3a9e03f3305de78c Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 2 Jun 2025 09:06:40 +0200 Subject: [PATCH 16/18] fix(migrate): update pg_dump command options Removed the '-c' and '-C' flags (adding statements to DROP and CREATE the DB) from the pg_dump command, as they are not necessary for the migration process and stop loading the database dump in a container. --- modules/dataverse-migrate-db/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dataverse-migrate-db/pom.xml b/modules/dataverse-migrate-db/pom.xml index 0aa2f850528..ae6a72bd780 100644 --- a/modules/dataverse-migrate-db/pom.xml +++ b/modules/dataverse-migrate-db/pom.xml @@ -132,7 +132,7 @@ sh -c - pg_dump -h migrationdb -p 5432 -U ${postgresql.username} -v -c -C -f /dumptarget/migrated_db_dump.sql ${postgresql.db} + pg_dump -h migrationdb -p 5432 -U ${postgresql.username} -v -f /dumptarget/migrated_db_dump.sql ${postgresql.db} From 6f492093b45951d639d541f894f0b0cd64721af3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 2 Jun 2025 09:07:13 +0200 Subject: [PATCH 17/18] style(migrate): correct volume file name in PostgreSQL config Fixed a typo in the volume file name from "dumb" to "dump" in the Docker PostgreSQL initialization configuration. --- modules/dataverse-migrate-db/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dataverse-migrate-db/pom.xml b/modules/dataverse-migrate-db/pom.xml index ae6a72bd780..567b8535dda 100644 --- a/modules/dataverse-migrate-db/pom.xml +++ b/modules/dataverse-migrate-db/pom.xml @@ -80,7 +80,7 @@ - ${postgresql.dump.file}:/docker-entrypoint-initdb.d/dumb.${postgresql.dump.file.ext} + ${postgresql.dump.file}:/docker-entrypoint-initdb.d/dump.${postgresql.dump.file.ext} From f888def074ee309fc0ad87ceb88a2eb3364bda84 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 30 Jul 2025 14:51:25 +0200 Subject: [PATCH 18/18] chore(migrate): align naming and types as well as references Adjusted existing migration scripts to reference which migration any of the workarounds for missing tables and indices mitigate. Ensured compatibility and alignment with names that would be autogenerated by EclipseLink for better consistency and maintainability. --- .../V5.1.1.0__prepare-table-exttooltype.sql | 19 ++------ .../V5.4.1.0__prepare-table-auxfile.sql | 2 +- .../V5.9.0.0__prepare-table-license.sql | 8 +--- ...0.0__prepare-tables-embargo-storageuse.sql | 4 +- .../V6.1.0.0__prepare-table-extvocab.sql | 2 +- .../V6.3.0.0__prepare-table-datasettype.sql | 48 +++++++++---------- ...V6.5.0.0__prepare-table-mdc-proc-state.sql | 13 ++--- ...0__prepare-table-dataversefeatureditem.sql | 16 +++++++ 8 files changed, 54 insertions(+), 58 deletions(-) create mode 100644 modules/dataverse-migrate-db/extra-migrations/V6.6.0.0__prepare-table-dataversefeatureditem.sql diff --git a/modules/dataverse-migrate-db/extra-migrations/V5.1.1.0__prepare-table-exttooltype.sql b/modules/dataverse-migrate-db/extra-migrations/V5.1.1.0__prepare-table-exttooltype.sql index f6161ddfdd7..a74c8aa1602 100644 --- a/modules/dataverse-migrate-db/extra-migrations/V5.1.1.0__prepare-table-exttooltype.sql +++ b/modules/dataverse-migrate-db/extra-migrations/V5.1.1.0__prepare-table-exttooltype.sql @@ -2,20 +2,9 @@ CREATE TABLE IF NOT EXISTS externaltooltype ( - id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, - type VARCHAR(255) NOT NULL, - externalTool_id BIGINT NOT NULL, - CONSTRAINT pk_externaltooltype PRIMARY KEY (id) + id SERIAL PRIMARY KEY, + type VARCHAR(255) NOT NULL, + externalTool_id BIGINT NOT NULL CONSTRAINT fk_externaltooltype_externaltool_id REFERENCES externaltool (id) ); -ALTER TABLE externaltooltype - ADD CONSTRAINT fk_externaltooltype_externaltool_id FOREIGN KEY (externalTool_id) REFERENCES externaltool (id); - -CREATE INDEX IF NOT EXISTS index_actionlogrecord_actiontype - ON externaltooltype (id); - -CREATE INDEX IF NOT EXISTS index_actionlogrecord_useridentifier - ON externaltooltype (externaltool_id); - -CREATE INDEX IF NOT EXISTS index_externaltooltype_externaltool_id - ON externaltooltype (externaltool_id); +CREATE INDEX IF NOT EXISTS index_externaltooltype_externaltool_id ON externaltooltype (externaltool_id); diff --git a/modules/dataverse-migrate-db/extra-migrations/V5.4.1.0__prepare-table-auxfile.sql b/modules/dataverse-migrate-db/extra-migrations/V5.4.1.0__prepare-table-auxfile.sql index 13a4e78705a..071c80cfc2c 100644 --- a/modules/dataverse-migrate-db/extra-migrations/V5.4.1.0__prepare-table-auxfile.sql +++ b/modules/dataverse-migrate-db/extra-migrations/V5.4.1.0__prepare-table-auxfile.sql @@ -1,4 +1,4 @@ --- This is a workaround for the missing auxiliaryfile table +-- This is a workaround for the missing auxiliaryfile table in migration V5.4.1.1 create table if not exists auxiliaryfile ( diff --git a/modules/dataverse-migrate-db/extra-migrations/V5.9.0.0__prepare-table-license.sql b/modules/dataverse-migrate-db/extra-migrations/V5.9.0.0__prepare-table-license.sql index 8ea09111c8c..df66ccff90f 100644 --- a/modules/dataverse-migrate-db/extra-migrations/V5.9.0.0__prepare-table-license.sql +++ b/modules/dataverse-migrate-db/extra-migrations/V5.9.0.0__prepare-table-license.sql @@ -1,4 +1,4 @@ --- This is a workaround for the missing license table +-- This is a workaround for the missing license table in migration V5.9.0.1 create table if not exists license ( @@ -13,9 +13,3 @@ create table if not exists license ); create index if not exists license_sortorder_id on license (sortorder, id); - --- Save information about CCBY license to temporary table (to be used in a migration after the main license migration) -CREATE TEMPORARY TABLE ccby( - id serial primary key -); -INSERT INTO ccby (id) SELECT id FROM termsofuseandaccess WHERE license = 'CCBY'; diff --git a/modules/dataverse-migrate-db/extra-migrations/V6.0.0.0__prepare-tables-embargo-storageuse.sql b/modules/dataverse-migrate-db/extra-migrations/V6.0.0.0__prepare-tables-embargo-storageuse.sql index f0933da16bb..9d57a82a0d9 100644 --- a/modules/dataverse-migrate-db/extra-migrations/V6.0.0.0__prepare-tables-embargo-storageuse.sql +++ b/modules/dataverse-migrate-db/extra-migrations/V6.0.0.0__prepare-tables-embargo-storageuse.sql @@ -1,4 +1,4 @@ --- This is a workaround for the missing embargo table +-- This is a workaround for the missing embargo table in migration V6.0.0.2 create table if not exists embargo ( id serial primary key, @@ -6,7 +6,7 @@ create table if not exists embargo reason text ); --- This is a workaround for the missing storageuse table +-- This is a workaround for the missing storageuse table in migration V6.0.0.5 create table if not exists storageuse ( id serial primary key, diff --git a/modules/dataverse-migrate-db/extra-migrations/V6.1.0.0__prepare-table-extvocab.sql b/modules/dataverse-migrate-db/extra-migrations/V6.1.0.0__prepare-table-extvocab.sql index 47b0d2a27af..76146039b98 100644 --- a/modules/dataverse-migrate-db/extra-migrations/V6.1.0.0__prepare-table-extvocab.sql +++ b/modules/dataverse-migrate-db/extra-migrations/V6.1.0.0__prepare-table-extvocab.sql @@ -1,4 +1,4 @@ --- This is a workaround for the missing externalvocabularyvalue table +-- This is a workaround for the missing externalvocabularyvalue table in migration V6.1.0.3 create table if not exists externalvocabularyvalue ( id serial primary key, diff --git a/modules/dataverse-migrate-db/extra-migrations/V6.3.0.0__prepare-table-datasettype.sql b/modules/dataverse-migrate-db/extra-migrations/V6.3.0.0__prepare-table-datasettype.sql index 331842666b7..e326c1ec69c 100644 --- a/modules/dataverse-migrate-db/extra-migrations/V6.3.0.0__prepare-table-datasettype.sql +++ b/modules/dataverse-migrate-db/extra-migrations/V6.3.0.0__prepare-table-datasettype.sql @@ -1,33 +1,29 @@ -CREATE TABLE IF NOT EXISTS DatasetType -( - id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, - name VARCHAR(255) NOT NULL, - CONSTRAINT pk_datasettype PRIMARY KEY (id) -); +-- This is a workaround for the missing dataset type tables in migration V6.3.0.3 -CREATE TABLE IF NOT EXISTS DatasetType_licenses +create table if not exists datasettype ( - DatasetType_id BIGINT NOT NULL, - licenses_id BIGINT NOT NULL + id serial primary key, + name varchar(255) not null constraint unq_datasettype_0 unique ); -CREATE TABLE IF NOT EXISTS DatasetType_metadataBlocks +create table if not exists datasettype_licenses ( - DatasetType_id BIGINT NOT NULL, - metadataBlocks_id BIGINT NOT NULL + datasettype_id bigint not null + constraint fk_datasettype_license_datasettype_id + references datasettype, + licenses_id bigint not null + constraint fk_datasettype_license_licenses_id + references license, + primary key (datasettype_id, licenses_id) ); -ALTER TABLE DatasetType - ADD CONSTRAINT uc_828171b214f7369ee92a28781 UNIQUE (name); - -ALTER TABLE DatasetType_licenses - ADD CONSTRAINT fk_datlic_on_dataset_type FOREIGN KEY (DatasetType_id) REFERENCES DatasetType (id); - -ALTER TABLE DatasetType_licenses - ADD CONSTRAINT fk_datlic_on_license FOREIGN KEY (licenses_id) REFERENCES License (id); - -ALTER TABLE DatasetType_metadataBlocks - ADD CONSTRAINT fk_datmet_on_dataset_type FOREIGN KEY (DatasetType_id) REFERENCES DatasetType (id); - -ALTER TABLE DatasetType_metadataBlocks - ADD CONSTRAINT fk_datmet_on_metadata_block FOREIGN KEY (metadataBlocks_id) REFERENCES MetadataBlock (id); \ No newline at end of file +create table if not exists datasettype_metadatablocks +( + datasettype_id bigint not null + constraint fk_datasettype_metadatablock_datasettype_id + references datasettype, + metadatablocks_id bigint not null + constraint fk_datasettype_metadatablock_metadatablocks_id + references public.metadatablock, + primary key (datasettype_id, metadatablocks_id) +); \ No newline at end of file diff --git a/modules/dataverse-migrate-db/extra-migrations/V6.5.0.0__prepare-table-mdc-proc-state.sql b/modules/dataverse-migrate-db/extra-migrations/V6.5.0.0__prepare-table-mdc-proc-state.sql index 23353351e15..0c2f1373721 100644 --- a/modules/dataverse-migrate-db/extra-migrations/V6.5.0.0__prepare-table-mdc-proc-state.sql +++ b/modules/dataverse-migrate-db/extra-migrations/V6.5.0.0__prepare-table-mdc-proc-state.sql @@ -1,11 +1,12 @@ +-- This is a workaround for the missing MakeDataCount tables in migration V6.5.0.10 + CREATE TABLE IF NOT EXISTS MakeDataCountProcessState ( - id BIGINT GENERATED BY DEFAULT AS IDENTITY NOT NULL, - yearMonth VARCHAR(255) NOT NULL, - state INTEGER NOT NULL, + id SERIAL PRIMARY KEY, + yearMonth VARCHAR(255) NOT NULL, + state INTEGER NOT NULL, stateChangeTimestamp TIMESTAMP WITHOUT TIME ZONE, - server VARCHAR(255), - CONSTRAINT pk_makedatacountprocessstate PRIMARY KEY (id) + server VARCHAR(255) ); -CREATE INDEX IF NOT EXISTS idx_18d578c22760ad21e31cc873d ON MakeDataCountProcessState (yearMonth); \ No newline at end of file +CREATE INDEX IF NOT EXISTS index_makedatacountprocessstate_yearmonth ON MakeDataCountProcessState (yearMonth); diff --git a/modules/dataverse-migrate-db/extra-migrations/V6.6.0.0__prepare-table-dataversefeatureditem.sql b/modules/dataverse-migrate-db/extra-migrations/V6.6.0.0__prepare-table-dataversefeatureditem.sql new file mode 100644 index 00000000000..15ca0d6e31b --- /dev/null +++ b/modules/dataverse-migrate-db/extra-migrations/V6.6.0.0__prepare-table-dataversefeatureditem.sql @@ -0,0 +1,16 @@ +-- This is a workaround for the missing tables in migration V6.6.0.2 + +create table if not exists dataversefeatureditem +( + id serial primary key, + content text, + displayorder integer not null, + imagefilename varchar(255), + type text, + dataverse_id bigint not null constraint fk_dataversefeatureditem_dataverse_id references dvobject, + dvobject_id bigint constraint fk_dataversefeatureditem_dvobject_id references dvobject +); + +create index if not exists index_dataversefeatureditem_displayorder on dataversefeatureditem (displayorder); +-- It's unclear why EclipseLink generated this particular index and with this name... Just going along with it. +create index if not exists index_harvestingclient_harvesttype on dataversefeatureditem (id);