From 378b188849075403da2746947d69608f758ef6eb Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Fri, 8 May 2026 07:10:04 +0000 Subject: [PATCH 1/8] added validation_config.json --- scripts/world_bank/wdi/validation_config.json | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 scripts/world_bank/wdi/validation_config.json diff --git a/scripts/world_bank/wdi/validation_config.json b/scripts/world_bank/wdi/validation_config.json new file mode 100644 index 0000000000..cfdd7f9aa0 --- /dev/null +++ b/scripts/world_bank/wdi/validation_config.json @@ -0,0 +1,28 @@ +{ + "schema_version": "1.0", + "rules": [ + + { + "rule_id": "check_deleted_records_count", + "validator": "DELETED_RECORDS_COUNT", + "params": { + "threshold": 340 + } + }, + { + "rule_id": "check_goldens_output_csv", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "golden_WorldBank.csv", + "input_files": "WorldBank.csv" + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "golden_data/golden_summary_report.csv" + } + } + ] +} \ No newline at end of file From 839f6a025ad53db662caf92ed1102e4d41bbde23 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Fri, 8 May 2026 08:02:06 +0000 Subject: [PATCH 2/8] added validation_config.json --- scripts/world_bank/wdi/validation_config.json | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/scripts/world_bank/wdi/validation_config.json b/scripts/world_bank/wdi/validation_config.json index cfdd7f9aa0..b4500a376b 100644 --- a/scripts/world_bank/wdi/validation_config.json +++ b/scripts/world_bank/wdi/validation_config.json @@ -1,28 +1,27 @@ { - "schema_version": "1.0", - "rules": [ - - { - "rule_id": "check_deleted_records_count", - "validator": "DELETED_RECORDS_COUNT", - "params": { - "threshold": 340 - } - }, - { - "rule_id": "check_goldens_output_csv", - "validator": "GOLDENS_CHECK", - "params": { - "golden_files": "golden_WorldBank.csv", - "input_files": "WorldBank.csv" - } - }, - { - "rule_id": "check_goldens_summary_report", - "validator": "GOLDENS_CHECK", - "params": { - "golden_files": "golden_data/golden_summary_report.csv" - } - } - ] + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_count", + "validator": "DELETED_RECORDS_COUNT", + "params": { + "threshold": 340 + } + }, + { + "rule_id": "check_goldens_output_csv", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "golden_data/golden_WorldBank.csv", + "input_files": "output/WorldBank.csv" + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "golden_data/golden_summary_report.csv" + } + } + ] } \ No newline at end of file From faccf01ba6ce89203579c9a01e8663ddda7119f6 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Fri, 8 May 2026 09:02:30 +0000 Subject: [PATCH 3/8] added validation_config.json --- scripts/world_bank/wdi/validation_config.json | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/scripts/world_bank/wdi/validation_config.json b/scripts/world_bank/wdi/validation_config.json index b4500a376b..2f65a4ac25 100644 --- a/scripts/world_bank/wdi/validation_config.json +++ b/scripts/world_bank/wdi/validation_config.json @@ -7,21 +7,6 @@ "params": { "threshold": 340 } - }, - { - "rule_id": "check_goldens_output_csv", - "validator": "GOLDENS_CHECK", - "params": { - "golden_files": "golden_data/golden_WorldBank.csv", - "input_files": "output/WorldBank.csv" - } - }, - { - "rule_id": "check_goldens_summary_report", - "validator": "GOLDENS_CHECK", - "params": { - "golden_files": "golden_data/golden_summary_report.csv" - } } ] } \ No newline at end of file From 7be6ddd9c79e2eeeee927af5074180466da9174e Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Fri, 8 May 2026 09:53:39 +0000 Subject: [PATCH 4/8] added validation_config.json --- scripts/world_bank/wdi/validation_config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/world_bank/wdi/validation_config.json b/scripts/world_bank/wdi/validation_config.json index 2f65a4ac25..be37af0f79 100644 --- a/scripts/world_bank/wdi/validation_config.json +++ b/scripts/world_bank/wdi/validation_config.json @@ -3,9 +3,9 @@ "rules": [ { "rule_id": "check_deleted_records_count", - "validator": "DELETED_RECORDS_COUNT", + "validator": "DELETED_RECORDS_PERCENT", "params": { - "threshold": 340 + "threshold": 0.01 } } ] From 4d1fdee984073f58df2d68ee46bbaff3a27ae46e Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Mon, 11 May 2026 09:51:35 +0000 Subject: [PATCH 5/8] changes made --- scripts/world_bank/wdi/validation_config.json | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/scripts/world_bank/wdi/validation_config.json b/scripts/world_bank/wdi/validation_config.json index be37af0f79..5a43083867 100644 --- a/scripts/world_bank/wdi/validation_config.json +++ b/scripts/world_bank/wdi/validation_config.json @@ -1,12 +1,28 @@ { - "schema_version": "1.0", - "rules": [ - { - "rule_id": "check_deleted_records_count", - "validator": "DELETED_RECORDS_PERCENT", - "params": { - "threshold": 0.01 - } - } - ] + "schema_version": "1.0", + "rules": [ + + { + "rule_id": "check_deleted_points_threshold", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 7 + } + }, + { + "rule_id": "check_goldens_output_csv", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "golden_data/golden_WorldBank.csv", + "input_files": "WorldBank.csv" + } + }, + { + "rule_id": "check_goldens_summary_report", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "golden_data/golden_summary_report.csv" + } + } + ] } \ No newline at end of file From 9ef40b8ba5fbaa3c813771920ac181cab1f1230a Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Tue, 12 May 2026 05:20:27 +0000 Subject: [PATCH 6/8] fixed validation config --- scripts/world_bank/wdi/validation_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/world_bank/wdi/validation_config.json b/scripts/world_bank/wdi/validation_config.json index 5a43083867..4cbe6d7d2f 100644 --- a/scripts/world_bank/wdi/validation_config.json +++ b/scripts/world_bank/wdi/validation_config.json @@ -14,7 +14,7 @@ "validator": "GOLDENS_CHECK", "params": { "golden_files": "golden_data/golden_WorldBank.csv", - "input_files": "WorldBank.csv" + "input_files": "output/WorldBank.csv" } }, { From 50ffd7fb65fb8f876d47e45cdf0cd051f00813e7 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Tue, 12 May 2026 13:00:01 +0000 Subject: [PATCH 7/8] fixed json --- scripts/world_bank/wdi/validation_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/world_bank/wdi/validation_config.json b/scripts/world_bank/wdi/validation_config.json index 4cbe6d7d2f..9c2948dd08 100644 --- a/scripts/world_bank/wdi/validation_config.json +++ b/scripts/world_bank/wdi/validation_config.json @@ -6,7 +6,7 @@ "rule_id": "check_deleted_points_threshold", "validator": "DELETED_RECORDS_PERCENT", "params": { - "threshold": 7 + "threshold": 0.08 } }, { From 49a82cd8996023a7777dc0fa5bfb75425580302b Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Tue, 12 May 2026 17:45:05 +0000 Subject: [PATCH 8/8] fixed json --- scripts/world_bank/wdi/manifest.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/world_bank/wdi/manifest.json b/scripts/world_bank/wdi/manifest.json index bc3927141e..eb427c0472 100644 --- a/scripts/world_bank/wdi/manifest.json +++ b/scripts/world_bank/wdi/manifest.json @@ -20,7 +20,8 @@ "WorldBankCountries.csv", "schema_csvs/WorldBankIndicators_prod.csv" ], - "cron_schedule": "0 11 * * 2" + "cron_schedule": "0 11 * * 2", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file