diff --git a/validation/chpa/01_dwd/validate_hierarchy_refactor.sql b/validation/chpa/01_dwd/validate_hierarchy_refactor.sql index 6983f97..23152c0 100644 --- a/validation/chpa/01_dwd/validate_hierarchy_refactor.sql +++ b/validation/chpa/01_dwd/validate_hierarchy_refactor.sql @@ -134,6 +134,7 @@ LEFT JOIN nfc_level_3 -- COMMAND ---------- -- Every check must return passed = true before replacing the scheduled jobs. +CREATE OR REPLACE TEMP VIEW validation_hierarchy_compatibility_checks AS WITH compatibility_checks AS ( SELECT 'atc_row_count' AS check_name, @@ -145,9 +146,35 @@ WITH compatibility_checks AS ( ( SELECT COUNT(*) FROM ( - SELECT * FROM dwd.dwd_ims_atc_hierarchy + SELECT + ATC1_ID, + ATC1_CODE, + ATC1_DES, + ATC2_ID, + ATC2_CODE, + ATC2_DES, + ATC3_ID, + ATC3_CODE, + ATC3_DES, + ATC4_ID, + ATC4_CODE, + ATC4_DES + FROM dwd.dwd_ims_atc_hierarchy EXCEPT ALL - SELECT * FROM validation_atc_hierarchy_candidate + SELECT + ATC1_ID, + ATC1_CODE, + ATC1_DES, + ATC2_ID, + ATC2_CODE, + ATC2_DES, + ATC3_ID, + ATC3_CODE, + ATC3_DES, + ATC4_ID, + ATC4_CODE, + ATC4_DES + FROM validation_atc_hierarchy_candidate ) AS differences ) AS actual_value, 0 AS expected_value @@ -157,9 +184,35 @@ WITH compatibility_checks AS ( ( SELECT COUNT(*) FROM ( - SELECT * FROM validation_atc_hierarchy_candidate + SELECT + ATC1_ID, + ATC1_CODE, + ATC1_DES, + ATC2_ID, + ATC2_CODE, + ATC2_DES, + ATC3_ID, + ATC3_CODE, + ATC3_DES, + ATC4_ID, + ATC4_CODE, + ATC4_DES + FROM validation_atc_hierarchy_candidate EXCEPT ALL - SELECT * FROM dwd.dwd_ims_atc_hierarchy + SELECT + ATC1_ID, + ATC1_CODE, + ATC1_DES, + ATC2_ID, + ATC2_CODE, + ATC2_DES, + ATC3_ID, + ATC3_CODE, + ATC3_DES, + ATC4_ID, + ATC4_CODE, + ATC4_DES + FROM dwd.dwd_ims_atc_hierarchy ) AS differences ) AS actual_value, 0 AS expected_value @@ -174,9 +227,29 @@ WITH compatibility_checks AS ( ( SELECT COUNT(*) FROM ( - SELECT * FROM dwd.dwd_ims_nfc_hierarchy + SELECT + NFC1_ID, + NFC1_CODE, + NFC1_DES, + NFC2_ID, + NFC2_CODE, + NFC2_DES, + NFC3_ID, + NFC3_CODE, + NFC3_DES + FROM dwd.dwd_ims_nfc_hierarchy EXCEPT ALL - SELECT * FROM validation_nfc_hierarchy_candidate + SELECT + NFC1_ID, + NFC1_CODE, + NFC1_DES, + NFC2_ID, + NFC2_CODE, + NFC2_DES, + NFC3_ID, + NFC3_CODE, + NFC3_DES + FROM validation_nfc_hierarchy_candidate ) AS differences ) AS actual_value, 0 AS expected_value @@ -186,9 +259,41 @@ WITH compatibility_checks AS ( ( SELECT COUNT(*) FROM ( - SELECT * FROM validation_nfc_hierarchy_candidate + SELECT + NFC1_ID, + NFC1_CODE, + NFC1_DES, + NFC2_ID, + NFC2_CODE, + NFC2_DES, + NFC3_ID, + NFC3_CODE, + NFC3_DES + FROM validation_nfc_hierarchy_candidate EXCEPT ALL - SELECT * FROM dwd.dwd_ims_nfc_hierarchy + SELECT + NFC1_ID, + NFC1_CODE, + NFC1_DES, + NFC2_ID, + NFC2_CODE, + NFC2_DES, + NFC3_ID, + NFC3_CODE, + NFC3_DES + FROM dwd.dwd_ims_nfc_hierarchy + ) AS differences + ) AS actual_value, + 0 AS expected_value + UNION ALL + SELECT + 'except_all_null_semantics' AS check_name, + ( + SELECT COUNT(*) + FROM ( + SELECT CAST(NULL AS STRING) AS nullable_value + EXCEPT ALL + SELECT CAST(NULL AS STRING) AS nullable_value ) AS differences ) AS actual_value, 0 AS expected_value @@ -199,6 +304,16 @@ SELECT expected_value, actual_value = expected_value AS passed FROM compatibility_checks +; + +-- COMMAND ---------- + +SELECT + check_name, + actual_value, + expected_value, + passed +FROM validation_hierarchy_compatibility_checks ORDER BY check_name; -- COMMAND ---------- @@ -222,9 +337,10 @@ SELECT FROM atc_duplicate_paths UNION ALL SELECT - 'atc_null_root_code' AS metric_name, - COUNT_IF(ATC1_CODE IS NULL) AS metric_value -FROM validation_atc_hierarchy_candidate + 'atc_source_level_1_required_key_null' AS metric_name, + COUNT_IF(therapeutic_id IS NULL OR therapeutic_code IS NULL) AS metric_value +FROM dwd.dwd_ims_td_therapeutic_class +WHERE therapeutic_level = '1' UNION ALL SELECT 'atc_unmatched_level_2' AS metric_name, @@ -247,9 +363,10 @@ SELECT FROM nfc_duplicate_paths UNION ALL SELECT - 'nfc_null_root_code' AS metric_name, - COUNT_IF(NFC1_CODE IS NULL) AS metric_value -FROM validation_nfc_hierarchy_candidate + 'nfc_source_level_1_required_key_null' AS metric_name, + COUNT_IF(newformclass_id IS NULL OR newformclass_code IS NULL) AS metric_value +FROM dwd.dwd_ims_td_new_form_class +WHERE newformclass_level = '1' UNION ALL SELECT 'nfc_unmatched_level_2' AS metric_name, @@ -261,3 +378,19 @@ SELECT COUNT_IF(NFC3_CODE IS NULL) AS metric_value FROM validation_nfc_hierarchy_candidate ORDER BY metric_name; + +-- COMMAND ---------- + +-- Abort the notebook when any compatibility check fails. +SELECT IF( + COUNT_IF(NOT passed) = 0, + 'All hierarchy compatibility checks passed.', + raise_error( + CONCAT( + 'Hierarchy compatibility validation failed: ', + CAST(COUNT_IF(NOT passed) AS STRING), + ' check(s) did not pass.' + ) + ) +) AS validation_result +FROM validation_hierarchy_compatibility_checks;