harden hierarchy validation checks
This commit is contained in:
@@ -134,6 +134,7 @@ LEFT JOIN nfc_level_3
|
||||
-- COMMAND ----------
|
||||
|
||||
-- Every check must return passed = true before replacing the scheduled jobs.
|
||||
CREATE OR REPLACE TEMP VIEW validation_hierarchy_compatibility_checks AS
|
||||
WITH compatibility_checks AS (
|
||||
SELECT
|
||||
'atc_row_count' AS check_name,
|
||||
@@ -145,9 +146,35 @@ WITH compatibility_checks AS (
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM (
|
||||
SELECT * FROM dwd.dwd_ims_atc_hierarchy
|
||||
SELECT
|
||||
ATC1_ID,
|
||||
ATC1_CODE,
|
||||
ATC1_DES,
|
||||
ATC2_ID,
|
||||
ATC2_CODE,
|
||||
ATC2_DES,
|
||||
ATC3_ID,
|
||||
ATC3_CODE,
|
||||
ATC3_DES,
|
||||
ATC4_ID,
|
||||
ATC4_CODE,
|
||||
ATC4_DES
|
||||
FROM dwd.dwd_ims_atc_hierarchy
|
||||
EXCEPT ALL
|
||||
SELECT * FROM validation_atc_hierarchy_candidate
|
||||
SELECT
|
||||
ATC1_ID,
|
||||
ATC1_CODE,
|
||||
ATC1_DES,
|
||||
ATC2_ID,
|
||||
ATC2_CODE,
|
||||
ATC2_DES,
|
||||
ATC3_ID,
|
||||
ATC3_CODE,
|
||||
ATC3_DES,
|
||||
ATC4_ID,
|
||||
ATC4_CODE,
|
||||
ATC4_DES
|
||||
FROM validation_atc_hierarchy_candidate
|
||||
) AS differences
|
||||
) AS actual_value,
|
||||
0 AS expected_value
|
||||
@@ -157,9 +184,35 @@ WITH compatibility_checks AS (
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM (
|
||||
SELECT * FROM validation_atc_hierarchy_candidate
|
||||
SELECT
|
||||
ATC1_ID,
|
||||
ATC1_CODE,
|
||||
ATC1_DES,
|
||||
ATC2_ID,
|
||||
ATC2_CODE,
|
||||
ATC2_DES,
|
||||
ATC3_ID,
|
||||
ATC3_CODE,
|
||||
ATC3_DES,
|
||||
ATC4_ID,
|
||||
ATC4_CODE,
|
||||
ATC4_DES
|
||||
FROM validation_atc_hierarchy_candidate
|
||||
EXCEPT ALL
|
||||
SELECT * FROM dwd.dwd_ims_atc_hierarchy
|
||||
SELECT
|
||||
ATC1_ID,
|
||||
ATC1_CODE,
|
||||
ATC1_DES,
|
||||
ATC2_ID,
|
||||
ATC2_CODE,
|
||||
ATC2_DES,
|
||||
ATC3_ID,
|
||||
ATC3_CODE,
|
||||
ATC3_DES,
|
||||
ATC4_ID,
|
||||
ATC4_CODE,
|
||||
ATC4_DES
|
||||
FROM dwd.dwd_ims_atc_hierarchy
|
||||
) AS differences
|
||||
) AS actual_value,
|
||||
0 AS expected_value
|
||||
@@ -174,9 +227,29 @@ WITH compatibility_checks AS (
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM (
|
||||
SELECT * FROM dwd.dwd_ims_nfc_hierarchy
|
||||
SELECT
|
||||
NFC1_ID,
|
||||
NFC1_CODE,
|
||||
NFC1_DES,
|
||||
NFC2_ID,
|
||||
NFC2_CODE,
|
||||
NFC2_DES,
|
||||
NFC3_ID,
|
||||
NFC3_CODE,
|
||||
NFC3_DES
|
||||
FROM dwd.dwd_ims_nfc_hierarchy
|
||||
EXCEPT ALL
|
||||
SELECT * FROM validation_nfc_hierarchy_candidate
|
||||
SELECT
|
||||
NFC1_ID,
|
||||
NFC1_CODE,
|
||||
NFC1_DES,
|
||||
NFC2_ID,
|
||||
NFC2_CODE,
|
||||
NFC2_DES,
|
||||
NFC3_ID,
|
||||
NFC3_CODE,
|
||||
NFC3_DES
|
||||
FROM validation_nfc_hierarchy_candidate
|
||||
) AS differences
|
||||
) AS actual_value,
|
||||
0 AS expected_value
|
||||
@@ -186,9 +259,41 @@ WITH compatibility_checks AS (
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM (
|
||||
SELECT * FROM validation_nfc_hierarchy_candidate
|
||||
SELECT
|
||||
NFC1_ID,
|
||||
NFC1_CODE,
|
||||
NFC1_DES,
|
||||
NFC2_ID,
|
||||
NFC2_CODE,
|
||||
NFC2_DES,
|
||||
NFC3_ID,
|
||||
NFC3_CODE,
|
||||
NFC3_DES
|
||||
FROM validation_nfc_hierarchy_candidate
|
||||
EXCEPT ALL
|
||||
SELECT * FROM dwd.dwd_ims_nfc_hierarchy
|
||||
SELECT
|
||||
NFC1_ID,
|
||||
NFC1_CODE,
|
||||
NFC1_DES,
|
||||
NFC2_ID,
|
||||
NFC2_CODE,
|
||||
NFC2_DES,
|
||||
NFC3_ID,
|
||||
NFC3_CODE,
|
||||
NFC3_DES
|
||||
FROM dwd.dwd_ims_nfc_hierarchy
|
||||
) AS differences
|
||||
) AS actual_value,
|
||||
0 AS expected_value
|
||||
UNION ALL
|
||||
SELECT
|
||||
'except_all_null_semantics' AS check_name,
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM (
|
||||
SELECT CAST(NULL AS STRING) AS nullable_value
|
||||
EXCEPT ALL
|
||||
SELECT CAST(NULL AS STRING) AS nullable_value
|
||||
) AS differences
|
||||
) AS actual_value,
|
||||
0 AS expected_value
|
||||
@@ -199,6 +304,16 @@ SELECT
|
||||
expected_value,
|
||||
actual_value = expected_value AS passed
|
||||
FROM compatibility_checks
|
||||
;
|
||||
|
||||
-- COMMAND ----------
|
||||
|
||||
SELECT
|
||||
check_name,
|
||||
actual_value,
|
||||
expected_value,
|
||||
passed
|
||||
FROM validation_hierarchy_compatibility_checks
|
||||
ORDER BY check_name;
|
||||
|
||||
-- COMMAND ----------
|
||||
@@ -222,9 +337,10 @@ SELECT
|
||||
FROM atc_duplicate_paths
|
||||
UNION ALL
|
||||
SELECT
|
||||
'atc_null_root_code' AS metric_name,
|
||||
COUNT_IF(ATC1_CODE IS NULL) AS metric_value
|
||||
FROM validation_atc_hierarchy_candidate
|
||||
'atc_source_level_1_required_key_null' AS metric_name,
|
||||
COUNT_IF(therapeutic_id IS NULL OR therapeutic_code IS NULL) AS metric_value
|
||||
FROM dwd.dwd_ims_td_therapeutic_class
|
||||
WHERE therapeutic_level = '1'
|
||||
UNION ALL
|
||||
SELECT
|
||||
'atc_unmatched_level_2' AS metric_name,
|
||||
@@ -247,9 +363,10 @@ SELECT
|
||||
FROM nfc_duplicate_paths
|
||||
UNION ALL
|
||||
SELECT
|
||||
'nfc_null_root_code' AS metric_name,
|
||||
COUNT_IF(NFC1_CODE IS NULL) AS metric_value
|
||||
FROM validation_nfc_hierarchy_candidate
|
||||
'nfc_source_level_1_required_key_null' AS metric_name,
|
||||
COUNT_IF(newformclass_id IS NULL OR newformclass_code IS NULL) AS metric_value
|
||||
FROM dwd.dwd_ims_td_new_form_class
|
||||
WHERE newformclass_level = '1'
|
||||
UNION ALL
|
||||
SELECT
|
||||
'nfc_unmatched_level_2' AS metric_name,
|
||||
@@ -261,3 +378,19 @@ SELECT
|
||||
COUNT_IF(NFC3_CODE IS NULL) AS metric_value
|
||||
FROM validation_nfc_hierarchy_candidate
|
||||
ORDER BY metric_name;
|
||||
|
||||
-- COMMAND ----------
|
||||
|
||||
-- Abort the notebook when any compatibility check fails.
|
||||
SELECT IF(
|
||||
COUNT_IF(NOT passed) = 0,
|
||||
'All hierarchy compatibility checks passed.',
|
||||
raise_error(
|
||||
CONCAT(
|
||||
'Hierarchy compatibility validation failed: ',
|
||||
CAST(COUNT_IF(NOT passed) AS STRING),
|
||||
' check(s) did not pass.'
|
||||
)
|
||||
)
|
||||
) AS validation_result
|
||||
FROM validation_hierarchy_compatibility_checks;
|
||||
|
||||
Reference in New Issue
Block a user