chenwu/REFACTOR-MA
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

harden hierarchy validation checks

Browse Source
This commit is contained in:
chenwu
2026-08-20 18:48:52 +08:00
parent 3f3dc443da
commit faddf38c9b
1 changed files with 147 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
validation/chpa/01_dwd/validate_hierarchy_refactor.sql
+147 -14
View File
@@ -134,6 +134,7 @@ LEFT JOIN nfc_level_3
-- COMMAND ---------- -- COMMAND ----------
-- Every check must return passed = true before replacing the scheduled jobs. -- Every check must return passed = true before replacing the scheduled jobs.
CREATE OR REPLACE TEMP VIEW validation_hierarchy_compatibility_checks AS
WITH compatibility_checks AS ( WITH compatibility_checks AS (
SELECT SELECT
'atc_row_count' AS check_name, 'atc_row_count' AS check_name,
@@ -145,9 +146,35 @@ WITH compatibility_checks AS (
( (
SELECT COUNT(*) SELECT COUNT(*)
FROM ( FROM (
SELECT * FROM dwd.dwd_ims_atc_hierarchy SELECT
ATC1_ID,
ATC1_CODE,
ATC1_DES,
ATC2_ID,
ATC2_CODE,
ATC2_DES,
ATC3_ID,
ATC3_CODE,
ATC3_DES,
ATC4_ID,
ATC4_CODE,
ATC4_DES
FROM dwd.dwd_ims_atc_hierarchy
EXCEPT ALL EXCEPT ALL
SELECT * FROM validation_atc_hierarchy_candidate SELECT
ATC1_ID,
ATC1_CODE,
ATC1_DES,
ATC2_ID,
ATC2_CODE,
ATC2_DES,
ATC3_ID,
ATC3_CODE,
ATC3_DES,
ATC4_ID,
ATC4_CODE,
ATC4_DES
FROM validation_atc_hierarchy_candidate
) AS differences ) AS differences
) AS actual_value, ) AS actual_value,
0 AS expected_value 0 AS expected_value
@@ -157,9 +184,35 @@ WITH compatibility_checks AS (
( (
SELECT COUNT(*) SELECT COUNT(*)
FROM ( FROM (
SELECT * FROM validation_atc_hierarchy_candidate SELECT
ATC1_ID,
ATC1_CODE,
ATC1_DES,
ATC2_ID,
ATC2_CODE,
ATC2_DES,
ATC3_ID,
ATC3_CODE,
ATC3_DES,
ATC4_ID,
ATC4_CODE,
ATC4_DES
FROM validation_atc_hierarchy_candidate
EXCEPT ALL EXCEPT ALL
SELECT * FROM dwd.dwd_ims_atc_hierarchy SELECT
ATC1_ID,
ATC1_CODE,
ATC1_DES,
ATC2_ID,
ATC2_CODE,
ATC2_DES,
ATC3_ID,
ATC3_CODE,
ATC3_DES,
ATC4_ID,
ATC4_CODE,
ATC4_DES
FROM dwd.dwd_ims_atc_hierarchy
) AS differences ) AS differences
) AS actual_value, ) AS actual_value,
0 AS expected_value 0 AS expected_value
@@ -174,9 +227,29 @@ WITH compatibility_checks AS (
( (
SELECT COUNT(*) SELECT COUNT(*)
FROM ( FROM (
SELECT * FROM dwd.dwd_ims_nfc_hierarchy SELECT
NFC1_ID,
NFC1_CODE,
NFC1_DES,
NFC2_ID,
NFC2_CODE,
NFC2_DES,
NFC3_ID,
NFC3_CODE,
NFC3_DES
FROM dwd.dwd_ims_nfc_hierarchy
EXCEPT ALL EXCEPT ALL
SELECT * FROM validation_nfc_hierarchy_candidate SELECT
NFC1_ID,
NFC1_CODE,
NFC1_DES,
NFC2_ID,
NFC2_CODE,
NFC2_DES,
NFC3_ID,
NFC3_CODE,
NFC3_DES
FROM validation_nfc_hierarchy_candidate
) AS differences ) AS differences
) AS actual_value, ) AS actual_value,
0 AS expected_value 0 AS expected_value
@@ -186,9 +259,41 @@ WITH compatibility_checks AS (
( (
SELECT COUNT(*) SELECT COUNT(*)
FROM ( FROM (
SELECT * FROM validation_nfc_hierarchy_candidate SELECT
NFC1_ID,
NFC1_CODE,
NFC1_DES,
NFC2_ID,
NFC2_CODE,
NFC2_DES,
NFC3_ID,
NFC3_CODE,
NFC3_DES
FROM validation_nfc_hierarchy_candidate
EXCEPT ALL EXCEPT ALL
SELECT * FROM dwd.dwd_ims_nfc_hierarchy SELECT
NFC1_ID,
NFC1_CODE,
NFC1_DES,
NFC2_ID,
NFC2_CODE,
NFC2_DES,
NFC3_ID,
NFC3_CODE,
NFC3_DES
FROM dwd.dwd_ims_nfc_hierarchy
) AS differences
) AS actual_value,
0 AS expected_value
UNION ALL
SELECT
'except_all_null_semantics' AS check_name,
(
SELECT COUNT(*)
FROM (
SELECT CAST(NULL AS STRING) AS nullable_value
EXCEPT ALL
SELECT CAST(NULL AS STRING) AS nullable_value
) AS differences ) AS differences
) AS actual_value, ) AS actual_value,
0 AS expected_value 0 AS expected_value
@@ -199,6 +304,16 @@ SELECT
expected_value, expected_value,
actual_value = expected_value AS passed actual_value = expected_value AS passed
FROM compatibility_checks FROM compatibility_checks
;
-- COMMAND ----------
SELECT
check_name,
actual_value,
expected_value,
passed
FROM validation_hierarchy_compatibility_checks
ORDER BY check_name; ORDER BY check_name;
-- COMMAND ---------- -- COMMAND ----------
@@ -222,9 +337,10 @@ SELECT
FROM atc_duplicate_paths FROM atc_duplicate_paths
UNION ALL UNION ALL
SELECT SELECT
'atc_null_root_code' AS metric_name, 'atc_source_level_1_required_key_null' AS metric_name,
COUNT_IF(ATC1_CODE IS NULL) AS metric_value COUNT_IF(therapeutic_id IS NULL OR therapeutic_code IS NULL) AS metric_value
FROM validation_atc_hierarchy_candidate FROM dwd.dwd_ims_td_therapeutic_class
WHERE therapeutic_level = '1'
UNION ALL UNION ALL
SELECT SELECT
'atc_unmatched_level_2' AS metric_name, 'atc_unmatched_level_2' AS metric_name,
@@ -247,9 +363,10 @@ SELECT
FROM nfc_duplicate_paths FROM nfc_duplicate_paths
UNION ALL UNION ALL
SELECT SELECT
'nfc_null_root_code' AS metric_name, 'nfc_source_level_1_required_key_null' AS metric_name,
COUNT_IF(NFC1_CODE IS NULL) AS metric_value COUNT_IF(newformclass_id IS NULL OR newformclass_code IS NULL) AS metric_value
FROM validation_nfc_hierarchy_candidate FROM dwd.dwd_ims_td_new_form_class
WHERE newformclass_level = '1'
UNION ALL UNION ALL
SELECT SELECT
'nfc_unmatched_level_2' AS metric_name, 'nfc_unmatched_level_2' AS metric_name,
@@ -261,3 +378,19 @@ SELECT
COUNT_IF(NFC3_CODE IS NULL) AS metric_value COUNT_IF(NFC3_CODE IS NULL) AS metric_value
FROM validation_nfc_hierarchy_candidate FROM validation_nfc_hierarchy_candidate
ORDER BY metric_name; ORDER BY metric_name;
-- COMMAND ----------
-- Abort the notebook when any compatibility check fails.
SELECT IF(
COUNT_IF(NOT passed) = 0,
'All hierarchy compatibility checks passed.',
raise_error(
CONCAT(
'Hierarchy compatibility validation failed: ',
CAST(COUNT_IF(NOT passed) AS STRING),
' check(s) did not pass.'
)
)
) AS validation_result
FROM validation_hierarchy_compatibility_checks;