chenwu/REFACTOR-MA
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

align hierarchy outputs with DWS naming

Browse Source
This commit is contained in:
chenwu
2026-08-20 19:01:46 +08:00
parent faddf38c9b
commit 33e60d02e3
6 changed files with 380 additions and 487 deletions
Download Patch File Download Diff File Expand all files Collapse all files
validation/chpa/01_dwd/validate_hierarchy_refactor.sql
-396
View File
@@ -1,396 +0,0 @@
-- Databricks notebook source
-- MAGIC %md
-- MAGIC # ATC/NFC hierarchy refactor validation
-- MAGIC Run this notebook before executing the refactored overwrite scripts.
-- MAGIC The legacy targets and source tables must represent the same data snapshot.
-- COMMAND ----------
-- Confirm the ATC target contains the expected 12 non-partition output columns.
DESCRIBE TABLE dwd.dwd_ims_atc_hierarchy;
-- COMMAND ----------
-- Confirm the NFC target contains the expected 9 non-partition output columns.
DESCRIBE TABLE dwd.dwd_ims_nfc_hierarchy;
-- COMMAND ----------
CREATE OR REPLACE TEMP VIEW validation_atc_hierarchy_candidate AS
WITH therapeutic_class AS (
SELECT
therapeutic_id,
therapeutic_code,
therapeutic_name,
therapeutic_level
FROM dwd.dwd_ims_td_therapeutic_class
),
atc_level_1 AS (
SELECT
therapeutic_id AS atc1_id,
therapeutic_code AS atc1_code,
therapeutic_name AS atc1_des
FROM therapeutic_class
WHERE therapeutic_level = '1'
),
atc_level_2 AS (
SELECT
therapeutic_id AS atc2_id,
therapeutic_code AS atc2_code,
therapeutic_name AS atc2_des
FROM therapeutic_class
WHERE therapeutic_level = '2'
),
atc_level_3 AS (
SELECT
therapeutic_id AS atc3_id,
therapeutic_code AS atc3_code,
therapeutic_name AS atc3_des
FROM therapeutic_class
WHERE therapeutic_level = '3'
),
atc_level_4 AS (
SELECT
therapeutic_id AS atc4_id,
therapeutic_code AS atc4_code,
therapeutic_name AS atc4_des
FROM therapeutic_class
WHERE therapeutic_level = '4'
)
SELECT
atc_level_1.atc1_id AS ATC1_ID,
atc_level_1.atc1_code AS ATC1_CODE,
atc_level_1.atc1_des AS ATC1_DES,
atc_level_2.atc2_id AS ATC2_ID,
atc_level_2.atc2_code AS ATC2_CODE,
atc_level_2.atc2_des AS ATC2_DES,
atc_level_3.atc3_id AS ATC3_ID,
atc_level_3.atc3_code AS ATC3_CODE,
atc_level_3.atc3_des AS ATC3_DES,
atc_level_4.atc4_id AS ATC4_ID,
atc_level_4.atc4_code AS ATC4_CODE,
atc_level_4.atc4_des AS ATC4_DES
FROM atc_level_1
LEFT JOIN atc_level_2
ON atc_level_1.atc1_code = LEFT(atc_level_2.atc2_code, 1)
LEFT JOIN atc_level_3
ON atc_level_2.atc2_code = LEFT(atc_level_3.atc3_code, 3)
LEFT JOIN atc_level_4
ON atc_level_3.atc3_code = LEFT(atc_level_4.atc4_code, 4)
;
-- COMMAND ----------
CREATE OR REPLACE TEMP VIEW validation_nfc_hierarchy_candidate AS
WITH new_form_class AS (
SELECT
newformclass_id,
newformclass_code,
newformclass_name,
newformclass_level
FROM dwd.dwd_ims_td_new_form_class
),
nfc_level_1 AS (
SELECT
newformclass_id AS nfc1_id,
newformclass_code AS nfc1_code,
newformclass_name AS nfc1_des
FROM new_form_class
WHERE newformclass_level = '1'
),
nfc_level_2 AS (
SELECT
newformclass_id AS nfc2_id,
newformclass_code AS nfc2_code,
newformclass_name AS nfc2_des
FROM new_form_class
WHERE newformclass_level = '2'
),
nfc_level_3 AS (
SELECT
newformclass_id AS nfc3_id,
newformclass_code AS nfc3_code,
newformclass_name AS nfc3_des
FROM new_form_class
WHERE newformclass_level = '3'
)
SELECT
nfc_level_1.nfc1_id AS NFC1_ID,
nfc_level_1.nfc1_code AS NFC1_CODE,
nfc_level_1.nfc1_des AS NFC1_DES,
nfc_level_2.nfc2_id AS NFC2_ID,
nfc_level_2.nfc2_code AS NFC2_CODE,
nfc_level_2.nfc2_des AS NFC2_DES,
nfc_level_3.nfc3_id AS NFC3_ID,
nfc_level_3.nfc3_code AS NFC3_CODE,
nfc_level_3.nfc3_des AS NFC3_DES
FROM nfc_level_1
LEFT JOIN nfc_level_2
ON nfc_level_1.nfc1_code = LEFT(nfc_level_2.nfc2_code, 1)
LEFT JOIN nfc_level_3
ON nfc_level_2.nfc2_code = LEFT(nfc_level_3.nfc3_code, 2)
;
-- COMMAND ----------
-- Every check must return passed = true before replacing the scheduled jobs.
CREATE OR REPLACE TEMP VIEW validation_hierarchy_compatibility_checks AS
WITH compatibility_checks AS (
SELECT
'atc_row_count' AS check_name,
(SELECT COUNT(*) FROM validation_atc_hierarchy_candidate) AS actual_value,
(SELECT COUNT(*) FROM dwd.dwd_ims_atc_hierarchy) AS expected_value
UNION ALL
SELECT
'atc_legacy_minus_candidate' AS check_name,
(
SELECT COUNT(*)
FROM (
SELECT
ATC1_ID,
ATC1_CODE,
ATC1_DES,
ATC2_ID,
ATC2_CODE,
ATC2_DES,
ATC3_ID,
ATC3_CODE,
ATC3_DES,
ATC4_ID,
ATC4_CODE,
ATC4_DES
FROM dwd.dwd_ims_atc_hierarchy
EXCEPT ALL
SELECT
ATC1_ID,
ATC1_CODE,
ATC1_DES,
ATC2_ID,
ATC2_CODE,
ATC2_DES,
ATC3_ID,
ATC3_CODE,
ATC3_DES,
ATC4_ID,
ATC4_CODE,
ATC4_DES
FROM validation_atc_hierarchy_candidate
) AS differences
) AS actual_value,
0 AS expected_value
UNION ALL
SELECT
'atc_candidate_minus_legacy' AS check_name,
(
SELECT COUNT(*)
FROM (
SELECT
ATC1_ID,
ATC1_CODE,
ATC1_DES,
ATC2_ID,
ATC2_CODE,
ATC2_DES,
ATC3_ID,
ATC3_CODE,
ATC3_DES,
ATC4_ID,
ATC4_CODE,
ATC4_DES
FROM validation_atc_hierarchy_candidate
EXCEPT ALL
SELECT
ATC1_ID,
ATC1_CODE,
ATC1_DES,
ATC2_ID,
ATC2_CODE,
ATC2_DES,
ATC3_ID,
ATC3_CODE,
ATC3_DES,
ATC4_ID,
ATC4_CODE,
ATC4_DES
FROM dwd.dwd_ims_atc_hierarchy
) AS differences
) AS actual_value,
0 AS expected_value
UNION ALL
SELECT
'nfc_row_count' AS check_name,
(SELECT COUNT(*) FROM validation_nfc_hierarchy_candidate) AS actual_value,
(SELECT COUNT(*) FROM dwd.dwd_ims_nfc_hierarchy) AS expected_value
UNION ALL
SELECT
'nfc_legacy_minus_candidate' AS check_name,
(
SELECT COUNT(*)
FROM (
SELECT
NFC1_ID,
NFC1_CODE,
NFC1_DES,
NFC2_ID,
NFC2_CODE,
NFC2_DES,
NFC3_ID,
NFC3_CODE,
NFC3_DES
FROM dwd.dwd_ims_nfc_hierarchy
EXCEPT ALL
SELECT
NFC1_ID,
NFC1_CODE,
NFC1_DES,
NFC2_ID,
NFC2_CODE,
NFC2_DES,
NFC3_ID,
NFC3_CODE,
NFC3_DES
FROM validation_nfc_hierarchy_candidate
) AS differences
) AS actual_value,
0 AS expected_value
UNION ALL
SELECT
'nfc_candidate_minus_legacy' AS check_name,
(
SELECT COUNT(*)
FROM (
SELECT
NFC1_ID,
NFC1_CODE,
NFC1_DES,
NFC2_ID,
NFC2_CODE,
NFC2_DES,
NFC3_ID,
NFC3_CODE,
NFC3_DES
FROM validation_nfc_hierarchy_candidate
EXCEPT ALL
SELECT
NFC1_ID,
NFC1_CODE,
NFC1_DES,
NFC2_ID,
NFC2_CODE,
NFC2_DES,
NFC3_ID,
NFC3_CODE,
NFC3_DES
FROM dwd.dwd_ims_nfc_hierarchy
) AS differences
) AS actual_value,
0 AS expected_value
UNION ALL
SELECT
'except_all_null_semantics' AS check_name,
(
SELECT COUNT(*)
FROM (
SELECT CAST(NULL AS STRING) AS nullable_value
EXCEPT ALL
SELECT CAST(NULL AS STRING) AS nullable_value
) AS differences
) AS actual_value,
0 AS expected_value
)
SELECT
check_name,
actual_value,
expected_value,
actual_value = expected_value AS passed
FROM compatibility_checks
;
-- COMMAND ----------
SELECT
check_name,
actual_value,
expected_value,
passed
FROM validation_hierarchy_compatibility_checks
ORDER BY check_name;
-- COMMAND ----------
-- Profile duplicates and unmatched levels. These are review metrics, not automatic failures.
WITH atc_duplicate_paths AS (
SELECT COUNT(*) AS path_count
FROM validation_atc_hierarchy_candidate
GROUP BY ATC1_CODE, ATC2_CODE, ATC3_CODE, ATC4_CODE
HAVING COUNT(*) > 1
),
nfc_duplicate_paths AS (
SELECT COUNT(*) AS path_count
FROM validation_nfc_hierarchy_candidate
GROUP BY NFC1_CODE, NFC2_CODE, NFC3_CODE
HAVING COUNT(*) > 1
)
SELECT
'atc_duplicate_path_rows' AS metric_name,
COALESCE(SUM(path_count - 1), 0) AS metric_value
FROM atc_duplicate_paths
UNION ALL
SELECT
'atc_source_level_1_required_key_null' AS metric_name,
COUNT_IF(therapeutic_id IS NULL OR therapeutic_code IS NULL) AS metric_value
FROM dwd.dwd_ims_td_therapeutic_class
WHERE therapeutic_level = '1'
UNION ALL
SELECT
'atc_unmatched_level_2' AS metric_name,
COUNT_IF(ATC2_CODE IS NULL) AS metric_value
FROM validation_atc_hierarchy_candidate
UNION ALL
SELECT
'atc_unmatched_level_3' AS metric_name,
COUNT_IF(ATC3_CODE IS NULL) AS metric_value
FROM validation_atc_hierarchy_candidate
UNION ALL
SELECT
'atc_unmatched_level_4' AS metric_name,
COUNT_IF(ATC4_CODE IS NULL) AS metric_value
FROM validation_atc_hierarchy_candidate
UNION ALL
SELECT
'nfc_duplicate_path_rows' AS metric_name,
COALESCE(SUM(path_count - 1), 0) AS metric_value
FROM nfc_duplicate_paths
UNION ALL
SELECT
'nfc_source_level_1_required_key_null' AS metric_name,
COUNT_IF(newformclass_id IS NULL OR newformclass_code IS NULL) AS metric_value
FROM dwd.dwd_ims_td_new_form_class
WHERE newformclass_level = '1'
UNION ALL
SELECT
'nfc_unmatched_level_2' AS metric_name,
COUNT_IF(NFC2_CODE IS NULL) AS metric_value
FROM validation_nfc_hierarchy_candidate
UNION ALL
SELECT
'nfc_unmatched_level_3' AS metric_name,
COUNT_IF(NFC3_CODE IS NULL) AS metric_value
FROM validation_nfc_hierarchy_candidate
ORDER BY metric_name;
-- COMMAND ----------
-- Abort the notebook when any compatibility check fails.
SELECT IF(
COUNT_IF(NOT passed) = 0,
'All hierarchy compatibility checks passed.',
raise_error(
CONCAT(
'Hierarchy compatibility validation failed: ',
CAST(COUNT_IF(NOT passed) AS STRING),
' check(s) did not pass.'
)
)
) AS validation_result
FROM validation_hierarchy_compatibility_checks;
validation/chpa/02_dws/validate_hierarchy_refactor.sql
+250
View File
@@ -0,0 +1,250 @@
-- Databricks notebook source
-- MAGIC %md
-- MAGIC # ATC/NFC DWS hierarchy validation
-- MAGIC Run this notebook after both DWS hierarchy scripts finish.
-- MAGIC The DWS outputs and legacy DWD baselines must represent the same source snapshot.
-- COMMAND ----------
DESCRIBE TABLE dws.dws_ext_td_ims_atc_hierarchy;
-- COMMAND ----------
DESCRIBE TABLE dws.dws_ext_td_ims_nfc_hierarchy;
-- COMMAND ----------
-- Fix the comparison column set and order at the view boundary.
CREATE OR REPLACE TEMP VIEW validation_atc_legacy AS
SELECT
ATC1_ID,
ATC1_CODE,
ATC1_DES,
ATC2_ID,
ATC2_CODE,
ATC2_DES,
ATC3_ID,
ATC3_CODE,
ATC3_DES,
ATC4_ID,
ATC4_CODE,
ATC4_DES
FROM dwd.dwd_ims_atc_hierarchy;
-- COMMAND ----------
CREATE OR REPLACE TEMP VIEW validation_atc_dws AS
SELECT
ATC1_ID,
ATC1_CODE,
ATC1_DES,
ATC2_ID,
ATC2_CODE,
ATC2_DES,
ATC3_ID,
ATC3_CODE,
ATC3_DES,
ATC4_ID,
ATC4_CODE,
ATC4_DES
FROM dws.dws_ext_td_ims_atc_hierarchy;
-- COMMAND ----------
CREATE OR REPLACE TEMP VIEW validation_nfc_legacy AS
SELECT
NFC1_ID,
NFC1_CODE,
NFC1_DES,
NFC2_ID,
NFC2_CODE,
NFC2_DES,
NFC3_ID,
NFC3_CODE,
NFC3_DES
FROM dwd.dwd_ims_nfc_hierarchy;
-- COMMAND ----------
CREATE OR REPLACE TEMP VIEW validation_nfc_dws AS
SELECT
NFC1_ID,
NFC1_CODE,
NFC1_DES,
NFC2_ID,
NFC2_CODE,
NFC2_DES,
NFC3_ID,
NFC3_CODE,
NFC3_DES
FROM dws.dws_ext_td_ims_nfc_hierarchy;
-- COMMAND ----------
CREATE OR REPLACE TEMP VIEW validation_hierarchy_compatibility_checks AS
WITH compatibility_checks AS (
SELECT
'atc_row_count' AS check_name,
(SELECT COUNT(*) FROM validation_atc_dws) AS actual_value,
(SELECT COUNT(*) FROM validation_atc_legacy) AS expected_value
UNION ALL
SELECT
'atc_legacy_minus_dws' AS check_name,
(
SELECT COUNT(*)
FROM (
SELECT * FROM validation_atc_legacy
EXCEPT ALL
SELECT * FROM validation_atc_dws
) AS differences
) AS actual_value,
0 AS expected_value
UNION ALL
SELECT
'atc_dws_minus_legacy' AS check_name,
(
SELECT COUNT(*)
FROM (
SELECT * FROM validation_atc_dws
EXCEPT ALL
SELECT * FROM validation_atc_legacy
) AS differences
) AS actual_value,
0 AS expected_value
UNION ALL
SELECT
'nfc_row_count' AS check_name,
(SELECT COUNT(*) FROM validation_nfc_dws) AS actual_value,
(SELECT COUNT(*) FROM validation_nfc_legacy) AS expected_value
UNION ALL
SELECT
'nfc_legacy_minus_dws' AS check_name,
(
SELECT COUNT(*)
FROM (
SELECT * FROM validation_nfc_legacy
EXCEPT ALL
SELECT * FROM validation_nfc_dws
) AS differences
) AS actual_value,
0 AS expected_value
UNION ALL
SELECT
'nfc_dws_minus_legacy' AS check_name,
(
SELECT COUNT(*)
FROM (
SELECT * FROM validation_nfc_dws
EXCEPT ALL
SELECT * FROM validation_nfc_legacy
) AS differences
) AS actual_value,
0 AS expected_value
UNION ALL
SELECT
'except_all_null_semantics' AS check_name,
(
SELECT COUNT(*)
FROM (
SELECT CAST(NULL AS STRING) AS nullable_value
EXCEPT ALL
SELECT CAST(NULL AS STRING) AS nullable_value
) AS differences
) AS actual_value,
0 AS expected_value
)
SELECT
check_name,
actual_value,
expected_value,
actual_value = expected_value AS passed
FROM compatibility_checks;
-- COMMAND ----------
SELECT
check_name,
actual_value,
expected_value,
passed
FROM validation_hierarchy_compatibility_checks
ORDER BY check_name;
-- COMMAND ----------
-- These quality metrics require business review; they do not fail automatically.
WITH atc_duplicate_paths AS (
SELECT COUNT(*) AS path_count
FROM validation_atc_dws
GROUP BY ATC1_CODE, ATC2_CODE, ATC3_CODE, ATC4_CODE
HAVING COUNT(*) > 1
),
nfc_duplicate_paths AS (
SELECT COUNT(*) AS path_count
FROM validation_nfc_dws
GROUP BY NFC1_CODE, NFC2_CODE, NFC3_CODE
HAVING COUNT(*) > 1
)
SELECT
'atc_duplicate_path_rows' AS metric_name,
COALESCE(SUM(path_count - 1), 0) AS metric_value
FROM atc_duplicate_paths
UNION ALL
SELECT
'atc_source_level_1_required_key_null' AS metric_name,
COUNT_IF(therapeutic_id IS NULL OR therapeutic_code IS NULL) AS metric_value
FROM dwd.dwd_ims_td_therapeutic_class
WHERE therapeutic_level = '1'
UNION ALL
SELECT
'atc_unmatched_level_2' AS metric_name,
COUNT_IF(ATC2_CODE IS NULL) AS metric_value
FROM validation_atc_dws
UNION ALL
SELECT
'atc_unmatched_level_3' AS metric_name,
COUNT_IF(ATC3_CODE IS NULL) AS metric_value
FROM validation_atc_dws
UNION ALL
SELECT
'atc_unmatched_level_4' AS metric_name,
COUNT_IF(ATC4_CODE IS NULL) AS metric_value
FROM validation_atc_dws
UNION ALL
SELECT
'nfc_duplicate_path_rows' AS metric_name,
COALESCE(SUM(path_count - 1), 0) AS metric_value
FROM nfc_duplicate_paths
UNION ALL
SELECT
'nfc_source_level_1_required_key_null' AS metric_name,
COUNT_IF(newformclass_id IS NULL OR newformclass_code IS NULL) AS metric_value
FROM dwd.dwd_ims_td_new_form_class
WHERE newformclass_level = '1'
UNION ALL
SELECT
'nfc_unmatched_level_2' AS metric_name,
COUNT_IF(NFC2_CODE IS NULL) AS metric_value
FROM validation_nfc_dws
UNION ALL
SELECT
'nfc_unmatched_level_3' AS metric_name,
COUNT_IF(NFC3_CODE IS NULL) AS metric_value
FROM validation_nfc_dws
ORDER BY metric_name;
-- COMMAND ----------
-- Block downstream migration when any DWS-to-DWD compatibility check fails.
SELECT IF(
COUNT_IF(NOT passed) = 0,
'All DWS hierarchy compatibility checks passed.',
raise_error(
CONCAT(
'DWS hierarchy compatibility validation failed: ',
CAST(COUNT_IF(NOT passed) AS STRING),
' check(s) did not pass.'
)
)
) AS validation_result
FROM validation_hierarchy_compatibility_checks;