397 lines
11 KiB
SQL
397 lines
11 KiB
SQL
-- Databricks notebook source
|
|
-- MAGIC %md
|
|
-- MAGIC # ATC/NFC hierarchy refactor validation
|
|
-- MAGIC Run this notebook before executing the refactored overwrite scripts.
|
|
-- MAGIC The legacy targets and source tables must represent the same data snapshot.
|
|
|
|
-- COMMAND ----------
|
|
|
|
-- Confirm the ATC target contains the expected 12 non-partition output columns.
|
|
DESCRIBE TABLE dwd.dwd_ims_atc_hierarchy;
|
|
|
|
-- COMMAND ----------
|
|
|
|
-- Confirm the NFC target contains the expected 9 non-partition output columns.
|
|
DESCRIBE TABLE dwd.dwd_ims_nfc_hierarchy;
|
|
|
|
-- COMMAND ----------
|
|
|
|
CREATE OR REPLACE TEMP VIEW validation_atc_hierarchy_candidate AS
|
|
WITH therapeutic_class AS (
|
|
SELECT
|
|
therapeutic_id,
|
|
therapeutic_code,
|
|
therapeutic_name,
|
|
therapeutic_level
|
|
FROM dwd.dwd_ims_td_therapeutic_class
|
|
),
|
|
atc_level_1 AS (
|
|
SELECT
|
|
therapeutic_id AS atc1_id,
|
|
therapeutic_code AS atc1_code,
|
|
therapeutic_name AS atc1_des
|
|
FROM therapeutic_class
|
|
WHERE therapeutic_level = '1'
|
|
),
|
|
atc_level_2 AS (
|
|
SELECT
|
|
therapeutic_id AS atc2_id,
|
|
therapeutic_code AS atc2_code,
|
|
therapeutic_name AS atc2_des
|
|
FROM therapeutic_class
|
|
WHERE therapeutic_level = '2'
|
|
),
|
|
atc_level_3 AS (
|
|
SELECT
|
|
therapeutic_id AS atc3_id,
|
|
therapeutic_code AS atc3_code,
|
|
therapeutic_name AS atc3_des
|
|
FROM therapeutic_class
|
|
WHERE therapeutic_level = '3'
|
|
),
|
|
atc_level_4 AS (
|
|
SELECT
|
|
therapeutic_id AS atc4_id,
|
|
therapeutic_code AS atc4_code,
|
|
therapeutic_name AS atc4_des
|
|
FROM therapeutic_class
|
|
WHERE therapeutic_level = '4'
|
|
)
|
|
SELECT
|
|
atc_level_1.atc1_id AS ATC1_ID,
|
|
atc_level_1.atc1_code AS ATC1_CODE,
|
|
atc_level_1.atc1_des AS ATC1_DES,
|
|
atc_level_2.atc2_id AS ATC2_ID,
|
|
atc_level_2.atc2_code AS ATC2_CODE,
|
|
atc_level_2.atc2_des AS ATC2_DES,
|
|
atc_level_3.atc3_id AS ATC3_ID,
|
|
atc_level_3.atc3_code AS ATC3_CODE,
|
|
atc_level_3.atc3_des AS ATC3_DES,
|
|
atc_level_4.atc4_id AS ATC4_ID,
|
|
atc_level_4.atc4_code AS ATC4_CODE,
|
|
atc_level_4.atc4_des AS ATC4_DES
|
|
FROM atc_level_1
|
|
LEFT JOIN atc_level_2
|
|
ON atc_level_1.atc1_code = LEFT(atc_level_2.atc2_code, 1)
|
|
LEFT JOIN atc_level_3
|
|
ON atc_level_2.atc2_code = LEFT(atc_level_3.atc3_code, 3)
|
|
LEFT JOIN atc_level_4
|
|
ON atc_level_3.atc3_code = LEFT(atc_level_4.atc4_code, 4)
|
|
;
|
|
|
|
-- COMMAND ----------
|
|
|
|
CREATE OR REPLACE TEMP VIEW validation_nfc_hierarchy_candidate AS
|
|
WITH new_form_class AS (
|
|
SELECT
|
|
newformclass_id,
|
|
newformclass_code,
|
|
newformclass_name,
|
|
newformclass_level
|
|
FROM dwd.dwd_ims_td_new_form_class
|
|
),
|
|
nfc_level_1 AS (
|
|
SELECT
|
|
newformclass_id AS nfc1_id,
|
|
newformclass_code AS nfc1_code,
|
|
newformclass_name AS nfc1_des
|
|
FROM new_form_class
|
|
WHERE newformclass_level = '1'
|
|
),
|
|
nfc_level_2 AS (
|
|
SELECT
|
|
newformclass_id AS nfc2_id,
|
|
newformclass_code AS nfc2_code,
|
|
newformclass_name AS nfc2_des
|
|
FROM new_form_class
|
|
WHERE newformclass_level = '2'
|
|
),
|
|
nfc_level_3 AS (
|
|
SELECT
|
|
newformclass_id AS nfc3_id,
|
|
newformclass_code AS nfc3_code,
|
|
newformclass_name AS nfc3_des
|
|
FROM new_form_class
|
|
WHERE newformclass_level = '3'
|
|
)
|
|
SELECT
|
|
nfc_level_1.nfc1_id AS NFC1_ID,
|
|
nfc_level_1.nfc1_code AS NFC1_CODE,
|
|
nfc_level_1.nfc1_des AS NFC1_DES,
|
|
nfc_level_2.nfc2_id AS NFC2_ID,
|
|
nfc_level_2.nfc2_code AS NFC2_CODE,
|
|
nfc_level_2.nfc2_des AS NFC2_DES,
|
|
nfc_level_3.nfc3_id AS NFC3_ID,
|
|
nfc_level_3.nfc3_code AS NFC3_CODE,
|
|
nfc_level_3.nfc3_des AS NFC3_DES
|
|
FROM nfc_level_1
|
|
LEFT JOIN nfc_level_2
|
|
ON nfc_level_1.nfc1_code = LEFT(nfc_level_2.nfc2_code, 1)
|
|
LEFT JOIN nfc_level_3
|
|
ON nfc_level_2.nfc2_code = LEFT(nfc_level_3.nfc3_code, 2)
|
|
;
|
|
|
|
-- COMMAND ----------
|
|
|
|
-- Every check must return passed = true before replacing the scheduled jobs.
|
|
CREATE OR REPLACE TEMP VIEW validation_hierarchy_compatibility_checks AS
|
|
WITH compatibility_checks AS (
|
|
SELECT
|
|
'atc_row_count' AS check_name,
|
|
(SELECT COUNT(*) FROM validation_atc_hierarchy_candidate) AS actual_value,
|
|
(SELECT COUNT(*) FROM dwd.dwd_ims_atc_hierarchy) AS expected_value
|
|
UNION ALL
|
|
SELECT
|
|
'atc_legacy_minus_candidate' AS check_name,
|
|
(
|
|
SELECT COUNT(*)
|
|
FROM (
|
|
SELECT
|
|
ATC1_ID,
|
|
ATC1_CODE,
|
|
ATC1_DES,
|
|
ATC2_ID,
|
|
ATC2_CODE,
|
|
ATC2_DES,
|
|
ATC3_ID,
|
|
ATC3_CODE,
|
|
ATC3_DES,
|
|
ATC4_ID,
|
|
ATC4_CODE,
|
|
ATC4_DES
|
|
FROM dwd.dwd_ims_atc_hierarchy
|
|
EXCEPT ALL
|
|
SELECT
|
|
ATC1_ID,
|
|
ATC1_CODE,
|
|
ATC1_DES,
|
|
ATC2_ID,
|
|
ATC2_CODE,
|
|
ATC2_DES,
|
|
ATC3_ID,
|
|
ATC3_CODE,
|
|
ATC3_DES,
|
|
ATC4_ID,
|
|
ATC4_CODE,
|
|
ATC4_DES
|
|
FROM validation_atc_hierarchy_candidate
|
|
) AS differences
|
|
) AS actual_value,
|
|
0 AS expected_value
|
|
UNION ALL
|
|
SELECT
|
|
'atc_candidate_minus_legacy' AS check_name,
|
|
(
|
|
SELECT COUNT(*)
|
|
FROM (
|
|
SELECT
|
|
ATC1_ID,
|
|
ATC1_CODE,
|
|
ATC1_DES,
|
|
ATC2_ID,
|
|
ATC2_CODE,
|
|
ATC2_DES,
|
|
ATC3_ID,
|
|
ATC3_CODE,
|
|
ATC3_DES,
|
|
ATC4_ID,
|
|
ATC4_CODE,
|
|
ATC4_DES
|
|
FROM validation_atc_hierarchy_candidate
|
|
EXCEPT ALL
|
|
SELECT
|
|
ATC1_ID,
|
|
ATC1_CODE,
|
|
ATC1_DES,
|
|
ATC2_ID,
|
|
ATC2_CODE,
|
|
ATC2_DES,
|
|
ATC3_ID,
|
|
ATC3_CODE,
|
|
ATC3_DES,
|
|
ATC4_ID,
|
|
ATC4_CODE,
|
|
ATC4_DES
|
|
FROM dwd.dwd_ims_atc_hierarchy
|
|
) AS differences
|
|
) AS actual_value,
|
|
0 AS expected_value
|
|
UNION ALL
|
|
SELECT
|
|
'nfc_row_count' AS check_name,
|
|
(SELECT COUNT(*) FROM validation_nfc_hierarchy_candidate) AS actual_value,
|
|
(SELECT COUNT(*) FROM dwd.dwd_ims_nfc_hierarchy) AS expected_value
|
|
UNION ALL
|
|
SELECT
|
|
'nfc_legacy_minus_candidate' AS check_name,
|
|
(
|
|
SELECT COUNT(*)
|
|
FROM (
|
|
SELECT
|
|
NFC1_ID,
|
|
NFC1_CODE,
|
|
NFC1_DES,
|
|
NFC2_ID,
|
|
NFC2_CODE,
|
|
NFC2_DES,
|
|
NFC3_ID,
|
|
NFC3_CODE,
|
|
NFC3_DES
|
|
FROM dwd.dwd_ims_nfc_hierarchy
|
|
EXCEPT ALL
|
|
SELECT
|
|
NFC1_ID,
|
|
NFC1_CODE,
|
|
NFC1_DES,
|
|
NFC2_ID,
|
|
NFC2_CODE,
|
|
NFC2_DES,
|
|
NFC3_ID,
|
|
NFC3_CODE,
|
|
NFC3_DES
|
|
FROM validation_nfc_hierarchy_candidate
|
|
) AS differences
|
|
) AS actual_value,
|
|
0 AS expected_value
|
|
UNION ALL
|
|
SELECT
|
|
'nfc_candidate_minus_legacy' AS check_name,
|
|
(
|
|
SELECT COUNT(*)
|
|
FROM (
|
|
SELECT
|
|
NFC1_ID,
|
|
NFC1_CODE,
|
|
NFC1_DES,
|
|
NFC2_ID,
|
|
NFC2_CODE,
|
|
NFC2_DES,
|
|
NFC3_ID,
|
|
NFC3_CODE,
|
|
NFC3_DES
|
|
FROM validation_nfc_hierarchy_candidate
|
|
EXCEPT ALL
|
|
SELECT
|
|
NFC1_ID,
|
|
NFC1_CODE,
|
|
NFC1_DES,
|
|
NFC2_ID,
|
|
NFC2_CODE,
|
|
NFC2_DES,
|
|
NFC3_ID,
|
|
NFC3_CODE,
|
|
NFC3_DES
|
|
FROM dwd.dwd_ims_nfc_hierarchy
|
|
) AS differences
|
|
) AS actual_value,
|
|
0 AS expected_value
|
|
UNION ALL
|
|
SELECT
|
|
'except_all_null_semantics' AS check_name,
|
|
(
|
|
SELECT COUNT(*)
|
|
FROM (
|
|
SELECT CAST(NULL AS STRING) AS nullable_value
|
|
EXCEPT ALL
|
|
SELECT CAST(NULL AS STRING) AS nullable_value
|
|
) AS differences
|
|
) AS actual_value,
|
|
0 AS expected_value
|
|
)
|
|
SELECT
|
|
check_name,
|
|
actual_value,
|
|
expected_value,
|
|
actual_value = expected_value AS passed
|
|
FROM compatibility_checks
|
|
;
|
|
|
|
-- COMMAND ----------
|
|
|
|
SELECT
|
|
check_name,
|
|
actual_value,
|
|
expected_value,
|
|
passed
|
|
FROM validation_hierarchy_compatibility_checks
|
|
ORDER BY check_name;
|
|
|
|
-- COMMAND ----------
|
|
|
|
-- Profile duplicates and unmatched levels. These are review metrics, not automatic failures.
|
|
WITH atc_duplicate_paths AS (
|
|
SELECT COUNT(*) AS path_count
|
|
FROM validation_atc_hierarchy_candidate
|
|
GROUP BY ATC1_CODE, ATC2_CODE, ATC3_CODE, ATC4_CODE
|
|
HAVING COUNT(*) > 1
|
|
),
|
|
nfc_duplicate_paths AS (
|
|
SELECT COUNT(*) AS path_count
|
|
FROM validation_nfc_hierarchy_candidate
|
|
GROUP BY NFC1_CODE, NFC2_CODE, NFC3_CODE
|
|
HAVING COUNT(*) > 1
|
|
)
|
|
SELECT
|
|
'atc_duplicate_path_rows' AS metric_name,
|
|
COALESCE(SUM(path_count - 1), 0) AS metric_value
|
|
FROM atc_duplicate_paths
|
|
UNION ALL
|
|
SELECT
|
|
'atc_source_level_1_required_key_null' AS metric_name,
|
|
COUNT_IF(therapeutic_id IS NULL OR therapeutic_code IS NULL) AS metric_value
|
|
FROM dwd.dwd_ims_td_therapeutic_class
|
|
WHERE therapeutic_level = '1'
|
|
UNION ALL
|
|
SELECT
|
|
'atc_unmatched_level_2' AS metric_name,
|
|
COUNT_IF(ATC2_CODE IS NULL) AS metric_value
|
|
FROM validation_atc_hierarchy_candidate
|
|
UNION ALL
|
|
SELECT
|
|
'atc_unmatched_level_3' AS metric_name,
|
|
COUNT_IF(ATC3_CODE IS NULL) AS metric_value
|
|
FROM validation_atc_hierarchy_candidate
|
|
UNION ALL
|
|
SELECT
|
|
'atc_unmatched_level_4' AS metric_name,
|
|
COUNT_IF(ATC4_CODE IS NULL) AS metric_value
|
|
FROM validation_atc_hierarchy_candidate
|
|
UNION ALL
|
|
SELECT
|
|
'nfc_duplicate_path_rows' AS metric_name,
|
|
COALESCE(SUM(path_count - 1), 0) AS metric_value
|
|
FROM nfc_duplicate_paths
|
|
UNION ALL
|
|
SELECT
|
|
'nfc_source_level_1_required_key_null' AS metric_name,
|
|
COUNT_IF(newformclass_id IS NULL OR newformclass_code IS NULL) AS metric_value
|
|
FROM dwd.dwd_ims_td_new_form_class
|
|
WHERE newformclass_level = '1'
|
|
UNION ALL
|
|
SELECT
|
|
'nfc_unmatched_level_2' AS metric_name,
|
|
COUNT_IF(NFC2_CODE IS NULL) AS metric_value
|
|
FROM validation_nfc_hierarchy_candidate
|
|
UNION ALL
|
|
SELECT
|
|
'nfc_unmatched_level_3' AS metric_name,
|
|
COUNT_IF(NFC3_CODE IS NULL) AS metric_value
|
|
FROM validation_nfc_hierarchy_candidate
|
|
ORDER BY metric_name;
|
|
|
|
-- COMMAND ----------
|
|
|
|
-- Abort the notebook when any compatibility check fails.
|
|
SELECT IF(
|
|
COUNT_IF(NOT passed) = 0,
|
|
'All hierarchy compatibility checks passed.',
|
|
raise_error(
|
|
CONCAT(
|
|
'Hierarchy compatibility validation failed: ',
|
|
CAST(COUNT_IF(NOT passed) AS STRING),
|
|
' check(s) did not pass.'
|
|
)
|
|
)
|
|
) AS validation_result
|
|
FROM validation_hierarchy_compatibility_checks;
|