-- Databricks notebook source
-- MAGIC %md
-- MAGIC # ATC/NFC hierarchy refactor validation
-- MAGIC Run this notebook before executing the refactored overwrite scripts.
-- MAGIC The legacy targets and source tables must represent the same data snapshot.

-- COMMAND ----------

-- Confirm the ATC target contains the expected 12 non-partition output columns.
DESCRIBE TABLE dwd.dwd_ims_atc_hierarchy;

-- COMMAND ----------

-- Confirm the NFC target contains the expected 9 non-partition output columns.
DESCRIBE TABLE dwd.dwd_ims_nfc_hierarchy;

-- COMMAND ----------

CREATE OR REPLACE TEMP VIEW validation_atc_hierarchy_candidate AS
WITH therapeutic_class AS (
    SELECT
        therapeutic_id,
        therapeutic_code,
        therapeutic_name,
        therapeutic_level
    FROM dwd.dwd_ims_td_therapeutic_class
),
atc_level_1 AS (
    SELECT
        therapeutic_id AS atc1_id,
        therapeutic_code AS atc1_code,
        therapeutic_name AS atc1_des
    FROM therapeutic_class
    WHERE therapeutic_level = '1'
),
atc_level_2 AS (
    SELECT
        therapeutic_id AS atc2_id,
        therapeutic_code AS atc2_code,
        therapeutic_name AS atc2_des
    FROM therapeutic_class
    WHERE therapeutic_level = '2'
),
atc_level_3 AS (
    SELECT
        therapeutic_id AS atc3_id,
        therapeutic_code AS atc3_code,
        therapeutic_name AS atc3_des
    FROM therapeutic_class
    WHERE therapeutic_level = '3'
),
atc_level_4 AS (
    SELECT
        therapeutic_id AS atc4_id,
        therapeutic_code AS atc4_code,
        therapeutic_name AS atc4_des
    FROM therapeutic_class
    WHERE therapeutic_level = '4'
)
SELECT
    atc_level_1.atc1_id AS ATC1_ID,
    atc_level_1.atc1_code AS ATC1_CODE,
    atc_level_1.atc1_des AS ATC1_DES,
    atc_level_2.atc2_id AS ATC2_ID,
    atc_level_2.atc2_code AS ATC2_CODE,
    atc_level_2.atc2_des AS ATC2_DES,
    atc_level_3.atc3_id AS ATC3_ID,
    atc_level_3.atc3_code AS ATC3_CODE,
    atc_level_3.atc3_des AS ATC3_DES,
    atc_level_4.atc4_id AS ATC4_ID,
    atc_level_4.atc4_code AS ATC4_CODE,
    atc_level_4.atc4_des AS ATC4_DES
FROM atc_level_1
LEFT JOIN atc_level_2
    ON atc_level_1.atc1_code = LEFT(atc_level_2.atc2_code, 1)
LEFT JOIN atc_level_3
    ON atc_level_2.atc2_code = LEFT(atc_level_3.atc3_code, 3)
LEFT JOIN atc_level_4
    ON atc_level_3.atc3_code = LEFT(atc_level_4.atc4_code, 4)
;

-- COMMAND ----------

CREATE OR REPLACE TEMP VIEW validation_nfc_hierarchy_candidate AS
WITH new_form_class AS (
    SELECT
        newformclass_id,
        newformclass_code,
        newformclass_name,
        newformclass_level
    FROM dwd.dwd_ims_td_new_form_class
),
nfc_level_1 AS (
    SELECT
        newformclass_id AS nfc1_id,
        newformclass_code AS nfc1_code,
        newformclass_name AS nfc1_des
    FROM new_form_class
    WHERE newformclass_level = '1'
),
nfc_level_2 AS (
    SELECT
        newformclass_id AS nfc2_id,
        newformclass_code AS nfc2_code,
        newformclass_name AS nfc2_des
    FROM new_form_class
    WHERE newformclass_level = '2'
),
nfc_level_3 AS (
    SELECT
        newformclass_id AS nfc3_id,
        newformclass_code AS nfc3_code,
        newformclass_name AS nfc3_des
    FROM new_form_class
    WHERE newformclass_level = '3'
)
SELECT
    nfc_level_1.nfc1_id AS NFC1_ID,
    nfc_level_1.nfc1_code AS NFC1_CODE,
    nfc_level_1.nfc1_des AS NFC1_DES,
    nfc_level_2.nfc2_id AS NFC2_ID,
    nfc_level_2.nfc2_code AS NFC2_CODE,
    nfc_level_2.nfc2_des AS NFC2_DES,
    nfc_level_3.nfc3_id AS NFC3_ID,
    nfc_level_3.nfc3_code AS NFC3_CODE,
    nfc_level_3.nfc3_des AS NFC3_DES
FROM nfc_level_1
LEFT JOIN nfc_level_2
    ON nfc_level_1.nfc1_code = LEFT(nfc_level_2.nfc2_code, 1)
LEFT JOIN nfc_level_3
    ON nfc_level_2.nfc2_code = LEFT(nfc_level_3.nfc3_code, 2)
;

-- COMMAND ----------

-- Every check must return passed = true before replacing the scheduled jobs.
WITH compatibility_checks AS (
    SELECT
        'atc_row_count' AS check_name,
        (SELECT COUNT(*) FROM validation_atc_hierarchy_candidate) AS actual_value,
        (SELECT COUNT(*) FROM dwd.dwd_ims_atc_hierarchy) AS expected_value
    UNION ALL
    SELECT
        'atc_legacy_minus_candidate' AS check_name,
        (
            SELECT COUNT(*)
            FROM (
                SELECT * FROM dwd.dwd_ims_atc_hierarchy
                EXCEPT ALL
                SELECT * FROM validation_atc_hierarchy_candidate
            ) AS differences
        ) AS actual_value,
        0 AS expected_value
    UNION ALL
    SELECT
        'atc_candidate_minus_legacy' AS check_name,
        (
            SELECT COUNT(*)
            FROM (
                SELECT * FROM validation_atc_hierarchy_candidate
                EXCEPT ALL
                SELECT * FROM dwd.dwd_ims_atc_hierarchy
            ) AS differences
        ) AS actual_value,
        0 AS expected_value
    UNION ALL
    SELECT
        'nfc_row_count' AS check_name,
        (SELECT COUNT(*) FROM validation_nfc_hierarchy_candidate) AS actual_value,
        (SELECT COUNT(*) FROM dwd.dwd_ims_nfc_hierarchy) AS expected_value
    UNION ALL
    SELECT
        'nfc_legacy_minus_candidate' AS check_name,
        (
            SELECT COUNT(*)
            FROM (
                SELECT * FROM dwd.dwd_ims_nfc_hierarchy
                EXCEPT ALL
                SELECT * FROM validation_nfc_hierarchy_candidate
            ) AS differences
        ) AS actual_value,
        0 AS expected_value
    UNION ALL
    SELECT
        'nfc_candidate_minus_legacy' AS check_name,
        (
            SELECT COUNT(*)
            FROM (
                SELECT * FROM validation_nfc_hierarchy_candidate
                EXCEPT ALL
                SELECT * FROM dwd.dwd_ims_nfc_hierarchy
            ) AS differences
        ) AS actual_value,
        0 AS expected_value
)
SELECT
    check_name,
    actual_value,
    expected_value,
    actual_value = expected_value AS passed
FROM compatibility_checks
ORDER BY check_name;

-- COMMAND ----------

-- Profile duplicates and unmatched levels. These are review metrics, not automatic failures.
WITH atc_duplicate_paths AS (
    SELECT COUNT(*) AS path_count
    FROM validation_atc_hierarchy_candidate
    GROUP BY ATC1_CODE, ATC2_CODE, ATC3_CODE, ATC4_CODE
    HAVING COUNT(*) > 1
),
nfc_duplicate_paths AS (
    SELECT COUNT(*) AS path_count
    FROM validation_nfc_hierarchy_candidate
    GROUP BY NFC1_CODE, NFC2_CODE, NFC3_CODE
    HAVING COUNT(*) > 1
)
SELECT
    'atc_duplicate_path_rows' AS metric_name,
    COALESCE(SUM(path_count - 1), 0) AS metric_value
FROM atc_duplicate_paths
UNION ALL
SELECT
    'atc_null_root_code' AS metric_name,
    COUNT_IF(ATC1_CODE IS NULL) AS metric_value
FROM validation_atc_hierarchy_candidate
UNION ALL
SELECT
    'atc_unmatched_level_2' AS metric_name,
    COUNT_IF(ATC2_CODE IS NULL) AS metric_value
FROM validation_atc_hierarchy_candidate
UNION ALL
SELECT
    'atc_unmatched_level_3' AS metric_name,
    COUNT_IF(ATC3_CODE IS NULL) AS metric_value
FROM validation_atc_hierarchy_candidate
UNION ALL
SELECT
    'atc_unmatched_level_4' AS metric_name,
    COUNT_IF(ATC4_CODE IS NULL) AS metric_value
FROM validation_atc_hierarchy_candidate
UNION ALL
SELECT
    'nfc_duplicate_path_rows' AS metric_name,
    COALESCE(SUM(path_count - 1), 0) AS metric_value
FROM nfc_duplicate_paths
UNION ALL
SELECT
    'nfc_null_root_code' AS metric_name,
    COUNT_IF(NFC1_CODE IS NULL) AS metric_value
FROM validation_nfc_hierarchy_candidate
UNION ALL
SELECT
    'nfc_unmatched_level_2' AS metric_name,
    COUNT_IF(NFC2_CODE IS NULL) AS metric_value
FROM validation_nfc_hierarchy_candidate
UNION ALL
SELECT
    'nfc_unmatched_level_3' AS metric_name,
    COUNT_IF(NFC3_CODE IS NULL) AS metric_value
FROM validation_nfc_hierarchy_candidate
ORDER BY metric_name;