-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_UNIONALL_MARKET (
--   MARKET STRING,
--   DATA_SOURCE STRING,
--   TA STRING,
--   AZ_RELATED STRING,
--   MARKET_DESC STRING,
--   TA_RN INT,
--   PRD_FLAG STRING,
--   ETL_INSERT_DT TIMESTAMP,
--   ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market';

-- COMMAND ----------

-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_UNIONALL_MARKET_TA (
--   MARKET STRING COMMENT '市场',
--   DATA_SOURCE STRING COMMENT '数据源',
--   TA STRING COMMENT '治疗领域',
--   TA_MAP STRING COMMENT '治疗领域匹配',
--   TA_RN INT COMMENT '治疗领域排名',
--   MARKET_DESC STRING COMMENT '市场描述',
--   ETL_INSERT_DT TIMESTAMP,
--   ETL_UPDATE_DT TIMESTAMP
--   )
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market_ta';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market_ta';

-- COMMAND ----------

-- DBTITLE 1,pbi页面展示ta标签
-- -- ***********************************************************
-- -- 表名：tmp.tmp_td_ext_unionall_market_ta_mapping
-- -- 功能：市场(Market)与治疗领域(TA)的映射关系表
-- -- 用途：统一各数据源的市场-治疗领域映射标准
-- -- 更新方式：INSERT OVERWRITE（全量覆盖）
-- -- ***********************************************************

-- CREATE OR REPLACE TABLE tmp.tmp_td_ext_unionall_market_ta_mapping
-- (
--     market      STRING COMMENT '市场名称',
--     ta          STRING COMMENT '治疗领域代码',
--     data_source STRING COMMENT '数据源类型及频率'
-- )
-- COMMENT '市场-治疗领域映射表，用于统一跨源数据关联标准';

-- -- 清空并插入映射数据
-- INSERT OVERWRITE TABLE tmp.tmp_td_ext_unionall_market_ta_mapping (market, ta, data_source) VALUES

-- -- ==================== EC(Monthly) 数据源 ====================
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'DM',      'EC(Monthly)'),   -- 非胰岛素抗糖尿病市场 → DM领域
-- ('PPI Oral Market',                         'GI',      'EC(Monthly)'),   -- 胃肠病市场 → GI领域
-- ('RE Market',                               'RE',      'EC(Monthly)'),   -- 呼吸市场 → RE领域
-- ('CVRM Market',                             'CVRM',    'EC(Monthly)'),   -- 心血管肾脏代谢市场 → CVRM领域

-- -- ==================== DTP(Quarterly) 数据源 ====================
-- ('ATTR_EPL',                                'CV',      'DTP(Quarterly)'), -- ATTR淀粉样变性 → CV领域
-- ('ONC Market',                              'ONCO',    'DTP(Quarterly)'), -- 肿瘤市场 → ONCO领域
-- ('RARE Market',                             'RARE',    'DTP(Quarterly)'), -- 罕见病市场 → RARE领域

-- -- ==================== Retail(Quarterly) 数据源 ====================
-- ('RE Market',                               'RE',      'Retail(Quarterly)'),   -- 呼吸市场 → RE领域
-- ('CV Market',                               'CV',      'Retail(Quarterly)'),   -- 心血管市场 → CV领域
-- ('RD Market',                               'RD',      'Retail(Quarterly)'),   -- 肾脏病市场 → RD领域
-- ('PPI Oral Market',                         'GI',      'Retail(Quarterly)'),   -- 胃肠病市场 → GI领域
-- ('CVRM Market',                             'CVRM',    'Retail(Quarterly)'),   -- 心血管肾脏代谢市场 → CVRM领域
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'DM',      'Retail(Quarterly)'),  -- 糖尿病市场 → DM领域

-- -- ==================== THC(Quarterly) 数据源 ====================
-- ('DM Market',                               'DM',      'THC(Quarterly)'),      -- 糖尿病市场 → DM领域
-- ('GI Market',                               'GI',      'THC(Quarterly)'),      -- 胃肠病市场 → GI领域
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD',    'THC(Quarterly)'),      -- 非胰岛素抗糖尿病市场 → NIAD领域
-- ('RE Market',                               'RE',      'THC(Quarterly)'),      -- 呼吸市场 → RE领域
-- ('ONC Market',                              'ONCO',    'THC(Quarterly)'),      -- 肿瘤市场 → ONCO领域
-- ('CV Market',                               'CV',      'THC(Quarterly)'),      -- 心血管市场 → CV领域
-- ('CVRM Market',                             'CVRM',    'THC(Quarterly)'),      -- 心血管肾脏代谢市场 → CVRM领域

-- -- ==================== XH Data(Quarterly) 数据源 ====================
-- ('RE Market',                               'RE',      'XH Data(Quarterly)'),  -- 呼吸市场 → RE领域
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD',    'XH Data(Quarterly)'),  -- 非胰岛素抗糖尿病市场 → NIAD领域
-- ('RD Market',                               'RD',      'XH Data(Quarterly)'),  -- 肾脏病市场 → RD领域
-- ('DM Market',                               'DM',      'XH Data(Quarterly)'),  -- 糖尿病市场 → DM领域
-- ('ONC Market',                              'ONCO',    'XH Data(Quarterly)'),  -- 肿瘤市场 → ONCO领域
-- ('RARE Market',                             'RARE',    'XH Data(Quarterly)'),  -- 罕见病市场 → RARE领域
-- ('CV Market',                               'CV',      'XH Data(Quarterly)'),  -- 心血管市场 → CV领域
-- ('CVRM Market',                             'CVRM',    'XH Data(Quarterly)'),  -- 心血管肾脏代谢市场 → CVRM领域

-- -- ==================== IQVIA-COUNTY(Quarterly) 数据源 ====================
-- ('DM Market',                               'DM',      'IQVIA-COUNTY(Quarterly)'),   -- 糖尿病市场 → DM领域
-- ('ONC Market',                              'ONCO',    'IQVIA-COUNTY(Quarterly)'),   -- 肿瘤市场 → ONCO领域
-- ('GI Market',                               'GI',      'IQVIA-COUNTY(Quarterly)'),   -- 胃肠病市场 → GI领域
-- ('RE Market',                               'RE',      'IQVIA-COUNTY(Quarterly)'),   -- 呼吸市场 → RE领域
-- ('CV Market',                               'CV',      'IQVIA-COUNTY(Quarterly)'),   -- 心血管市场 → CV领域
-- ('RARE Market',                             'RARE',    'IQVIA-COUNTY(Quarterly)'),   -- 罕见病市场 → RARE领域
-- ('RD Market',                               'RD',      'IQVIA-COUNTY(Quarterly)'),   -- 肾脏病市场 → RD领域
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD',    'IQVIA-COUNTY(Quarterly)'),   -- 非胰岛素抗糖尿病市场 → NIAD领域
-- ('CVRM Market',                             'CVRM',    'IQVIA-COUNTY(Quarterly)'),   -- 心血管肾脏代谢市场 → CVRM领域

-- -- ==================== IQVIA-CHPA(Monthly) 数据源 ====================
-- ('ONC Market',                              'ONCO',    'IQVIA-CHPA(Monthly)'),   -- 肿瘤市场 → ONCO领域
-- ('RE Market',                               'RE',      'IQVIA-CHPA(Monthly)'),   -- 呼吸市场 → RE领域
-- ('RD Market',                               'RD',      'IQVIA-CHPA(Monthly)'),   -- 肾脏病市场 → RD领域
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD',    'IQVIA-CHPA(Monthly)'),   -- 非胰岛素抗糖尿病市场 → NIAD领域
-- ('RARE Market',                             'RARE',    'IQVIA-CHPA(Monthly)'),   -- 罕见病市场 → RARE领域
-- ('DM Market',                               'DM',      'IQVIA-CHPA(Monthly)'),   -- 糖尿病市场 → DM领域
-- ('GI Market',                               'GI',      'IQVIA-CHPA(Monthly)'),   -- 胃肠病市场 → GI领域
-- ('CV Market',                               'CV',      'IQVIA-CHPA(Monthly)'),   -- 心血管市场 → CV领域
-- ('CVRM Market',                             'CVRM',    'IQVIA-CHPA(Monthly)'),   -- 心血管肾脏代谢市场 → CVRM领域

-- -- ==================== AIA(Monthly) 数据源 ====================
-- ('DM Market',                               'DM',      'AIA(Monthly)'),      -- 糖尿病市场 → DM领域
-- ('CV Market',                               'CV',      'AIA(Monthly)'),      -- 心血管市场 → CV领域
-- ('GI Market',                               'GI',      'AIA(Monthly)'),      -- 胃肠病市场 → GI领域
-- ('RE Market',                               'RE',      'AIA(Monthly)'),      -- 呼吸市场 → RE领域
-- ('RARE Market',                             'RARE',    'AIA(Monthly)'),      -- 罕见病市场 → RARE领域
-- ('ONC Market',                              'ONCO',    'AIA(Monthly)'),      -- 肿瘤市场 → ONCO领域
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD',    'AIA(Monthly)'),      -- 非胰岛素抗糖尿病市场 → NIAD领域
-- ('RD Market',                               'RD',      'AIA(Monthly)'),      -- 肾脏病市场 → RD领域
-- ('CVRM Market',                             'CVRM',    'AIA(Monthly)'),      -- 心血管肾脏代谢市场 → CVRM领域

-- -- ==================== CHC(Quarterly) 数据源 ====================
-- ('GI Market',                               'GI',      'CHC(Quarterly)'),    -- 胃肠病市场 → GI领域
-- ('DM Market',                               'DM',      'CHC(Quarterly)'),    -- 糖尿病市场 → DM领域
-- ('RE Market',                               'RE',      'CHC(Quarterly)'),    -- 呼吸市场 → RE领域
-- ('CV Market',                               'CV',      'CHC(Quarterly)'),    -- 心血管市场 → CV领域
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD',    'CHC(Quarterly)'),    -- 非胰岛素抗糖尿病市场 → NIAD领域
-- ('CVRM Market',                             'CVRM',    'CHC(Quarterly)')     -- 心血管肾脏代谢市场 → CVRM领域

-- ;


-- COMMAND ----------

-- DBTITLE 1,藏数逻辑标签
-- -- 建表
-- CREATE TABLE IF NOT EXISTS tmp.tmp_td_ext_market_hide_flag (
--     data_source VARCHAR(20),
--     market VARCHAR(100),
--     flag VARCHAR(1)
-- );

-- -- 插入数据（对应 case when 中返回 '1' 的规则）
-- INSERT overwrite tmp.tmp_td_ext_market_hide_flag (data_source, market, flag) VALUES
-- -- Retail(Quarterly) 部分
-- ('Retail(Quarterly)', 'Antacid anti GI swelling agent+PPI Oral', '1'),
-- ('Retail(Quarterly)', 'HTN Market', '1'),
-- ('Retail(Quarterly)', 'NIAD (Non-Insulin Anti-Diabetic) Market', '1'),
-- ('Retail(Quarterly)', 'Respules Market', '1'),
-- ('Retail(Quarterly)', 'Statin+XZK Market', '1'),
-- ('Retail(Quarterly)', 'BUDESONIDE Market', '1'),
-- ('Retail(Quarterly)', 'Rosuvastatin Market', '1'),
-- ('Retail(Quarterly)', 'Atorvastatin Market', '1'),
-- ('Retail(Quarterly)', 'Metoprolol Succinate Market', '1'),
-- ('Retail(Quarterly)', 'PPI Oral ESOMEPRAZOLE Market', '1'),
-- ('Retail(Quarterly)', 'PPI Oral OMEPRAZOLE Market', '1'),
-- ('Retail(Quarterly)', 'Inhaled Extended Market', '1'),
-- ('Retail(Quarterly)', 'Non-Omeprazole', '1'),
-- ('Retail(Quarterly)', 'Metoprolol Tartrate Market', '1'),
-- ('Retail(Quarterly)', 'Pediatric Cough Market', '1'),
-- ('Retail(Quarterly)', 'Anti-HER2 Market', '1'),
-- ('Retail(Quarterly)', 'Anti-HER2 mBC Market', '1'),
-- ('Retail(Quarterly)', 'Fasenra Market', '1'),
-- ('Retail(Quarterly)', 'Severe Asthma', '1'),
-- ('Retail(Quarterly)', 'DAPA Market', '1'),

-- -- DTP(Quarterly) 部分
-- ('DTP(Quarterly)', 'EGFR TKI Market', '1'),
-- ('DTP(Quarterly)', 'Anti-HER2 Market', '1'),
-- ('DTP(Quarterly)', 'Anti-HER2 mBC Market', '1'),
-- ('DTP(Quarterly)', 'Fasenra/Tezspire Related Market', '1'),
-- ('DTP(Quarterly)', 'Severe Asthma', '1'),
-- ('DTP(Quarterly)', 'Tezspire CRSwNP Market', '1'),
-- ('DTP(Quarterly)', 'CRSwNP Market', '1'),
-- ('DTP(Quarterly)', 'PAM Market', '1');

-- COMMAND ----------

-- ============================================================
-- 目标表：dm.dm_td_ext_unionall_market
-- 功能：整合各数据源的市场、TA、AZ关联、产品标志等维度信息
-- 调度：全量覆盖写入（INSERT OVERWRITE）
-- 作者/更新：chenwu
-- ============================================================
INSERT OVERWRITE dm.dm_td_ext_unionall_market (
  MARKET, DATA_SOURCE, TA, AZ_RELATED, MARKET_DESC, TA_RN, PRD_FLAG, ETL_INSERT_DT, ETL_UPDATE_DT
)

WITH

-- ------------------------------------------------------------
-- market_info：从包映射表取市场-数据源组合，关联竞品表补充市场描述
-- 用 MAX(MARKET_DESC) 避免一对多 JOIN 导致重复行
-- ------------------------------------------------------------
market_info AS (
  SELECT
    t1.MARKET,
    t1.DATASOURCE          AS DATA_SOURCE,
    MAX(t2.MARKETDESC)     AS MARKET_DESC
  FROM dm.dm_td_ext_unionall_market_pack_mapping t1
  LEFT JOIN dm.dm_td_external_keycompatitor      t2
    ON t1.MARKET = t2.MARKET
  GROUP BY 1, 2
),

-- ------------------------------------------------------------
-- TA_info：从市场属性表取市场-TA-排序，过滤占位行（MARKET_ORDER=999）
-- 末尾 UNION ALL 补充"ALL Market"汇总行
-- 改用 SELECT DISTINCT 替代无聚合的 GROUP BY
-- ------------------------------------------------------------
TA_info AS (
  SELECT DISTINCT
    MARKET,
    TA,
    TA_ORDER
  FROM DM.DM_IMS_TD_MARKET_PROPERTY
  WHERE MARKET_ORDER <> 999
  UNION ALL

  -- SELECT
  --   'CVRM Market' AS MARKET,
  --   'CVRM'        AS TA,
  --   0           AS TA_ORDER
  -- UNION ALL

  SELECT
    'ALL Market' AS MARKET,
    'ALL'        AS TA,
    -1           AS TA_ORDER
),

-- ------------------------------------------------------------
-- AZ_RELATED_info：从临时映射表取市场-数据源-AZ关联关系（已去重）
-- ------------------------------------------------------------
AZ_RELATED_info AS (
  SELECT DISTINCT
    MARKET,
    TA           AS AZ_RELATED,
    DATA_SOURCE
  FROM tmp.tmp_td_ext_unionall_market_ta_mapping
)

-- ------------------------------------------------------------
-- 主查询：以 market_info 为驱动表，依次关联 TA、AZ关联、隐藏标志
-- hide_flag 为小维表，添加 BROADCAST hint 避免 shuffle
-- PRD_FLAG=1 表示市场在隐藏标志表中存在（需隐藏/特殊处理）
-- ------------------------------------------------------------
SELECT /*+ BROADCAST(t4) */
DISTINCT
  t1.MARKET,
  t1.DATA_SOURCE,
  t2.TA,
  t3.AZ_RELATED,
  t1.MARKET_DESC,
  t2.TA_ORDER                                        AS TA_RN,
  IF(t4.MARKET IS NULL, 0, 1)                        AS PRD_FLAG,
  FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8')   AS ETL_INSERT_DT,
  FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8')   AS ETL_UPDATE_DT
FROM market_info t1
LEFT JOIN TA_info            t2
  ON  t1.MARKET      = t2.MARKET
LEFT JOIN AZ_RELATED_info    t3
  ON  t1.MARKET      = t3.MARKET
  AND t1.DATA_SOURCE = t3.DATA_SOURCE
LEFT JOIN 
     tmp.tmp_td_ext_market_hide_flag t4
  ON  t1.MARKET      = t4.MARKET
  AND t1.DATA_SOURCE = t4.DATA_SOURCE

-- COMMAND ----------

insert overwrite dm.dm_td_ext_unionall_market_ta (
  MARKET, DATA_SOURCE, TA, TA_RN, MARKET_DESC, ETL_INSERT_DT, ETL_UPDATE_DT
)
  SELECT
    MARKET,
    DATA_SOURCE,
    AZ_RELATED TA,
    TA_RN,
    MARKET_DESC,
    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
  FROM
    DM.dm_td_ext_unionall_market
  -- WHERE MARKET NOT LIKE '%ALL Market%'  and MARKET <>'Retail All Market'
  -- UNION all
  -- SELECT
  --   MARKET,
  --   DATA_SOURCE,
  --   'CVRM' TA,
  --   0 TA_RN,
  --   MARKET_DESC,
  --   FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
  --   FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
  -- FROM
  --   DM.dm_td_ext_unionall_market
  -- WHERE
  --   MARKET = 'CVRM Market'

-- COMMAND ----------

-----------------------------------------------------------------------------------
--修改时间：20240822
--修改人：FanXujia
--修改内容：
--Retail、EC渠道，新增NIAD这个TA。数据与DM相同
-----------------------------------------------------------------------------------
insert into dm.dm_td_ext_unionall_market_ta (
  MARKET, DATA_SOURCE, TA, TA_RN, MARKET_DESC, ETL_INSERT_DT, ETL_UPDATE_DT
)
  select
    market,
    DATA_SOURCE,
    'NIAD' as TA,
    7 as TA_RN,
    MARKET_DESC,
    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
  from
    dm.dm_td_ext_unionall_market_ta
  where
    DATA_SOURCE = 'Retail(Quarterly)'
    and ta = 'DM'
  union all
  select
    market,
    DATA_SOURCE,
    'NIAD' as TA,
    7 as TA_RN,
    MARKET_DESC,
    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
  from
    dm.dm_td_ext_unionall_market_ta
  where
    DATA_SOURCE = 'EC(Monthly)'
    and ta = 'DM'
  --20250701 chenwu 新增RARE = ONCO的Soliris Related Market
  union all
  select
    market,
    DATA_SOURCE,
    'RARE' as TA,
    8 as TA_RN,
    MARKET_DESC,
    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
  from
    dm.dm_td_ext_unionall_market_ta
  where
    DATA_SOURCE = 'DTP(Quarterly)'
    and ta = 'ONCO'
    and MARKET = 'Soliris Related Market';

-- COMMAND ----------

-- log 20250226 auth:庄伟
-- 对market_ta表中市场进行补全，并依据手工文件维护的 market 与 ta 映射，匹配出完整市场对应的TA_NAME
insert overwrite table dm.dm_td_ext_unionall_market_ta (
  MARKET, DATA_SOURCE, TA, TA_MAP, TA_RN, MARKET_DESC, ETL_INSERT_DT, ETL_UPDATE_DT
)
  -----找出在market维度表中不存在的market列表
  with temp_ma_ext_market as (select distinct
      MARKET,
      DATA_SOURCE
    from
      dm.dm_td_ext_unionall_market
    where
      MARKET NOT IN (
        select distinct
          MARKET
        from
          dm.dm_td_ext_unionall_market_ta
      )
  )
  ------补充完整的market列表，并通过market匹配手工文件维护的market对应的TA
  select
  distinct
    dim_mkt_ta.MARKET,
    DATA_SOURCE,
    dim_mkt_ta.TA,
    case
      when dim_mkt_ta.MARKET = 'ALL Market' then 'ALL'
      else nvl(mkt_property.ta, mkt_property_key.TA)
    end as TA_MAP,
    dim_mkt_ta.TA_RN,
    dim_mkt_ta.MARKET_DESC,
    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
  from
    (
      select
        MARKET,
        DATA_SOURCE,
        TA,
        TA_RN,
        MARKET_DESC
      from
        dm.dm_td_ext_unionall_market_ta
    ) dim_mkt_ta
      left join dm.dm_ims_td_market_property mkt_property
        on dim_mkt_ta.market = mkt_property.market
      left join dm.dm_ims_td_market_property mkt_property_key
        on dim_mkt_ta.MARKET = mkt_property_key.MARKET_KEY
  union all
  select
    ext_market.MARKET,
    ext_market.DATA_SOURCE,
    null TA,
    mkt_property.TA TA_MAP,
    '99' as TA_RN,
    null MARKET_DESC,
    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
  from
    temp_ma_ext_market ext_market
      left join dm.dm_ims_td_market_property mkt_property
        on ext_market.market = mkt_property.MARKET
  where
    nvl(mkt_property.TA, '') <> '';