update20260427

2026-04-27 11:04:09 +08:00
commit 3e14e78af3
173 changed files with 62579 additions and 0 deletions
--- a/DM_TD_EXTERNAL_MARKET_NEW.sql
+++ b/DM_TD_EXTERNAL_MARKET_NEW.sql
@@ -0,0 +1,393 @@
+-- Databricks notebook source
+-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_UNIONALL_MARKET (
+--   MARKET STRING,
+--   DATA_SOURCE STRING,
+--   TA STRING,
+--   AZ_RELATED STRING,
+--   MARKET_DESC STRING,
+--   TA_RN INT,
+--   PRD_FLAG STRING,
+--   ETL_INSERT_DT TIMESTAMP,
+--   ETL_UPDATE_DT TIMESTAMP)
+-- USING delta
+-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market';
+-- -- 上面是生产环境location，下面是测试环境location
+-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market';
+
+-- COMMAND ----------
+
+-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_UNIONALL_MARKET_TA (
+--   MARKET STRING COMMENT '市场',
+--   DATA_SOURCE STRING COMMENT '数据源',
+--   TA STRING COMMENT '治疗领域',
+--   TA_MAP STRING COMMENT '治疗领域匹配',
+--   TA_RN INT COMMENT '治疗领域排名',
+--   MARKET_DESC STRING COMMENT '市场描述',
+--   ETL_INSERT_DT TIMESTAMP,
+--   ETL_UPDATE_DT TIMESTAMP
+--   )
+-- USING delta
+-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market_ta';
+-- -- 上面是生产环境location，下面是测试环境location
+-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market_ta';
+
+-- COMMAND ----------
+
+-- DBTITLE 1,pbi页面展示ta标签
+-- create or replace table tmp.tmp_td_ext_unionall_market_ta_mapping
+-- (
+-- market string,
+-- ta string,
+-- data_source string);
+-- INSERT OVERWRITE tmp.tmp_td_ext_unionall_market_ta_mapping (market, ta, data_source) VALUES
+-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'DM', 'EC(Monthly)'),
+-- ('GI Market', 'GI', 'EC(Monthly)'),
+-- ('RE Market', 'RE', 'EC(Monthly)'),
+-- ('ATTR_EPL', 'CV', 'DTP(Quarterly)'),
+-- ('ONC Market', 'ONCO', 'DTP(Quarterly)'),
+-- ('RARE Market', 'RARE', 'DTP(Quarterly)'),
+-- ('RE Market', 'RE', 'Retail(Quarterly)'),
+-- ('CV Market', 'CV', 'Retail(Quarterly)'),
+-- ('RD Market', 'RD', 'Retail(Quarterly)'),
+-- ('GI Market', 'GI', 'Retail(Quarterly)'),
+-- ('DM Market', 'DM', 'THC(Quarterly)'),
+-- ('GI Market', 'GI', 'THC(Quarterly)'),
+-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'THC(Quarterly)'),
+-- ('RE Market', 'RE', 'THC(Quarterly)'),
+-- ('ONC Market', 'ONCO', 'THC(Quarterly)'),
+-- ('CV Market', 'CV', 'THC(Quarterly)'),
+-- ('RE Market', 'RE', 'XH Data(Quarterly)'),
+-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'XH Data(Quarterly)'),
+-- ('RD Market', 'RD', 'XH Data(Quarterly)'),
+-- ('DM Market', 'DM', 'XH Data(Quarterly)'),
+-- ('ONC Market', 'ONCO', 'XH Data(Quarterly)'),
+-- ('RARE Market', 'RARE', 'XH Data(Quarterly)'),
+-- ('CV Market', 'CV', 'XH Data(Quarterly)'),
+-- ('DM Market', 'DM', 'IQVIA-COUNTY(Quarterly)'),
+-- ('ONC Market', 'ONCO', 'IQVIA-COUNTY(Quarterly)'),
+-- ('GI Market', 'GI', 'IQVIA-COUNTY(Quarterly)'),
+-- ('RE Market', 'RE', 'IQVIA-COUNTY(Quarterly)'),
+-- ('CV Market', 'CV', 'IQVIA-COUNTY(Quarterly)'),
+-- ('RARE Market', 'RARE', 'IQVIA-COUNTY(Quarterly)'),
+-- ('RD Market', 'RD', 'IQVIA-COUNTY(Quarterly)'),
+-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'IQVIA-COUNTY(Quarterly)'),
+-- ('ONC Market', 'ONCO', 'IQVIA-CHPA(Monthly)'),
+-- ('RE Market', 'RE', 'IQVIA-CHPA(Monthly)'),
+-- ('RD Market', 'RD', 'IQVIA-CHPA(Monthly)'),
+-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'IQVIA-CHPA(Monthly)'),
+-- ('RARE Market', 'RARE', 'IQVIA-CHPA(Monthly)'),
+-- ('DM Market', 'DM', 'IQVIA-CHPA(Monthly)'),
+-- ('GI Market', 'GI', 'IQVIA-CHPA(Monthly)'),
+-- ('CV Market', 'CV', 'IQVIA-CHPA(Monthly)'),
+-- ('DM Market', 'DM', 'AIA(Monthly)'),
+-- ('CV Market', 'CV', 'AIA(Monthly)'),
+-- ('GI Market', 'GI', 'AIA(Monthly)'),
+-- ('RE Market', 'RE', 'AIA(Monthly)'),
+-- ('RARE Market', 'RARE', 'AIA(Monthly)'),
+-- ('ONC Market', 'ONCO', 'AIA(Monthly)'),
+-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'AIA(Monthly)'),
+-- ('RD Market', 'RD', 'AIA(Monthly)'),
+-- ('GI Market', 'GI', 'CHC(Quarterly)'),
+-- ('DM Market', 'DM', 'CHC(Quarterly)'),
+-- ('RE Market', 'RE', 'CHC(Quarterly)'),
+-- ('CV Market', 'CV', 'CHC(Quarterly)'),
+-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'CHC(Quarterly)'),
+
+-- ('CVRM Market', 'CVRM', 'Retail(Quarterly)'),
+-- ('CVRM Market', 'CVRM', 'EC(Monthly)'),
+-- ('CVRM Market', 'CVRM', 'CHC(Quarterly)'),
+-- ('CVRM Market', 'CVRM', 'AIA(Monthly)'),
+-- ('CVRM Market', 'CVRM', 'IQVIA-COUNTY(Quarterly)'),
+-- ('CVRM Market', 'CVRM', 'XH Data(Quarterly)'),
+-- ('CVRM Market', 'CVRM', 'IQVIA-CHPA(Monthly)'),
+-- ('CVRM Market', 'CVRM', 'THC(Quarterly)')
+-- ;
+
+-- COMMAND ----------
+
+-- DBTITLE 1,藏数逻辑标签
+-- -- 建表
+-- CREATE TABLE IF NOT EXISTS tmp.tmp_td_ext_market_hide_flag (
+--     data_source VARCHAR(20),
+--     market VARCHAR(100),
+--     flag VARCHAR(1)
+-- );
+
+-- -- 插入数据（对应 case when 中返回 '1' 的规则）
+-- INSERT overwrite tmp.tmp_td_ext_market_hide_flag (data_source, market, flag) VALUES
+-- -- Retail(Quarterly) 部分
+-- ('Retail(Quarterly)', 'Antacid anti GI swelling agent+PPI Oral', '1'),
+-- ('Retail(Quarterly)', 'HTN Market', '1'),
+-- ('Retail(Quarterly)', 'NIAD (Non-Insulin Anti-Diabetic) Market', '1'),
+-- ('Retail(Quarterly)', 'Respules Market', '1'),
+-- ('Retail(Quarterly)', 'Statin+XZK Market', '1'),
+-- ('Retail(Quarterly)', 'BUDESONIDE Market', '1'),
+-- ('Retail(Quarterly)', 'Rosuvastatin Market', '1'),
+-- ('Retail(Quarterly)', 'Atorvastatin Market', '1'),
+-- ('Retail(Quarterly)', 'Metoprolol Succinate Market', '1'),
+-- ('Retail(Quarterly)', 'PPI Oral ESOMEPRAZOLE Market', '1'),
+-- ('Retail(Quarterly)', 'PPI Oral OMEPRAZOLE Market', '1'),
+-- ('Retail(Quarterly)', 'Inhaled Extended Market', '1'),
+-- ('Retail(Quarterly)', 'Non-Omeprazole', '1'),
+-- ('Retail(Quarterly)', 'Metoprolol Tartrate Market', '1'),
+-- ('Retail(Quarterly)', 'Pediatric Cough Market', '1'),
+-- ('Retail(Quarterly)', 'Anti-HER2 Market', '1'),
+-- ('Retail(Quarterly)', 'Anti-HER2 mBC Market', '1'),
+-- ('Retail(Quarterly)', 'Fasenra Market', '1'),
+-- ('Retail(Quarterly)', 'Severe Asthma', '1'),
+-- ('Retail(Quarterly)', 'DAPA Market', '1'),
+
+-- -- DTP(Quarterly) 部分
+-- ('DTP(Quarterly)', 'EGFR TKI Market', '1'),
+-- ('DTP(Quarterly)', 'Anti-HER2 Market', '1'),
+-- ('DTP(Quarterly)', 'Anti-HER2 mBC Market', '1'),
+-- ('DTP(Quarterly)', 'Fasenra/Tezspire Related Market', '1'),
+-- ('DTP(Quarterly)', 'Severe Asthma', '1'),
+-- ('DTP(Quarterly)', 'Tezspire CRSwNP Market', '1'),
+-- ('DTP(Quarterly)', 'CRSwNP Market', '1'),
+-- ('DTP(Quarterly)', 'PAM Market', '1');
+
+-- COMMAND ----------
+
+-- ============================================================
+-- 目标表：dm.dm_td_ext_unionall_market
+-- 功能：整合各数据源的市场、TA、AZ关联、产品标志等维度信息
+-- 调度：全量覆盖写入（INSERT OVERWRITE）
+-- 作者/更新：chenwu
+-- ============================================================
+INSERT OVERWRITE dm.dm_td_ext_unionall_market (
+  MARKET, DATA_SOURCE, TA, AZ_RELATED, MARKET_DESC, TA_RN, PRD_FLAG, ETL_INSERT_DT, ETL_UPDATE_DT
+)
+
+WITH
+
+-- ------------------------------------------------------------
+-- market_info：从包映射表取市场-数据源组合，关联竞品表补充市场描述
+-- 用 MAX(MARKET_DESC) 避免一对多 JOIN 导致重复行
+-- ------------------------------------------------------------
+market_info AS (
+  SELECT
+    t1.MARKET,
+    t1.DATASOURCE          AS DATA_SOURCE,
+    MAX(t2.MARKETDESC)     AS MARKET_DESC
+  FROM dm.dm_td_ext_unionall_market_pack_mapping t1
+  LEFT JOIN dm.dm_td_external_keycompatitor      t2
+    ON t1.MARKET = t2.MARKET
+  GROUP BY 1, 2
+),
+
+-- ------------------------------------------------------------
+-- TA_info：从市场属性表取市场-TA-排序，过滤占位行（MARKET_ORDER=999）
+-- 末尾 UNION ALL 补充"ALL Market"汇总行
+-- 改用 SELECT DISTINCT 替代无聚合的 GROUP BY
+-- ------------------------------------------------------------
+TA_info AS (
+  SELECT DISTINCT
+    MARKET,
+    TA,
+    TA_ORDER
+  FROM DM.DM_IMS_TD_MARKET_PROPERTY
+  WHERE MARKET_ORDER <> 999
+  UNION ALL
+
+  -- SELECT
+  --   'CVRM Market' AS MARKET,
+  --   'CVRM'        AS TA,
+  --   0           AS TA_ORDER
+  -- UNION ALL
+
+  SELECT
+    'ALL Market' AS MARKET,
+    'ALL'        AS TA,
+    -1           AS TA_ORDER
+),
+
+-- ------------------------------------------------------------
+-- AZ_RELATED_info：从临时映射表取市场-数据源-AZ关联关系（已去重）
+-- ------------------------------------------------------------
+AZ_RELATED_info AS (
+  SELECT DISTINCT
+    MARKET,
+    TA           AS AZ_RELATED,
+    DATA_SOURCE
+  FROM tmp.tmp_td_ext_unionall_market_ta_mapping
+)
+
+-- ------------------------------------------------------------
+-- 主查询：以 market_info 为驱动表，依次关联 TA、AZ关联、隐藏标志
+-- hide_flag 为小维表，添加 BROADCAST hint 避免 shuffle
+-- PRD_FLAG=1 表示市场在隐藏标志表中存在（需隐藏/特殊处理）
+-- ------------------------------------------------------------
+SELECT /*+ BROADCAST(t4) */
+DISTINCT
+  t1.MARKET,
+  t1.DATA_SOURCE,
+  t2.TA,
+  t3.AZ_RELATED,
+  t1.MARKET_DESC,
+  t2.TA_ORDER                                        AS TA_RN,
+  IF(t4.MARKET IS NULL, 0, 1)                        AS PRD_FLAG,
+  FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8')   AS ETL_INSERT_DT,
+  FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8')   AS ETL_UPDATE_DT
+FROM market_info t1
+LEFT JOIN TA_info            t2
+  ON  t1.MARKET      = t2.MARKET
+LEFT JOIN AZ_RELATED_info    t3
+  ON  t1.MARKET      = t3.MARKET
+  AND t1.DATA_SOURCE = t3.DATA_SOURCE
+LEFT JOIN 
+     tmp.tmp_td_ext_market_hide_flag t4
+  ON  t1.MARKET      = t4.MARKET
+  AND t1.DATA_SOURCE = t4.DATA_SOURCE
+
+-- COMMAND ----------
+
+insert overwrite dm.dm_td_ext_unionall_market_ta (
+  MARKET, DATA_SOURCE, TA, TA_RN, MARKET_DESC, ETL_INSERT_DT, ETL_UPDATE_DT
+)
+  SELECT
+    MARKET,
+    DATA_SOURCE,
+    AZ_RELATED TA,
+    TA_RN,
+    MARKET_DESC,
+    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
+    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
+  FROM
+    DM.dm_td_ext_unionall_market
+  -- WHERE MARKET NOT LIKE '%ALL Market%'  and MARKET <>'Retail All Market'
+  -- UNION all
+  -- SELECT
+  --   MARKET,
+  --   DATA_SOURCE,
+  --   'CVRM' TA,
+  --   0 TA_RN,
+  --   MARKET_DESC,
+  --   FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
+  --   FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
+  -- FROM
+  --   DM.dm_td_ext_unionall_market
+  -- WHERE
+  --   MARKET = 'CVRM Market'
+
+-- COMMAND ----------
+
+-----------------------------------------------------------------------------------
+--修改时间：20240822
+--修改人：FanXujia
+--修改内容：
+--Retail、EC渠道，新增NIAD这个TA。数据与DM相同
+-----------------------------------------------------------------------------------
+insert into dm.dm_td_ext_unionall_market_ta (
+  MARKET, DATA_SOURCE, TA, TA_RN, MARKET_DESC, ETL_INSERT_DT, ETL_UPDATE_DT
+)
+  select
+    market,
+    DATA_SOURCE,
+    'NIAD' as TA,
+    7 as TA_RN,
+    MARKET_DESC,
+    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
+    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
+  from
+    dm.dm_td_ext_unionall_market_ta
+  where
+    DATA_SOURCE = 'Retail(Quarterly)'
+    and ta = 'DM'
+  union all
+  select
+    market,
+    DATA_SOURCE,
+    'NIAD' as TA,
+    7 as TA_RN,
+    MARKET_DESC,
+    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
+    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
+  from
+    dm.dm_td_ext_unionall_market_ta
+  where
+    DATA_SOURCE = 'EC(Monthly)'
+    and ta = 'DM'
+  --20250701 chenwu 新增RARE = ONCO的Soliris Related Market
+  union all
+  select
+    market,
+    DATA_SOURCE,
+    'RARE' as TA,
+    8 as TA_RN,
+    MARKET_DESC,
+    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
+    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
+  from
+    dm.dm_td_ext_unionall_market_ta
+  where
+    DATA_SOURCE = 'DTP(Quarterly)'
+    and ta = 'ONCO'
+    and MARKET = 'Soliris Related Market';
+
+-- COMMAND ----------
+
+-- log 20250226 auth:庄伟
+-- 对market_ta表中市场进行补全，并依据手工文件维护的 market 与 ta 映射，匹配出完整市场对应的TA_NAME
+insert overwrite table dm.dm_td_ext_unionall_market_ta (
+  MARKET, DATA_SOURCE, TA, TA_MAP, TA_RN, MARKET_DESC, ETL_INSERT_DT, ETL_UPDATE_DT
+)
+  -----找出在market维度表中不存在的market列表
+  with temp_ma_ext_market as (select distinct
+      MARKET,
+      DATA_SOURCE
+    from
+      dm.dm_td_ext_unionall_market
+    where
+      MARKET NOT IN (
+        select distinct
+          MARKET
+        from
+          dm.dm_td_ext_unionall_market_ta
+      )
+  )
+  ------补充完整的market列表，并通过market匹配手工文件维护的market对应的TA
+  select
+  distinct
+    dim_mkt_ta.MARKET,
+    DATA_SOURCE,
+    dim_mkt_ta.TA,
+    case
+      when dim_mkt_ta.MARKET = 'ALL Market' then 'ALL'
+      else nvl(mkt_property.ta, mkt_property_key.TA)
+    end as TA_MAP,
+    dim_mkt_ta.TA_RN,
+    dim_mkt_ta.MARKET_DESC,
+    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
+    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
+  from
+    (
+      select
+        MARKET,
+        DATA_SOURCE,
+        TA,
+        TA_RN,
+        MARKET_DESC
+      from
+        dm.dm_td_ext_unionall_market_ta
+    ) dim_mkt_ta
+      left join dm.dm_ims_td_market_property mkt_property
+        on dim_mkt_ta.market = mkt_property.market
+      left join dm.dm_ims_td_market_property mkt_property_key
+        on dim_mkt_ta.MARKET = mkt_property_key.MARKET_KEY
+  union all
+  select
+    ext_market.MARKET,
+    ext_market.DATA_SOURCE,
+    null TA,
+    mkt_property.TA TA_MAP,
+    '99' as TA_RN,
+    null MARKET_DESC,
+    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
+    FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
+  from
+    temp_ma_ext_market ext_market
+      left join dm.dm_ims_td_market_property mkt_property
+        on ext_market.market = mkt_property.MARKET
+  where
+    nvl(mkt_property.TA, '') <> '';