chenwu/MarketAnalysis-ETL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
16327a9a32d9a5d35c3344258ca0ad9507053f23
MarketAnalysis-ETL/11 DM_TD_EXTERNAL_MARKET_NEW.sql
zhuchenwu@chenwuzhu.cn c8ef67ca0e ec/retail_ta 
1.1 零售/EC的DM=NIAD (Non-Insulin Anti-Diabetic) Market    GI=PPI Oral Market
2026-04-27 08:06:31 +00:00

425 lines
19 KiB
SQL
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_UNIONALL_MARKET (
-- MARKET STRING,
-- DATA_SOURCE STRING,
-- TA STRING,
-- AZ_RELATED STRING,
-- MARKET_DESC STRING,
-- TA_RN INT,
-- PRD_FLAG STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market';
-- COMMAND ----------
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_UNIONALL_MARKET_TA (
-- MARKET STRING COMMENT '市场',
-- DATA_SOURCE STRING COMMENT '数据源',
-- TA STRING COMMENT '治疗领域',
-- TA_MAP STRING COMMENT '治疗领域匹配',
-- TA_RN INT COMMENT '治疗领域排名',
-- MARKET_DESC STRING COMMENT '市场描述',
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP
-- )
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market_ta';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market_ta';
-- COMMAND ----------
-- DBTITLE 1,pbi页面展示ta标签
-- -- ***********************************************************
-- -- 表名：tmp.tmp_td_ext_unionall_market_ta_mapping
-- -- 功能：市场(Market)与治疗领域(TA)的映射关系表
-- -- 用途：统一各数据源的市场-治疗领域映射标准
-- -- 更新方式：INSERT OVERWRITE（全量覆盖）
-- -- ***********************************************************
-- CREATE OR REPLACE TABLE tmp.tmp_td_ext_unionall_market_ta_mapping
-- (
-- market STRING COMMENT '市场名称',
-- ta STRING COMMENT '治疗领域代码',
-- data_source STRING COMMENT '数据源类型及频率'
-- )
-- COMMENT '市场-治疗领域映射表，用于统一跨源数据关联标准';
-- -- 清空并插入映射数据
-- INSERT OVERWRITE TABLE tmp.tmp_td_ext_unionall_market_ta_mapping (market, ta, data_source) VALUES
-- -- ==================== EC(Monthly) 数据源 ====================
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'DM', 'EC(Monthly)'), -- 非胰岛素抗糖尿病市场 → DM领域
-- ('PPI Oral Market', 'GI', 'EC(Monthly)'), -- 胃肠病市场 → GI领域
-- ('RE Market', 'RE', 'EC(Monthly)'), -- 呼吸市场 → RE领域
-- ('CVRM Market', 'CVRM', 'EC(Monthly)'), -- 心血管肾脏代谢市场 → CVRM领域
-- -- ==================== DTP(Quarterly) 数据源 ====================
-- ('ATTR_EPL', 'CV', 'DTP(Quarterly)'), -- ATTR淀粉样变性 → CV领域
-- ('ONC Market', 'ONCO', 'DTP(Quarterly)'), -- 肿瘤市场 → ONCO领域
-- ('RARE Market', 'RARE', 'DTP(Quarterly)'), -- 罕见病市场 → RARE领域
-- -- ==================== Retail(Quarterly) 数据源 ====================
-- ('RE Market', 'RE', 'Retail(Quarterly)'), -- 呼吸市场 → RE领域
-- ('CV Market', 'CV', 'Retail(Quarterly)'), -- 心血管市场 → CV领域
-- ('RD Market', 'RD', 'Retail(Quarterly)'), -- 肾脏病市场 → RD领域
-- ('PPI Oral Market', 'GI', 'Retail(Quarterly)'), -- 胃肠病市场 → GI领域
-- ('CVRM Market', 'CVRM', 'Retail(Quarterly)'), -- 心血管肾脏代谢市场 → CVRM领域
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'DM', 'Retail(Quarterly)'), -- 糖尿病市场 → DM领域
-- -- ==================== THC(Quarterly) 数据源 ====================
-- ('DM Market', 'DM', 'THC(Quarterly)'), -- 糖尿病市场 → DM领域
-- ('GI Market', 'GI', 'THC(Quarterly)'), -- 胃肠病市场 → GI领域
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'THC(Quarterly)'), -- 非胰岛素抗糖尿病市场 → NIAD领域
-- ('RE Market', 'RE', 'THC(Quarterly)'), -- 呼吸市场 → RE领域
-- ('ONC Market', 'ONCO', 'THC(Quarterly)'), -- 肿瘤市场 → ONCO领域
-- ('CV Market', 'CV', 'THC(Quarterly)'), -- 心血管市场 → CV领域
-- ('CVRM Market', 'CVRM', 'THC(Quarterly)'), -- 心血管肾脏代谢市场 → CVRM领域
-- -- ==================== XH Data(Quarterly) 数据源 ====================
-- ('RE Market', 'RE', 'XH Data(Quarterly)'), -- 呼吸市场 → RE领域
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'XH Data(Quarterly)'), -- 非胰岛素抗糖尿病市场 → NIAD领域
-- ('RD Market', 'RD', 'XH Data(Quarterly)'), -- 肾脏病市场 → RD领域
-- ('DM Market', 'DM', 'XH Data(Quarterly)'), -- 糖尿病市场 → DM领域
-- ('ONC Market', 'ONCO', 'XH Data(Quarterly)'), -- 肿瘤市场 → ONCO领域
-- ('RARE Market', 'RARE', 'XH Data(Quarterly)'), -- 罕见病市场 → RARE领域
-- ('CV Market', 'CV', 'XH Data(Quarterly)'), -- 心血管市场 → CV领域
-- ('CVRM Market', 'CVRM', 'XH Data(Quarterly)'), -- 心血管肾脏代谢市场 → CVRM领域
-- -- ==================== IQVIA-COUNTY(Quarterly) 数据源 ====================
-- ('DM Market', 'DM', 'IQVIA-COUNTY(Quarterly)'), -- 糖尿病市场 → DM领域
-- ('ONC Market', 'ONCO', 'IQVIA-COUNTY(Quarterly)'), -- 肿瘤市场 → ONCO领域
-- ('GI Market', 'GI', 'IQVIA-COUNTY(Quarterly)'), -- 胃肠病市场 → GI领域
-- ('RE Market', 'RE', 'IQVIA-COUNTY(Quarterly)'), -- 呼吸市场 → RE领域
-- ('CV Market', 'CV', 'IQVIA-COUNTY(Quarterly)'), -- 心血管市场 → CV领域
-- ('RARE Market', 'RARE', 'IQVIA-COUNTY(Quarterly)'), -- 罕见病市场 → RARE领域
-- ('RD Market', 'RD', 'IQVIA-COUNTY(Quarterly)'), -- 肾脏病市场 → RD领域
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'IQVIA-COUNTY(Quarterly)'), -- 非胰岛素抗糖尿病市场 → NIAD领域
-- ('CVRM Market', 'CVRM', 'IQVIA-COUNTY(Quarterly)'), -- 心血管肾脏代谢市场 → CVRM领域
-- -- ==================== IQVIA-CHPA(Monthly) 数据源 ====================
-- ('ONC Market', 'ONCO', 'IQVIA-CHPA(Monthly)'), -- 肿瘤市场 → ONCO领域
-- ('RE Market', 'RE', 'IQVIA-CHPA(Monthly)'), -- 呼吸市场 → RE领域
-- ('RD Market', 'RD', 'IQVIA-CHPA(Monthly)'), -- 肾脏病市场 → RD领域
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'IQVIA-CHPA(Monthly)'), -- 非胰岛素抗糖尿病市场 → NIAD领域
-- ('RARE Market', 'RARE', 'IQVIA-CHPA(Monthly)'), -- 罕见病市场 → RARE领域
-- ('DM Market', 'DM', 'IQVIA-CHPA(Monthly)'), -- 糖尿病市场 → DM领域
-- ('GI Market', 'GI', 'IQVIA-CHPA(Monthly)'), -- 胃肠病市场 → GI领域
-- ('CV Market', 'CV', 'IQVIA-CHPA(Monthly)'), -- 心血管市场 → CV领域
-- ('CVRM Market', 'CVRM', 'IQVIA-CHPA(Monthly)'), -- 心血管肾脏代谢市场 → CVRM领域
-- -- ==================== AIA(Monthly) 数据源 ====================
-- ('DM Market', 'DM', 'AIA(Monthly)'), -- 糖尿病市场 → DM领域
-- ('CV Market', 'CV', 'AIA(Monthly)'), -- 心血管市场 → CV领域
-- ('GI Market', 'GI', 'AIA(Monthly)'), -- 胃肠病市场 → GI领域
-- ('RE Market', 'RE', 'AIA(Monthly)'), -- 呼吸市场 → RE领域
-- ('RARE Market', 'RARE', 'AIA(Monthly)'), -- 罕见病市场 → RARE领域
-- ('ONC Market', 'ONCO', 'AIA(Monthly)'), -- 肿瘤市场 → ONCO领域
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'AIA(Monthly)'), -- 非胰岛素抗糖尿病市场 → NIAD领域
-- ('RD Market', 'RD', 'AIA(Monthly)'), -- 肾脏病市场 → RD领域
-- ('CVRM Market', 'CVRM', 'AIA(Monthly)'), -- 心血管肾脏代谢市场 → CVRM领域
-- -- ==================== CHC(Quarterly) 数据源 ====================
-- ('GI Market', 'GI', 'CHC(Quarterly)'), -- 胃肠病市场 → GI领域
-- ('DM Market', 'DM', 'CHC(Quarterly)'), -- 糖尿病市场 → DM领域
-- ('RE Market', 'RE', 'CHC(Quarterly)'), -- 呼吸市场 → RE领域
-- ('CV Market', 'CV', 'CHC(Quarterly)'), -- 心血管市场 → CV领域
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'CHC(Quarterly)'), -- 非胰岛素抗糖尿病市场 → NIAD领域
-- ('CVRM Market', 'CVRM', 'CHC(Quarterly)') -- 心血管肾脏代谢市场 → CVRM领域
-- ;
-- COMMAND ----------
-- DBTITLE 1,藏数逻辑标签
-- -- 建表
-- CREATE TABLE IF NOT EXISTS tmp.tmp_td_ext_market_hide_flag (
-- data_source VARCHAR(20),
-- market VARCHAR(100),
-- flag VARCHAR(1)
-- );
-- -- 插入数据（对应 case when 中返回 '1' 的规则）
-- INSERT overwrite tmp.tmp_td_ext_market_hide_flag (data_source, market, flag) VALUES
-- -- Retail(Quarterly) 部分
-- ('Retail(Quarterly)', 'Antacid anti GI swelling agent+PPI Oral', '1'),
-- ('Retail(Quarterly)', 'HTN Market', '1'),
-- ('Retail(Quarterly)', 'NIAD (Non-Insulin Anti-Diabetic) Market', '1'),
-- ('Retail(Quarterly)', 'Respules Market', '1'),
-- ('Retail(Quarterly)', 'Statin+XZK Market', '1'),
-- ('Retail(Quarterly)', 'BUDESONIDE Market', '1'),
-- ('Retail(Quarterly)', 'Rosuvastatin Market', '1'),
-- ('Retail(Quarterly)', 'Atorvastatin Market', '1'),
-- ('Retail(Quarterly)', 'Metoprolol Succinate Market', '1'),
-- ('Retail(Quarterly)', 'PPI Oral ESOMEPRAZOLE Market', '1'),
-- ('Retail(Quarterly)', 'PPI Oral OMEPRAZOLE Market', '1'),
-- ('Retail(Quarterly)', 'Inhaled Extended Market', '1'),
-- ('Retail(Quarterly)', 'Non-Omeprazole', '1'),
-- ('Retail(Quarterly)', 'Metoprolol Tartrate Market', '1'),
-- ('Retail(Quarterly)', 'Pediatric Cough Market', '1'),
-- ('Retail(Quarterly)', 'Anti-HER2 Market', '1'),
-- ('Retail(Quarterly)', 'Anti-HER2 mBC Market', '1'),
-- ('Retail(Quarterly)', 'Fasenra Market', '1'),
-- ('Retail(Quarterly)', 'Severe Asthma', '1'),
-- ('Retail(Quarterly)', 'DAPA Market', '1'),
-- -- DTP(Quarterly) 部分
-- ('DTP(Quarterly)', 'EGFR TKI Market', '1'),
-- ('DTP(Quarterly)', 'Anti-HER2 Market', '1'),
-- ('DTP(Quarterly)', 'Anti-HER2 mBC Market', '1'),
-- ('DTP(Quarterly)', 'Fasenra/Tezspire Related Market', '1'),
-- ('DTP(Quarterly)', 'Severe Asthma', '1'),
-- ('DTP(Quarterly)', 'Tezspire CRSwNP Market', '1'),
-- ('DTP(Quarterly)', 'CRSwNP Market', '1'),
-- ('DTP(Quarterly)', 'PAM Market', '1');
-- COMMAND ----------
-- ============================================================
-- 目标表：dm.dm_td_ext_unionall_market
-- 功能：整合各数据源的市场、TA、AZ关联、产品标志等维度信息
-- 调度：全量覆盖写入（INSERT OVERWRITE）
-- 作者/更新：chenwu
-- ============================================================
INSERT OVERWRITE dm.dm_td_ext_unionall_market (
MARKET, DATA_SOURCE, TA, AZ_RELATED, MARKET_DESC, TA_RN, PRD_FLAG, ETL_INSERT_DT, ETL_UPDATE_DT
)
WITH
-- ------------------------------------------------------------
-- market_info：从包映射表取市场-数据源组合，关联竞品表补充市场描述
-- 用 MAX(MARKET_DESC) 避免一对多 JOIN 导致重复行
-- ------------------------------------------------------------
market_info AS (
SELECT
t1.MARKET,
t1.DATASOURCE AS DATA_SOURCE,
MAX(t2.MARKETDESC) AS MARKET_DESC
FROM dm.dm_td_ext_unionall_market_pack_mapping t1
LEFT JOIN dm.dm_td_external_keycompatitor t2
ON t1.MARKET = t2.MARKET
GROUP BY 1, 2
),
-- ------------------------------------------------------------
-- TA_info：从市场属性表取市场-TA-排序，过滤占位行（MARKET_ORDER=999）
-- 末尾 UNION ALL 补充"ALL Market"汇总行
-- 改用 SELECT DISTINCT 替代无聚合的 GROUP BY
-- ------------------------------------------------------------
TA_info AS (
SELECT DISTINCT
MARKET,
TA,
TA_ORDER
FROM DM.DM_IMS_TD_MARKET_PROPERTY
WHERE MARKET_ORDER <> 999
UNION ALL
-- SELECT
-- 'CVRM Market' AS MARKET,
-- 'CVRM' AS TA,
-- 0 AS TA_ORDER
-- UNION ALL
SELECT
'ALL Market' AS MARKET,
'ALL' AS TA,
-1 AS TA_ORDER
),
-- ------------------------------------------------------------
-- AZ_RELATED_info：从临时映射表取市场-数据源-AZ关联关系（已去重）
-- ------------------------------------------------------------
AZ_RELATED_info AS (
SELECT DISTINCT
MARKET,
TA AS AZ_RELATED,
DATA_SOURCE
FROM tmp.tmp_td_ext_unionall_market_ta_mapping
)
-- ------------------------------------------------------------
-- 主查询：以 market_info 为驱动表，依次关联 TA、AZ关联、隐藏标志
-- hide_flag 为小维表，添加 BROADCAST hint 避免 shuffle
-- PRD_FLAG=1 表示市场在隐藏标志表中存在（需隐藏/特殊处理）
-- ------------------------------------------------------------
SELECT /*+ BROADCAST(t4) */
DISTINCT
t1.MARKET,
t1.DATA_SOURCE,
t2.TA,
t3.AZ_RELATED,
t1.MARKET_DESC,
t2.TA_ORDER AS TA_RN,
IF(t4.MARKET IS NULL, 0, 1) AS PRD_FLAG,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
FROM market_info t1
LEFT JOIN TA_info t2
ON t1.MARKET = t2.MARKET
LEFT JOIN AZ_RELATED_info t3
ON t1.MARKET = t3.MARKET
AND t1.DATA_SOURCE = t3.DATA_SOURCE
LEFT JOIN
tmp.tmp_td_ext_market_hide_flag t4
ON t1.MARKET = t4.MARKET
AND t1.DATA_SOURCE = t4.DATA_SOURCE
-- COMMAND ----------
insert overwrite dm.dm_td_ext_unionall_market_ta (
MARKET, DATA_SOURCE, TA, TA_RN, MARKET_DESC, ETL_INSERT_DT, ETL_UPDATE_DT
)
SELECT
MARKET,
DATA_SOURCE,
AZ_RELATED TA,
TA_RN,
MARKET_DESC,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
FROM
DM.dm_td_ext_unionall_market
-- WHERE MARKET NOT LIKE '%ALL Market%' and MARKET <>'Retail All Market'
-- UNION all
-- SELECT
-- MARKET,
-- DATA_SOURCE,
-- 'CVRM' TA,
-- 0 TA_RN,
-- MARKET_DESC,
-- FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
-- FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
-- FROM
-- DM.dm_td_ext_unionall_market
-- WHERE
-- MARKET = 'CVRM Market'
-- COMMAND ----------
-----------------------------------------------------------------------------------
--修改时间：20240822
--修改人：FanXujia
--修改内容：
--Retail、EC渠道，新增NIAD这个TA。数据与DM相同
-----------------------------------------------------------------------------------
insert into dm.dm_td_ext_unionall_market_ta (
MARKET, DATA_SOURCE, TA, TA_RN, MARKET_DESC, ETL_INSERT_DT, ETL_UPDATE_DT
)
select
market,
DATA_SOURCE,
'NIAD' as TA,
7 as TA_RN,
MARKET_DESC,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
from
dm.dm_td_ext_unionall_market_ta
where
DATA_SOURCE = 'Retail(Quarterly)'
and ta = 'DM'
union all
select
market,
DATA_SOURCE,
'NIAD' as TA,
7 as TA_RN,
MARKET_DESC,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
from
dm.dm_td_ext_unionall_market_ta
where
DATA_SOURCE = 'EC(Monthly)'
and ta = 'DM'
--20250701 chenwu 新增RARE = ONCO的Soliris Related Market
union all
select
market,
DATA_SOURCE,
'RARE' as TA,
8 as TA_RN,
MARKET_DESC,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
from
dm.dm_td_ext_unionall_market_ta
where
DATA_SOURCE = 'DTP(Quarterly)'
and ta = 'ONCO'
and MARKET = 'Soliris Related Market';
-- COMMAND ----------
-- log 20250226 auth:庄伟
-- 对market_ta表中市场进行补全，并依据手工文件维护的 market 与 ta 映射，匹配出完整市场对应的TA_NAME
insert overwrite table dm.dm_td_ext_unionall_market_ta (
MARKET, DATA_SOURCE, TA, TA_MAP, TA_RN, MARKET_DESC, ETL_INSERT_DT, ETL_UPDATE_DT
)
-----找出在market维度表中不存在的market列表
with temp_ma_ext_market as (select distinct
MARKET,
DATA_SOURCE
from
dm.dm_td_ext_unionall_market
where
MARKET NOT IN (
select distinct
MARKET
from
dm.dm_td_ext_unionall_market_ta
)
)
------补充完整的market列表，并通过market匹配手工文件维护的market对应的TA
select
distinct
dim_mkt_ta.MARKET,
DATA_SOURCE,
dim_mkt_ta.TA,
case
when dim_mkt_ta.MARKET = 'ALL Market' then 'ALL'
else nvl(mkt_property.ta, mkt_property_key.TA)
end as TA_MAP,
dim_mkt_ta.TA_RN,
dim_mkt_ta.MARKET_DESC,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
from
(
select
MARKET,
DATA_SOURCE,
TA,
TA_RN,
MARKET_DESC
from
dm.dm_td_ext_unionall_market_ta
) dim_mkt_ta
left join dm.dm_ims_td_market_property mkt_property
on dim_mkt_ta.market = mkt_property.market
left join dm.dm_ims_td_market_property mkt_property_key
on dim_mkt_ta.MARKET = mkt_property_key.MARKET_KEY
union all
select
ext_market.MARKET,
ext_market.DATA_SOURCE,
null TA,
mkt_property.TA TA_MAP,
'99' as TA_RN,
null MARKET_DESC,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
from
temp_ma_ext_market ext_market
left join dm.dm_ims_td_market_property mkt_property
on ext_market.market = mkt_property.MARKET
where
nvl(mkt_property.TA, '') <> '';
Reference in New Issue View Git Blame Copy Permalink