chenwu/MarketAnalysis-ETL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update20260427

Browse Source
This commit is contained in:
chenwu
2026-04-27 11:04:09 +08:00
commit 3e14e78af3
173 changed files with 62579 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

541
UNIONALL/DM_TD_EXT_UNIONALL_MARKET_PACK_MAPPING.sql Normal file
View File

@@ -0,0 +1,541 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_UNIONALL_MARKET_PACK_MAPPING
-- (
-- PACK_CODE STRING,
-- MARKET STRING,
-- CLASS STRING,
-- KEY_COMPETITOR STRING,
-- MARKET_RATIO STRING,
-- STARTTIME STRING,
-- ENDTIME STRING,
-- VALUE_BRAND_RATIO STRING,
-- VALUE_BRAND_RATIO_START STRING,
-- VALUE_BRAND_RATIO_END STRING,
-- UNIT_BRAND_RATIO STRING,
-- UNIT_BRAND_START STRING,
-- UNIT_BRAND_END STRING,
-- COUNTINGUNIT_BRAND_RATIO STRING,
-- COUNTINGUNIT_BRAND_START STRING,
-- COUNTINGUNIT_BRAND_END STRING,
-- PDOT_BRAND_RATIO STRING,
-- PDOT_BRAND_START STRING,
-- PDOT_BRAND_END STRING,
-- DATASOURCE STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP
-- )
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market_pack_mapping';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market_pack_mapping';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_UNIONALL_MARKET_PACK_MAPPING
SELECT
PACK_CODE,
MARKET,
CLASS,
KEY_COMPETITOR,
MARKET_RATIO,
STARTTIME,
ENDTIME,
VALUE_BRAND_RATIO,
VALUE_BRAND_RATIO_START,
VALUE_BRAND_RATIO_END,
UNIT_BRAND_RATIO,
UNIT_BRAND_START,
UNIT_BRAND_END,
COUNTINGUNIT_BRAND_RATIO,
COUNTINGUNIT_BRAND_START,
COUNTINGUNIT_BRAND_END,
PDOT_BRAND_RATIO,
PDOT_BRAND_START,
PDOT_BRAND_END,
'AIA(Monthly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_AIA_MARKET_PACK_MAPPING
WHERE UPPER(MARKET) NOT LIKE '%ALL%'
UNION ALL
SELECT
PACK_CODE,
MARKET,
CLASS,
KEY_COMPETITOR,
MARKET_RATIO,
STARTTIME,
ENDTIME,
VALUE_BRAND_RATIO,
VALUE_BRAND_RATIO_START,
VALUE_BRAND_RATIO_END,
UNIT_BRAND_RATIO,
UNIT_BRAND_START,
UNIT_BRAND_END,
COUNTINGUNIT_BRAND_RATIO,
COUNTINGUNIT_BRAND_START,
COUNTINGUNIT_BRAND_END,
PDOT_BRAND_RATIO,
PDOT_BRAND_START,
PDOT_BRAND_END,
'IQVIA-CHPA(Monthly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_CHPA_MARKET_PACK_MAPPING
WHERE UPPER(MARKET) NOT LIKE '%ALL%'
UNION ALL
SELECT
PACK_CODE,
MARKET,
CLASS,
KEY_COMPETITOR,
MARKET_RATIO,
STARTTIME,
ENDTIME,
VALUE_BRAND_RATIO,
VALUE_BRAND_RATIO_START,
VALUE_BRAND_RATIO_END,
UNIT_BRAND_RATIO,
UNIT_BRAND_START,
UNIT_BRAND_END,
COUNTINGUNIT_BRAND_RATIO,
COUNTINGUNIT_BRAND_START,
COUNTINGUNIT_BRAND_END,
PDOT_BRAND_RATIO,
PDOT_BRAND_START,
PDOT_BRAND_END,
'Retail(Quarterly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_RETAIL_MARKET_PACK_MAPPING
WHERE UPPER(MARKET) NOT LIKE '%ALL%'
UNION ALL
SELECT
PACK_CODE,
MARKET,
CLASS,
KEY_COMPETITOR,
MARKET_RATIO,
STARTTIME,
ENDTIME,
VALUE_BRAND_RATIO,
VALUE_BRAND_RATIO_START,
VALUE_BRAND_RATIO_END,
UNIT_BRAND_RATIO,
UNIT_BRAND_START,
UNIT_BRAND_END,
COUNTINGUNIT_BRAND_RATIO,
COUNTINGUNIT_BRAND_START,
COUNTINGUNIT_BRAND_END,
PDOT_BRAND_RATIO,
PDOT_BRAND_START,
PDOT_BRAND_END,
'DTP(Quarterly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_DTP_MARKET_PACK_MAPPING
WHERE UPPER(MARKET) NOT LIKE '%ALL%'
UNION ALL
SELECT
PACK_CODE,
MARKET,
CLASS,
KEY_COMPETITOR,
MARKET_RATIO,
STARTTIME,
ENDTIME,
VALUE_BRAND_RATIO,
VALUE_BRAND_RATIO_START,
VALUE_BRAND_RATIO_END,
UNIT_BRAND_RATIO,
UNIT_BRAND_START,
UNIT_BRAND_END,
COUNTINGUNIT_BRAND_RATIO,
COUNTINGUNIT_BRAND_START,
COUNTINGUNIT_BRAND_END,
PDOT_BRAND_RATIO,
PDOT_BRAND_START,
PDOT_BRAND_END,
'CHC(Quarterly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_CHC_MARKET_PACK_MAPPING
WHERE UPPER(MARKET) NOT LIKE '%ALL%'
UNION ALL
SELECT
PACK_CODE,
MARKET,
CLASS,
KEY_COMPETITOR,
MARKET_RATIO,
STARTTIME,
ENDTIME,
VALUE_BRAND_RATIO,
VALUE_BRAND_RATIO_START,
VALUE_BRAND_RATIO_END,
UNIT_BRAND_RATIO,
UNIT_BRAND_START,
UNIT_BRAND_END,
COUNTINGUNIT_BRAND_RATIO,
COUNTINGUNIT_BRAND_START,
COUNTINGUNIT_BRAND_END,
PDOT_BRAND_RATIO,
PDOT_BRAND_START,
PDOT_BRAND_END,
'IQVIA-COUNTY(Quarterly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_COUNTY_MARKET_PACK_MAPPING
WHERE UPPER(MARKET) NOT LIKE '%ALL%'
UNION ALL
SELECT
PACK_CODE,
MARKET,
CLASS,
KEY_COMPETITOR,
MARKET_RATIO,
STARTTIME,
ENDTIME,
VALUE_BRAND_RATIO,
VALUE_BRAND_RATIO_START,
VALUE_BRAND_RATIO_END,
UNIT_BRAND_RATIO,
UNIT_BRAND_START,
UNIT_BRAND_END,
COUNTINGUNIT_BRAND_RATIO,
COUNTINGUNIT_BRAND_START,
COUNTINGUNIT_BRAND_END,
PDOT_BRAND_RATIO,
PDOT_BRAND_START,
PDOT_BRAND_END,
'XH Data(Quarterly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_XIEHE_MARKET_PACK_MAPPING
WHERE UPPER(MARKET) NOT LIKE '%ALL%'
UNION ALL
SELECT
PACK_CODE,
MARKET,
CLASS,
KEY_COMPETITOR,
MARKET_RATIO,
STARTTIME,
ENDTIME,
VALUE_BRAND_RATIO,
VALUE_BRAND_RATIO_START,
VALUE_BRAND_RATIO_END,
UNIT_BRAND_RATIO,
UNIT_BRAND_START,
UNIT_BRAND_END,
COUNTINGUNIT_BRAND_RATIO,
COUNTINGUNIT_BRAND_START,
COUNTINGUNIT_BRAND_END,
PDOT_BRAND_RATIO,
PDOT_BRAND_START,
PDOT_BRAND_END,
'EC(Monthly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_EC_MARKET_PACK_MAPPING
WHERE UPPER(MARKET) NOT LIKE '%ALL%'
UNION ALL
SELECT
PACK_CODE,
MARKET,
CLASS,
KEY_COMPETITOR,
MARKET_RATIO,
STARTTIME,
ENDTIME,
VALUE_BRAND_RATIO,
VALUE_BRAND_RATIO_START,
VALUE_BRAND_RATIO_END,
UNIT_BRAND_RATIO,
UNIT_BRAND_START,
UNIT_BRAND_END,
COUNTINGUNIT_BRAND_RATIO,
COUNTINGUNIT_BRAND_START,
COUNTINGUNIT_BRAND_END,
PDOT_BRAND_RATIO,
PDOT_BRAND_START,
PDOT_BRAND_END,
'THC(Quarterly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_THC_MARKET_PACK_MAPPING
WHERE UPPER(MARKET) NOT LIKE '%ALL%'
UNION ALL
SELECT
PACK_CODE,
'ALL Market' AS MARKET,
'Others' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
299912 AS ENDTIME,
1 AS VALUE_BRAND_RATIO,
200001 AS VALUE_BRAND_RATIO_START,
299912 AS VALUE_BRAND_RATIO_END,
1 AS UNIT_BRAND_RATIO,
200001 AS UNIT_BRAND_START,
299912 AS UNIT_BRAND_END,
1 AS COUNTINGUNIT_BRAND_RATIO,
200001 AS COUNTINGUNIT_BRAND_START,
299912 AS COUNTINGUNIT_BRAND_END,
1 AS PDOT_BRAND_RATIO,
200001 AS PDOT_BRAND_START,
299912 AS PDOT_BRAND_END,
'EC(Monthly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_EC_MARKET_PACK_MAPPING
GROUP BY
PACK_CODE
UNION ALL
SELECT
PACK_CODE,
'ALL Market' AS MARKET,
'Others' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
299912 AS ENDTIME,
1 AS VALUE_BRAND_RATIO,
200001 AS VALUE_BRAND_RATIO_START,
299912 AS VALUE_BRAND_RATIO_END,
1 AS UNIT_BRAND_RATIO,
200001 AS UNIT_BRAND_START,
299912 AS UNIT_BRAND_END,
1 AS COUNTINGUNIT_BRAND_RATIO,
200001 AS COUNTINGUNIT_BRAND_START,
299912 AS COUNTINGUNIT_BRAND_END,
1 AS PDOT_BRAND_RATIO,
200001 AS PDOT_BRAND_START,
299912 AS PDOT_BRAND_END,
'CHC(Quarterly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_CHC_MARKET_PACK_MAPPING
GROUP BY
PACK_CODE
UNION ALL
SELECT
PACK_CODE,
'ALL Market'MARKET,
'Others' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
299912 AS ENDTIME,
1 AS VALUE_BRAND_RATIO,
200001 AS VALUE_BRAND_RATIO_START,
299912 AS VALUE_BRAND_RATIO_END,
1 AS UNIT_BRAND_RATIO,
200001 AS UNIT_BRAND_START,
299912 AS UNIT_BRAND_END,
1 AS COUNTINGUNIT_BRAND_RATIO,
200001 AS COUNTINGUNIT_BRAND_START,
299912 AS COUNTINGUNIT_BRAND_END,
1 AS PDOT_BRAND_RATIO,
200001 AS PDOT_BRAND_START,
299912 AS PDOT_BRAND_END,
'Retail(Quarterly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_RETAIL_MARKET_PACK_MAPPING
GROUP BY
PACK_CODE
UNION ALL
SELECT
PACK_CODE,
'ALL Market'MARKET,
'Others' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
299912 AS ENDTIME,
1 AS VALUE_BRAND_RATIO,
200001 AS VALUE_BRAND_RATIO_START,
299912 AS VALUE_BRAND_RATIO_END,
1 AS UNIT_BRAND_RATIO,
200001 AS UNIT_BRAND_START,
299912 AS UNIT_BRAND_END,
1 AS COUNTINGUNIT_BRAND_RATIO,
200001 AS COUNTINGUNIT_BRAND_START,
299912 AS COUNTINGUNIT_BRAND_END,
1 AS PDOT_BRAND_RATIO,
200001 AS PDOT_BRAND_START,
299912 AS PDOT_BRAND_END,
'DTP(Quarterly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_DTP_MARKET_PACK_MAPPING
GROUP BY
PACK_CODE
UNION ALL
SELECT
PACK_CODE,
'ALL Market'MARKET,
'Others' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
299912 AS ENDTIME,
1 AS VALUE_BRAND_RATIO,
200001 AS VALUE_BRAND_RATIO_START,
299912 AS VALUE_BRAND_RATIO_END,
1 AS UNIT_BRAND_RATIO,
200001 AS UNIT_BRAND_START,
299912 AS UNIT_BRAND_END,
1 AS COUNTINGUNIT_BRAND_RATIO,
200001 AS COUNTINGUNIT_BRAND_START,
299912 AS COUNTINGUNIT_BRAND_END,
1 AS PDOT_BRAND_RATIO,
200001 AS PDOT_BRAND_START,
299912 AS PDOT_BRAND_END,
'THC(Quarterly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_THC_MARKET_PACK_MAPPING
GROUP BY
PACK_CODE
UNION ALL
SELECT
PACK_CODE,
'ALL Market'MARKET,
'Others' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
299912 AS ENDTIME,
1 AS VALUE_BRAND_RATIO,
200001 AS VALUE_BRAND_RATIO_START,
299912 AS VALUE_BRAND_RATIO_END,
1 AS UNIT_BRAND_RATIO,
200001 AS UNIT_BRAND_START,
299912 AS UNIT_BRAND_END,
1 AS COUNTINGUNIT_BRAND_RATIO,
200001 AS COUNTINGUNIT_BRAND_START,
299912 AS COUNTINGUNIT_BRAND_END,
1 AS PDOT_BRAND_RATIO,
200001 AS PDOT_BRAND_START,
299912 AS PDOT_BRAND_END,
'IQVIA-CHPA(Monthly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_CHPA_MARKET_PACK_MAPPING
GROUP BY
PACK_CODE
UNION ALL
SELECT
PACK_CODE,
'ALL Market'MARKET,
'Others' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
299912 AS ENDTIME,
1 AS VALUE_BRAND_RATIO,
200001 AS VALUE_BRAND_RATIO_START,
299912 AS VALUE_BRAND_RATIO_END,
1 AS UNIT_BRAND_RATIO,
200001 AS UNIT_BRAND_START,
299912 AS UNIT_BRAND_END,
1 AS COUNTINGUNIT_BRAND_RATIO,
200001 AS COUNTINGUNIT_BRAND_START,
299912 AS COUNTINGUNIT_BRAND_END,
1 AS PDOT_BRAND_RATIO,
200001 AS PDOT_BRAND_START,
299912 AS PDOT_BRAND_END,
'XH Data(Quarterly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_XIEHE_MARKET_PACK_MAPPING
GROUP BY
PACK_CODE
UNION ALL
SELECT
PACK_CODE,
'ALL Market'MARKET,
'Others' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
299912 AS ENDTIME,
1 AS VALUE_BRAND_RATIO,
200001 AS VALUE_BRAND_RATIO_START,
299912 AS VALUE_BRAND_RATIO_END,
1 AS UNIT_BRAND_RATIO,
200001 AS UNIT_BRAND_START,
299912 AS UNIT_BRAND_END,
1 AS COUNTINGUNIT_BRAND_RATIO,
200001 AS COUNTINGUNIT_BRAND_START,
299912 AS COUNTINGUNIT_BRAND_END,
1 AS PDOT_BRAND_RATIO,
200001 AS PDOT_BRAND_START,
299912 AS PDOT_BRAND_END,
'AIA(Monthly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_AIA_MARKET_PACK_MAPPING
GROUP BY
PACK_CODE
UNION ALL
SELECT
PACK_CODE,
'ALL Market'MARKET,
'Others' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
299912 AS ENDTIME,
1 AS VALUE_BRAND_RATIO,
200001 AS VALUE_BRAND_RATIO_START,
299912 AS VALUE_BRAND_RATIO_END,
1 AS UNIT_BRAND_RATIO,
200001 AS UNIT_BRAND_START,
299912 AS UNIT_BRAND_END,
1 AS COUNTINGUNIT_BRAND_RATIO,
200001 AS COUNTINGUNIT_BRAND_START,
299912 AS COUNTINGUNIT_BRAND_END,
1 AS PDOT_BRAND_RATIO,
200001 AS PDOT_BRAND_START,
299912 AS PDOT_BRAND_END,
'IQVIA-COUNTY(Quarterly)' AS DATASOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_COUNTY_MARKET_PACK_MAPPING
GROUP BY
PACK_CODE
-- COMMAND ----------
update DM.DM_TD_EXT_UNIONALL_MARKET_PACK_MAPPING set class = 'Others' where lower(CLASS) = 'others' or CLASS IS null

1837
UNIONALL/DM_TD_EXT_UNIONALL_PACKINFO.sql Normal file
View File

File diff suppressed because it is too large Load Diff

524
UNIONALL/DM_TF_EXT_UNIONALL_MARKET_SALES.sql Normal file
View File

@@ -0,0 +1,524 @@
-- Databricks notebook source
-- DBTITLE 1,分区优化
-- CREATE OR REPLACE TABLE DM.DM_TF_EXT_UNIONALL_MARKET_SALES (
-- MARKET STRING,
-- KEY_COMPETITOR STRING,
-- CLASS STRING,
-- YYYYMM STRING,
-- PACK_COD STRING,
-- CORP_COD STRING,
-- AUDIT_COD STRING,
-- PLATFORM_TYPE STRING,
-- STORE_NAME STRING,
-- STORE_TYPE STRING,
-- REGION_TYPE STRING,
-- DATA_SOURCE STRING,
-- PACK_FLAG STRING,
-- PROD_FLAG STRING,
-- DTP_FLAG STRING,
-- CMPS_FLAG STRING,
-- NEW_CODE STRING COMMENT '主数据关联CODE',
-- INST_CODE STRING COMMENT '内部机构编码',
-- AIA_HP_FLAG STRING,
-- DEPT_NAME STRING COMMENT '科室名称',
-- H_LEVEL STRING,
-- REIMBURSE STRING COMMENT '报销情况',
-- REIMBURSE_TYPE STRING COMMENT '报销类型',
-- PRESCRIPTION_TYPE STRING COMMENT '处方来源',
-- PRESCRIPTION DECIMAL(35,10),
-- PRESCRIPTION_LY DECIMAL(35,10),
-- VA STRING,
-- UT STRING,
-- CU STRING,
-- PT STRING,
-- SALES_UNIT_CAL DECIMAL(35,6),
-- SALES_UNIT_CAL_LY DECIMAL(35,6),
-- SALES_VALUE_CAL DECIMAL(35,6),
-- SALES_VALUE_CAL_LY DECIMAL(35,6),
-- CONUTING_UNIT DECIMAL(35,6),
-- CONUTING_UNIT_LY DECIMAL(35,6),
-- PDOT DECIMAL(35,6),
-- PDOT_LY DECIMAL(35,6),
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP
-- )
-- USING delta
-- PARTITIONED BY (DATA_SOURCE)
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_market_sales';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_market_sales';
-- COMMAND ----------
-- DBTITLE 1,性能优化
-- -- 按最常用的过滤和 JOIN 字段做 Z-Order 聚簇
-- OPTIMIZE DM.DM_TF_EXT_UNIONALL_SALES
-- ZORDER BY (DATA_SOURCE, PACK_CODE, YYYYMM);
-- -- 收集统计信息，帮助优化器选择更好的执行计划
-- ANALYZE TABLE DM.DM_TF_EXT_UNIONALL_SALES COMPUTE STATISTICS FOR ALL COLUMNS;
-- ANALYZE TABLE DM.dm_td_ext_unionall_market_pack_mapping COMPUTE STATISTICS FOR ALL COLUMNS;
-- COMMAND ----------
-- DBTITLE 1,测试跑数据
-- delete from DM.DM_TF_EXT_UNIONALL_MARKET_SALES where DATA_SOURCE IN ('Retail(Quarterly)', 'EC(Monthly)')
-- COMMAND ----------
-- DBTITLE 1,测试跑数据
-- INSERT INTO TABLE
-- DM.DM_TF_EXT_UNIONALL_MARKET_SALES
-- WITH
-- t1_filtered AS (
-- SELECT
-- *
-- FROM
-- DM.DM_TF_EXT_UNIONALL_SALES
-- WHERE
-- DATA_SOURCE IN (
-- -- 'IQVIA-CHPA(Monthly)',
-- -- 'AIA(Monthly)'
-- -- 'CHC(Quarterly)',
-- -- 'IQVIA-COUNTY(Quarterly)',
-- -- 'THC(Quarterly)'
-- 'Retail(Quarterly)',
-- -- 'DTP(Quarterly)',
-- 'EC(Monthly)'
-- -- 'XH Data(Quarterly)'
-- ) -- 谓词下推，最先过滤
-- ),
-- mapping_cleaned AS (
-- SELECT
-- *
-- FROM
-- (
-- SELECT
-- PACK_CODE,
-- DATASOURCE,
-- MARKET,
-- KEY_COMPETITOR,
-- CLASS,
-- MARKET_RATIO,
-- Value_brand_Ratio,
-- Unit_brand_Ratio,
-- CountingUnit_brand_Ratio,
-- PDOT_brand_Ratio,
-- GREATEST (
-- starttime,
-- Value_brand_Ratio_start,
-- Unit_brand_start,
-- CountingUnit_brand_start,
-- PDOT_brand_start
-- ) AS eff_start,
-- LEAST (
-- endtime,
-- Value_brand_Ratio_end,
-- Unit_brand_end,
-- CountingUnit_brand_end,
-- PDOT_brand_end
-- ) AS eff_end
-- FROM
-- DM.dm_td_ext_unionall_market_pack_mapping
-- ) tmp
-- WHERE
-- eff_start <= eff_end -- 在外层子查询中过滤
-- ),
-- joined AS (
-- SELECT
-- /*+ BROADCAST(t2, t3) */
-- t1.*,
-- -- 当年 ratio
-- t2.MARKET,
-- t2.KEY_COMPETITOR,
-- t2.CLASS,
-- t2.MARKET_RATIO AS cy_market_ratio,
-- t2.Value_brand_Ratio AS cy_value_ratio,
-- t2.Unit_brand_Ratio AS cy_unit_ratio,
-- t2.CountingUnit_brand_Ratio AS cy_cu_ratio,
-- t2.PDOT_brand_Ratio AS cy_pdot_ratio,
-- -- 去年 ratio
-- t3.MARKET_RATIO AS ly_market_ratio,
-- t3.Value_brand_Ratio AS ly_value_ratio,
-- t3.Unit_brand_Ratio AS ly_unit_ratio,
-- t3.CountingUnit_brand_Ratio AS ly_cu_ratio,
-- t3.PDOT_brand_Ratio AS ly_pdot_ratio
-- FROM
-- t1_filtered t1
-- LEFT JOIN mapping_cleaned t2 ON t1.PACK_CODE = t2.PACK_CODE
-- AND t1.DATA_SOURCE = t2.DATASOURCE
-- AND t1.YYYYMM >= t2.eff_start
-- AND t1.YYYYMM <= t2.eff_end
-- LEFT JOIN mapping_cleaned t3 ON t1.PACK_CODE = t3.PACK_CODE
-- AND t1.DATA_SOURCE = t3.DATASOURCE
-- AND (t1.YYYYMM - 100) >= t3.eff_start
-- AND (t1.YYYYMM - 100) <= t3.eff_end
-- AND t2.MARKET = t3.MARKET -- 保持原逻辑
-- )
-- SELECT
-- NVL(MARKET, 'ALL Market') AS MARKET,
-- NVL(KEY_COMPETITOR, 'OTHERS') AS KEY_COMPETITOR,
-- CASE
-- WHEN trim(lower(CLASS)) = 'others' THEN 'OTHERS'
-- WHEN CLASS IS NULL THEN 'OTHERS'
-- ELSE CLASS
-- END AS CLASS,
-- YYYYMM,
-- PACK_CODE,
-- CORP_CODE,
-- AUDIT_CODE,
-- PLATFORM_TYPE,
-- STORE_NAME,
-- STORE_TYPE,
-- REGION_TYPE,
-- DATA_SOURCE,
-- PACK_FLAG,
-- PROD_FLAG,
-- DTP_FLAG,
-- CMPS_FLAG,
-- NEW_CODE,
-- INST_CODE,
-- NULL AS AIA_HP_FLAG,
-- DEPT_NAME,
-- H_LEVEL,
-- REIMBURSE,
-- REIMBURSE_TYPE,
-- PRESCRIPTION_SOURCE AS PRESCRIPTION_TYPE,
-- PRESCRIPTION,
-- PRESCRIPTION_LY,
-- -- 预计算公用因子，避免重复计算
-- NVL(NVL(cy_market_ratio, 1) * cy_value_ratio, 1) AS VA,
-- NVL(NVL(cy_market_ratio, 1) * cy_unit_ratio, 1) AS UT,
-- NVL(NVL(cy_market_ratio, 1) * cy_cu_ratio, 1) AS CU,
-- NVL(NVL(cy_market_ratio, 1) * cy_pdot_ratio, 1) AS PT,
-- CAST(
-- SALES_UNIT_CAL * NVL(cy_unit_ratio, 1) * NVL(cy_market_ratio, 1) AS DECIMAL(35, 6)
-- ) AS SALES_UNIT_CAL,
-- CAST(
-- SALES_UNIT_CAL_LY * NVL(ly_unit_ratio, 1) * NVL(ly_market_ratio, 1) AS DECIMAL(35, 6)
-- ) AS SALES_UNIT_CAL_LY,
-- CAST(
-- SALES_VALUE_CAL * NVL(cy_value_ratio, 1) * NVL(cy_market_ratio, 1) AS DECIMAL(35, 6)
-- ) AS SALES_VALUE_CAL,
-- CAST(
-- SALES_VALUE_CAL_LY * NVL(ly_value_ratio, 1) * NVL(ly_market_ratio, 1) AS DECIMAL(35, 6)
-- ) AS SALES_VALUE_CAL_LY,
-- CAST(
-- CONUTING_UNIT * NVL(cy_cu_ratio, 1) * NVL(cy_market_ratio, 1) AS DECIMAL(35, 6)
-- ) AS CONUTING_UNIT,
-- CAST(
-- CONUTING_UNIT_LY * NVL(ly_cu_ratio, 1) * NVL(ly_market_ratio, 1) AS DECIMAL(35, 6)
-- ) AS CONUTING_UNIT_LY,
-- CAST(
-- CONUTING_UNIT * NVL(cy_pdot_ratio, 1) * NVL(cy_market_ratio, 1) AS DECIMAL(35, 6)
-- ) AS PDOT,
-- CAST(
-- CONUTING_UNIT_LY * NVL(ly_pdot_ratio, 1) * NVL(ly_market_ratio, 1) AS DECIMAL(35, 6)
-- ) AS PDOT_LY,
-- FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
-- FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
-- FROM
-- joined;
-- COMMAND ----------
INSERT OVERWRITE TABLE
DM.DM_TF_EXT_UNIONALL_MARKET_SALES
WITH
t1_filtered AS (
SELECT
*
FROM
DM.DM_TF_EXT_UNIONALL_SALES
WHERE
DATA_SOURCE IN (
'IQVIA-CHPA(Monthly)',
'AIA(Monthly)',
'CHC(Quarterly)',
'IQVIA-COUNTY(Quarterly)',
'THC(Quarterly)',
'Retail(Quarterly)',
'DTP(Quarterly)',
'EC(Monthly)',
'XH Data(Quarterly)'
) -- 谓词下推，最先过滤
),
mapping_cleaned AS (
SELECT
*
FROM
(
SELECT
PACK_CODE,
DATASOURCE,
MARKET,
KEY_COMPETITOR,
CLASS,
MARKET_RATIO,
Value_brand_Ratio,
Unit_brand_Ratio,
CountingUnit_brand_Ratio,
PDOT_brand_Ratio,
GREATEST (
starttime,
Value_brand_Ratio_start,
Unit_brand_start,
CountingUnit_brand_start,
PDOT_brand_start
) AS eff_start,
LEAST (
endtime,
Value_brand_Ratio_end,
Unit_brand_end,
CountingUnit_brand_end,
PDOT_brand_end
) AS eff_end
FROM
DM.dm_td_ext_unionall_market_pack_mapping
) tmp
WHERE
eff_start <= eff_end -- 在外层子查询中过滤
),
joined AS (
SELECT
/*+ BROADCAST(t2, t3) */
t1.*,
-- 当年 ratio
t2.MARKET,
t2.KEY_COMPETITOR,
t2.CLASS,
t2.MARKET_RATIO AS cy_market_ratio,
t2.Value_brand_Ratio AS cy_value_ratio,
t2.Unit_brand_Ratio AS cy_unit_ratio,
t2.CountingUnit_brand_Ratio AS cy_cu_ratio,
t2.PDOT_brand_Ratio AS cy_pdot_ratio,
-- 去年 ratio
t3.MARKET_RATIO AS ly_market_ratio,
t3.Value_brand_Ratio AS ly_value_ratio,
t3.Unit_brand_Ratio AS ly_unit_ratio,
t3.CountingUnit_brand_Ratio AS ly_cu_ratio,
t3.PDOT_brand_Ratio AS ly_pdot_ratio
FROM
t1_filtered t1
LEFT JOIN mapping_cleaned t2 ON t1.PACK_CODE = t2.PACK_CODE
AND t1.DATA_SOURCE = t2.DATASOURCE
AND t1.YYYYMM >= t2.eff_start
AND t1.YYYYMM <= t2.eff_end
LEFT JOIN mapping_cleaned t3 ON t1.PACK_CODE = t3.PACK_CODE
AND t1.DATA_SOURCE = t3.DATASOURCE
AND (t1.YYYYMM - 100) >= t3.eff_start
AND (t1.YYYYMM - 100) <= t3.eff_end
AND t2.MARKET = t3.MARKET -- 保持原逻辑
)
SELECT
NVL(MARKET, 'ALL Market') AS MARKET,
NVL(KEY_COMPETITOR, 'OTHERS') AS KEY_COMPETITOR,
CASE
WHEN trim(lower(CLASS)) = 'others' THEN 'OTHERS'
WHEN CLASS IS NULL THEN 'OTHERS'
ELSE CLASS
END AS CLASS,
YYYYMM,
PACK_CODE,
CORP_CODE,
AUDIT_CODE,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
DATA_SOURCE,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
CMPS_FLAG,
NEW_CODE,
INST_CODE,
NULL AS AIA_HP_FLAG,
DEPT_NAME,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE AS PRESCRIPTION_TYPE,
PRESCRIPTION,
PRESCRIPTION_LY,
-- 预计算公用因子，避免重复计算
NVL(NVL(cy_market_ratio, 1) * cy_value_ratio, 1) AS VA,
NVL(NVL(cy_market_ratio, 1) * cy_unit_ratio, 1) AS UT,
NVL(NVL(cy_market_ratio, 1) * cy_cu_ratio, 1) AS CU,
NVL(NVL(cy_market_ratio, 1) * cy_pdot_ratio, 1) AS PT,
CAST(
SALES_UNIT_CAL * NVL(cy_unit_ratio, 1) * NVL(cy_market_ratio, 1) AS DECIMAL(35, 6)
) AS SALES_UNIT_CAL,
CAST(
SALES_UNIT_CAL_LY * NVL(ly_unit_ratio, 1) * NVL(ly_market_ratio, 1) AS DECIMAL(35, 6)
) AS SALES_UNIT_CAL_LY,
CAST(
SALES_VALUE_CAL * NVL(cy_value_ratio, 1) * NVL(cy_market_ratio, 1) AS DECIMAL(35, 6)
) AS SALES_VALUE_CAL,
CAST(
SALES_VALUE_CAL_LY * NVL(ly_value_ratio, 1) * NVL(ly_market_ratio, 1) AS DECIMAL(35, 6)
) AS SALES_VALUE_CAL_LY,
CAST(
CONUTING_UNIT * NVL(cy_cu_ratio, 1) * NVL(cy_market_ratio, 1) AS DECIMAL(35, 6)
) AS CONUTING_UNIT,
CAST(
CONUTING_UNIT_LY * NVL(ly_cu_ratio, 1) * NVL(ly_market_ratio, 1) AS DECIMAL(35, 6)
) AS CONUTING_UNIT_LY,
CAST(
CONUTING_UNIT * NVL(cy_pdot_ratio, 1) * NVL(cy_market_ratio, 1) AS DECIMAL(35, 6)
) AS PDOT,
CAST(
CONUTING_UNIT_LY * NVL(ly_pdot_ratio, 1) * NVL(ly_market_ratio, 1) AS DECIMAL(35, 6)
) AS PDOT_LY,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
FROM
joined;
-- COMMAND ----------
-- DBTITLE 1,maket_sales_cht
INSERT OVERWRITE TABLE
DM.DM_TF_EXT_THC_MARKET_SALES_CHT
WITH
t1_filtered AS (
SELECT
*
FROM
DM.DM_TF_EXT_THC_SALES
WHERE DATA_SOURCE = 'THC(Quarterly)'
and AUDIT_CODE <> 'CHT'
-- 谓词下推，最先过滤
),
mapping_cleaned AS (
SELECT
*
FROM
(
SELECT
PACK_CODE,
'THC(Quarterly)' as DATASOURCE,
MARKET,
KEY_COMPETITOR,
CLASS,
MARKET_RATIO,
Value_brand_Ratio,
Unit_brand_Ratio,
CountingUnit_brand_Ratio,
PDOT_brand_Ratio,
GREATEST (
starttime,
Value_brand_Ratio_start,
Unit_brand_start,
CountingUnit_brand_start,
PDOT_brand_start
) AS eff_start,
LEAST (
endtime,
Value_brand_Ratio_end,
Unit_brand_end,
CountingUnit_brand_end,
PDOT_brand_end
) AS eff_end
FROM
DM.dm_td_ext_THC_market_pack_mapping
) tmp
WHERE
eff_start <= eff_end -- 在外层子查询中过滤
),
joined AS (
SELECT
/*+ BROADCAST(t2, t3) */
t1.*,
-- 当年 ratio
t2.MARKET,
t2.KEY_COMPETITOR,
t2.CLASS,
t2.MARKET_RATIO AS cy_market_ratio,
t2.Value_brand_Ratio AS cy_value_ratio,
t2.Unit_brand_Ratio AS cy_unit_ratio,
t2.CountingUnit_brand_Ratio AS cy_cu_ratio,
t2.PDOT_brand_Ratio AS cy_pdot_ratio,
-- 去年 ratio
t3.MARKET_RATIO AS ly_market_ratio,
t3.Value_brand_Ratio AS ly_value_ratio,
t3.Unit_brand_Ratio AS ly_unit_ratio,
t3.CountingUnit_brand_Ratio AS ly_cu_ratio,
t3.PDOT_brand_Ratio AS ly_pdot_ratio
FROM
t1_filtered t1
LEFT JOIN mapping_cleaned t2 ON t1.PACK_CODE = t2.PACK_CODE
AND t1.DATA_SOURCE = t2.DATASOURCE
AND t1.YYYYMM >= t2.eff_start
AND t1.YYYYMM <= t2.eff_end
LEFT JOIN mapping_cleaned t3 ON t1.PACK_CODE = t3.PACK_CODE
AND t1.DATA_SOURCE = t3.DATASOURCE
AND (t1.YYYYMM - 100) >= t3.eff_start
AND (t1.YYYYMM - 100) <= t3.eff_end
AND t2.MARKET = t3.MARKET -- 保持原逻辑
)
SELECT
NVL(MARKET, 'ALL Market') AS MARKET,
NVL(KEY_COMPETITOR, 'OTHERS') AS KEY_COMPETITOR,
CASE
WHEN trim(lower(CLASS)) = 'others' THEN 'OTHERS'
WHEN CLASS IS NULL THEN 'OTHERS'
ELSE CLASS
END AS CLASS,
YYYYMM,
PACK_CODE,
CORP_CODE,
AUDIT_CODE,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
DATA_SOURCE,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
CMPS_FLAG,
NEW_CODE,
INST_CODE,
NULL AS AIA_HP_FLAG,
DEPT_NAME,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE AS PRESCRIPTION_TYPE,
PRESCRIPTION,
PRESCRIPTION_LY,
-- 预计算公用因子，避免重复计算
NVL(NVL(cy_market_ratio, 1) * cy_value_ratio, 1) AS VA,
NVL(NVL(cy_market_ratio, 1) * cy_unit_ratio, 1) AS UT,
NVL(NVL(cy_market_ratio, 1) * cy_cu_ratio, 1) AS CU,
NVL(NVL(cy_market_ratio, 1) * cy_pdot_ratio, 1) AS PT,
CAST(
SALES_UNIT_CAL * NVL(cy_unit_ratio, 1) * NVL(cy_market_ratio, 1) AS DECIMAL(35, 6)
) AS SALES_UNIT_CAL,
CAST(
SALES_UNIT_CAL_LY * NVL(ly_unit_ratio, 1) * NVL(ly_market_ratio, 1) AS DECIMAL(35, 6)
) AS SALES_UNIT_CAL_LY,
CAST(
SALES_VALUE_CAL * NVL(cy_value_ratio, 1) * NVL(cy_market_ratio, 1) AS DECIMAL(35, 6)
) AS SALES_VALUE_CAL,
CAST(
SALES_VALUE_CAL_LY * NVL(ly_value_ratio, 1) * NVL(ly_market_ratio, 1) AS DECIMAL(35, 6)
) AS SALES_VALUE_CAL_LY,
CAST(
CONUTING_UNIT * NVL(cy_cu_ratio, 1) * NVL(cy_market_ratio, 1) AS DECIMAL(35, 6)
) AS CONUTING_UNIT,
CAST(
CONUTING_UNIT_LY * NVL(ly_cu_ratio, 1) * NVL(ly_market_ratio, 1) AS DECIMAL(35, 6)
) AS CONUTING_UNIT_LY,
CAST(
CONUTING_UNIT * NVL(cy_pdot_ratio, 1) * NVL(cy_market_ratio, 1) AS DECIMAL(35, 6)
) AS PDOT,
CAST(
CONUTING_UNIT_LY * NVL(ly_pdot_ratio, 1) * NVL(ly_market_ratio, 1) AS DECIMAL(35, 6)
) AS PDOT_LY,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
FROM
joined;

1012
UNIONALL/DM_TF_EXT_UNIONALL_MARKET_SALES_v1.0.sql Normal file
View File

File diff suppressed because it is too large Load Diff

349
UNIONALL/DM_TF_EXT_UNIONALL_SALES.sql Normal file
View File

@@ -0,0 +1,349 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TF_EXT_UNIONALL_SALES (
-- YYYYMM STRING,
-- PACK_CODE STRING,
-- CORP_CODE STRING,
-- AUDIT_CODE STRING,
-- PLATFORM_TYPE STRING,
-- STORE_NAME STRING,
-- STORE_TYPE STRING,
-- REGION_TYPE STRING,
-- PACK_FLAG INT,
-- PROD_FLAG INT,
-- DTP_FLAG INT,
-- SALES_UNIT_CAL DECIMAL(38,10),
-- SALES_UNIT_CAL_LY DECIMAL(38,10),
-- SALES_VALUE_CAL DECIMAL(38,10),
-- SALES_VALUE_CAL_LY DECIMAL(38,10),
-- CONUTING_UNIT DECIMAL(38,10),
-- CONUTING_UNIT_LY DECIMAL(38,10),
-- DATA_SOURCE STRING,
-- INST_CODE STRING COMMENT '内部机构编码',
-- CMPS_FLAG STRING COMMENT '分子式标签',
-- DEPT_NAME STRING COMMENT '科室名称',
-- PRESCRIPTION DECIMAL(38,10) COMMENT '处方张数',
-- PRESCRIPTION_LY DECIMAL(38,10) COMMENT '去年同期处方张数',
-- NEW_CODE STRING COMMENT '主数据关联CODE',
-- AREA STRING COMMENT '城市',
-- H_LEVEL STRING COMMENT '医院类型',
-- REIMBURSE STRING COMMENT '报销情况',
-- REIMBURSE_TYPE STRING COMMENT '报销类型',
-- PRESCRIPTION_SOURCE STRING COMMENT '处方来源',
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TF_EXT_UNIONALL_SALES
SELECT
YYYYMM,
PACK_CODE,
CORP_CODE,
AUDIT_CODE,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
DATA_SOURCE,
INST_CODE,
CMPS_FLAG,
DEPT_NAME,
PRESCRIPTION,
PRESCRIPTION_LY,
NEW_CODE,
AREA,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TF_EXT_AIA_SALES
UNION ALL
SELECT
YYYYMM,
PACK_CODE,
CORP_CODE,
AUDIT_CODE,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
DATA_SOURCE,
INST_CODE,
CMPS_FLAG,
DEPT_NAME,
PRESCRIPTION,
PRESCRIPTION_LY,
NEW_CODE,
AREA,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TF_EXT_CHC_SALES
UNION ALL
SELECT
YYYYMM,
PACK_CODE,
CORP_CODE,
AUDIT_CODE,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
DATA_SOURCE,
INST_CODE,
CMPS_FLAG,
DEPT_NAME,
PRESCRIPTION,
PRESCRIPTION_LY,
NEW_CODE,
AREA,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TF_EXT_CHPA_SALES
UNION ALL
SELECT
YYYYMM,
PACK_CODE,
CORP_CODE,
AUDIT_CODE,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
DATA_SOURCE,
INST_CODE,
CMPS_FLAG,
DEPT_NAME,
PRESCRIPTION,
PRESCRIPTION_LY,
NEW_CODE,
AREA,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TF_EXT_COUNTY_SALES
UNION ALL
SELECT
YYYYMM,
PACK_CODE,
CORP_CODE,
AUDIT_CODE,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
DATA_SOURCE,
INST_CODE,
CMPS_FLAG,
DEPT_NAME,
PRESCRIPTION,
PRESCRIPTION_LY,
NEW_CODE,
AREA,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TF_EXT_EC_SALES
UNION ALL
SELECT
YYYYMM,
PACK_CODE,
CORP_CODE,
AUDIT_CODE,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
DATA_SOURCE,
INST_CODE,
CMPS_FLAG,
DEPT_NAME,
PRESCRIPTION,
PRESCRIPTION_LY,
NEW_CODE,
AREA,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TF_EXT_RETAIL_SALES
UNION ALL
SELECT
YYYYMM,
PACK_CODE,
CORP_CODE,
AUDIT_CODE,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
DATA_SOURCE,
INST_CODE,
CMPS_FLAG,
DEPT_NAME,
PRESCRIPTION,
PRESCRIPTION_LY,
NEW_CODE,
AREA,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TF_EXT_RETAIL_DTP_SALES
UNION ALL
SELECT
YYYYMM,
PACK_CODE,
CORP_CODE,
AUDIT_CODE,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
DATA_SOURCE,
INST_CODE,
CMPS_FLAG,
DEPT_NAME,
PRESCRIPTION,
PRESCRIPTION_LY,
NEW_CODE,
AREA,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TF_EXT_THC_SALES
WHERE
AUDIT_CODE = 'CHT'--只放全国数据
UNION ALL
SELECT
YYYYMM,
PACK_CODE,
CORP_CODE,
AUDIT_CODE,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
DATA_SOURCE,
INST_CODE,
CMPS_FLAG,
DEPT_NAME,
PRESCRIPTION,
PRESCRIPTION_LY,
NEW_CODE,
AREA,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TF_EXT_XIEHE_SALES
;

458
UNIONALL/DM_TF_EXT_UNIONALL_SALES_MAPPING.sql Normal file
View File

@@ -0,0 +1,458 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TF_EXT_UNIONALL_SALES_MAPPING
-- (
-- MARKET STRING,
-- KEY_COMPETITOR STRING,
-- CLASS STRING,
-- YYYYMM STRING,
-- PACK_COD STRING,
-- CORP_COD STRING,
-- AUDIT_COD STRING,
-- PLATFORM_TYPE STRING,
-- STORE_NAME STRING,
-- STORE_TYPE STRING,
-- REGION_TYPE STRING,
-- DATA_SOURCE STRING,
-- PACK_FLAG INT,
-- PROD_FLAG INT,
-- DTP_FLAG INT,
-- CMPS_FLAG INT,
-- NEW_CODE STRING COMMENT '主数据关联CODE',
-- INST_CODE STRING COMMENT '内部机构编码',
-- AIA_HP_FLAG INT,
-- DEPT_NAME STRING COMMENT '科室名称',
-- H_LEVEL INT,
-- REIMBURSE STRING COMMENT '报销情况',
-- REIMBURSE_TYPE STRING COMMENT '报销类型',
-- PRESCRIPTION_TYPE STRING COMMENT '处方来源',
-- VA decimal(35,10),
-- UT decimal(35,10),
-- CU decimal(35,10),
-- PT decimal(35,10),
-- SALES_UNIT_CAL decimal(35,10),
-- SALES_UNIT_CAL_LY decimal(35,10),
-- SALES_VALUE_CAL decimal(35,10),
-- SALES_VALUE_CAL_LY decimal(35,10),
-- CONUTING_UNIT decimal(35,10),
-- CONUTING_UNIT_LY decimal(35,10),
-- PDOT decimal(35,10),
-- PDOT_LY decimal(35,10),
-- PRESCRIPTION INT,
-- PRESCRIPTION_LY INT,
-- AIA_REMOVE INT,
-- MERGEDATA_FLAG STRING,
-- MERGEDATA_FLAG_MKT STRING,
-- MERGEDATA_FLAG_CHPA_DTP INT,
-- MERGEDATA_FLAG_DTP_NOROC INT,
-- MERGED_DATA_DTP_OTHERS INT,
-- TARGET_INS STRING
-- -- ,
-- -- ETL_INSERT_DT TIMESTAMP,
-- -- ETL_UPDATE_DT TIMESTAMP
-- )
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales_mapping';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales_mapping';
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW ALL_EXTERNAL_SALES AS
--DTP
SELECT
T0.MARKET,
T0.KEY_COMPETITOR,
T0.CLASS,
T0.YYYYMM,
T0.PACK_COD,
T0.CORP_COD,
T0.AUDIT_COD,
T0.PLATFORM_TYPE,
T0.STORE_NAME,
T0.STORE_TYPE,
T0.REGION_TYPE,
T0.DATA_SOURCE,
T0.PACK_FLAG,
T0.PROD_FLAG,
T0.DTP_FLAG,
T0.CMPS_FLAG,
T0.NEW_CODE,
T0.INST_CODE,
T0.AIA_HP_FLAG,
T0.DEPT_NAME,
T0.H_LEVEL,
T0.REIMBURSE,
T0.REIMBURSE_TYPE,
T0.PRESCRIPTION_TYPE,
T0.VA,
T0.UT,
T0.CU,
T0.PT,
T0.SALES_UNIT_CAL,
T0.SALES_UNIT_CAL_LY,
T0.SALES_VALUE_CAL,
T0.SALES_VALUE_CAL_LY,
T0.CONUTING_UNIT,
T0.CONUTING_UNIT_LY,
T0.PDOT,
T0.PDOT_LY,
T0.PRESCRIPTION,
T0.PRESCRIPTION_LY,
1 MERGED_DATA_DTP_OTHERS
FROM
DM.DM_TF_EXT_UNIONALL_MARKET_SALES AS T0
WHERE
T0.DATA_SOURCE = 'DTP(Quarterly)'
--Retail
UNION ALL
SELECT
T0.MARKET,
T0.KEY_COMPETITOR,
T0.CLASS,
T0.YYYYMM,
T0.PACK_COD,
T0.CORP_COD,
T0.AUDIT_COD,
T0.PLATFORM_TYPE,
T0.STORE_NAME,
T0.STORE_TYPE,
T0.REGION_TYPE,
T0.DATA_SOURCE,
T0.PACK_FLAG,
T0.PROD_FLAG,
T0.DTP_FLAG,
T0.CMPS_FLAG,
T0.NEW_CODE,
T0.INST_CODE,
T0.AIA_HP_FLAG,
T0.DEPT_NAME,
T0.H_LEVEL,
T0.REIMBURSE,
T0.REIMBURSE_TYPE,
T0.PRESCRIPTION_TYPE,
T0.VA,
T0.UT,
T0.CU,
T0.PT,
T0.SALES_UNIT_CAL,
T0.SALES_UNIT_CAL_LY,
T0.SALES_VALUE_CAL,
T0.SALES_VALUE_CAL_LY,
T0.CONUTING_UNIT,
T0.CONUTING_UNIT_LY,
T0.PDOT,
T0.PDOT_LY,
T0.PRESCRIPTION,
T0.PRESCRIPTION_LY,
1 MERGED_DATA_DTP_OTHERS
FROM
DM.DM_TF_EXT_UNIONALL_MARKET_SALES AS T0
LEFT JOIN (
SELECT
OTHERS_MONTH
FROM
tmp.tmp_dm_td_ext_retail_niad_month
) T1 --Retail取非niad先来数据部分
WHERE
T0.DATA_SOURCE = 'Retail(Quarterly)'
AND SUBSTR(T0.YYYYMM, 5, 2) IN ('03', '06', '09', '12')
--EC
UNION ALL
SELECT
T0.MARKET,
T0.KEY_COMPETITOR,
T0.CLASS,
T0.YYYYMM,
T0.PACK_COD,
T0.CORP_COD,
T0.AUDIT_COD,
T0.PLATFORM_TYPE,
T0.STORE_NAME,
T0.STORE_TYPE,
T0.REGION_TYPE,
T0.DATA_SOURCE,
T0.PACK_FLAG,
T0.PROD_FLAG,
T0.DTP_FLAG,
T0.CMPS_FLAG,
T0.NEW_CODE,
T0.INST_CODE,
T0.AIA_HP_FLAG,
T0.DEPT_NAME,
T0.H_LEVEL,
T0.REIMBURSE,
T0.REIMBURSE_TYPE,
T0.PRESCRIPTION_TYPE,
T0.VA,
T0.UT,
T0.CU,
T0.PT,
T0.SALES_UNIT_CAL,
T0.SALES_UNIT_CAL_LY,
T0.SALES_VALUE_CAL,
T0.SALES_VALUE_CAL_LY,
T0.CONUTING_UNIT,
T0.CONUTING_UNIT_LY,
T0.PDOT,
T0.PDOT_LY,
T0.PRESCRIPTION,
T0.PRESCRIPTION_LY,
1 MERGED_DATA_DTP_OTHERS
FROM
DM.DM_TF_EXT_UNIONALL_MARKET_SALES AS T0
LEFT JOIN (
SELECT
OTHERS_MONTH
FROM
tmp.tmp_dm_td_ext_ec_niad_month
) T1 --EC取非niad先来数据部分
WHERE
T0.DATA_SOURCE = 'EC(Monthly)'
AND SUBSTR(T0.YYYYMM, 5, 2) IN ('03', '06', '09', '12')
--OTHERS DATA SOURCE
UNION ALL
SELECT
T0.MARKET,
T0.KEY_COMPETITOR,
T0.CLASS,
T0.YYYYMM,
T0.PACK_COD,
T0.CORP_COD,
T0.AUDIT_COD,
T0.PLATFORM_TYPE,
T0.STORE_NAME,
T0.STORE_TYPE,
T0.REGION_TYPE,
T0.DATA_SOURCE,
T0.PACK_FLAG,
T0.PROD_FLAG,
T0.DTP_FLAG,
T0.CMPS_FLAG,
T0.NEW_CODE,
T0.INST_CODE,
T0.AIA_HP_FLAG,
T0.DEPT_NAME,
T0.H_LEVEL,
T0.REIMBURSE,
T0.REIMBURSE_TYPE,
T0.PRESCRIPTION_TYPE,
T0.VA,
T0.UT,
T0.CU,
T0.PT,
T0.SALES_UNIT_CAL,
T0.SALES_UNIT_CAL_LY,
T0.SALES_VALUE_CAL,
T0.SALES_VALUE_CAL_LY,
T0.CONUTING_UNIT,
T0.CONUTING_UNIT_LY,
T0.PDOT,
T0.PDOT_LY,
T0.PRESCRIPTION,
T0.PRESCRIPTION_LY,
1 MERGED_DATA_DTP_OTHERS
FROM
DM.DM_TF_EXT_UNIONALL_MARKET_SALES AS T0
WHERE
T0.DATA_SOURCE NOT IN ('Retail(Quarterly)', 'EC(Monthly)', 'DTP(Quarterly)')
-- COMMAND ----------
create or replace table tmp.tmp_tf_ext_aia_target_inst
-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/TMP/tmp_tf_ext_aia_target_inst';
-- 上面是生产环境location，下面是测试环境location
LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/TMP/tmp_tf_ext_aia_target_inst'
as
select distinct
MARKET,
AUDIT_COD
from
dm.dm_td_ext_aia_org
where
MR_TRTY_CODE <> 'Vacant'
UNION ALL
SELECT distinct
'ALL Market',
AUDIT_COD
from
dm.dm_td_ext_aia_org
where
MR_TRTY_CODE <> 'Vacant'
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW DM_TF_EXT_UNIONALL_SALES_MAPPING
AS
SELECT
A.MARKET,
A.KEY_COMPETITOR,
A.CLASS,
A.YYYYMM,
A.PACK_COD AS PACK_COD,
A.CORP_COD AS CORP_COD,
A.AUDIT_COD AS AUDIT_COD,
A.PLATFORM_TYPE,
A.STORE_NAME,
A.STORE_TYPE,
A.REGION_TYPE,
A.DATA_SOURCE,
A.PACK_FLAG,
A.PROD_FLAG,
A.DTP_FLAG,
INT(A.CMPS_FLAG) AS CMPS_FLAG,
A.NEW_CODE,
A.INST_CODE,
B.AIA_HP_FLAG,
A.DEPT_NAME,
ROUND(A.H_LEVEL, 0) AS H_LEVEL,
A.REIMBURSE,
A.REIMBURSE_TYPE,
A.PRESCRIPTION_TYPE,
A.VA,
A.UT,
A.CU,
A.PT,
CASE
WHEN A.SALES_UNIT_CAL = 0 THEN NULL
ELSE CAST(A.SALES_UNIT_CAL AS DECIMAL(35, 10))
END AS SALES_UNIT_CAL,
CASE
WHEN A.SALES_UNIT_CAL_LY = 0 THEN NULL
ELSE CAST(A.SALES_UNIT_CAL_LY AS DECIMAL(35, 10))
END AS SALES_UNIT_CAL_LY,
CASE
WHEN A.SALES_VALUE_CAL = 0 THEN NULL
ELSE CAST(A.SALES_VALUE_CAL AS DECIMAL(35, 10))
END AS SALES_VALUE_CAL,
CASE
WHEN A.SALES_VALUE_CAL_LY = 0 THEN NULL
ELSE CAST(A.SALES_VALUE_CAL_LY AS DECIMAL(35, 10))
END AS SALES_VALUE_CAL_LY,
CASE
WHEN A.CONUTING_UNIT = 0 THEN NULL
ELSE CAST(A.CONUTING_UNIT AS DECIMAL(35, 10))
END AS CONUTING_UNIT,
CASE
WHEN A.CONUTING_UNIT_LY = 0 THEN NULL
ELSE CAST(A.CONUTING_UNIT_LY AS DECIMAL(35, 10))
END AS CONUTING_UNIT_LY,
CASE
WHEN A.PDOT = 0 THEN NULL
ELSE CAST(A.PDOT AS DECIMAL(35, 10))
END AS PDOT,
CASE
WHEN A.PDOT_LY = 0 THEN NULL
ELSE CAST(A.PDOT_LY AS DECIMAL(35, 10))
END AS PDOT_LY,
CASE
WHEN A.PRESCRIPTION = 0 THEN NULL
ELSE CAST(A.PRESCRIPTION AS DECIMAL(35, 10))
END AS PRESCRIPTION,
CASE
WHEN A.PRESCRIPTION_LY = 0 THEN NULL
ELSE CAST(A.PRESCRIPTION_LY AS DECIMAL(35, 10))
END PRESCRIPTION_LY
--AIA remove ins which is wrong
,
IF (D.ins_code IS NULL, 1, 0) AS AIA_REMOVE
--Merged Data part
,
null as MERGEDATA_FLAG
,
CASE
when A.DATA_SOURCE='IQVIA-CHPA(Monthly)' and A.AUDIT_COD = 'ROC' THEN 'CHPA'
WHEN A.DTP_FLAG = '1' THEN 'DTP'
WHEN F.SOURCENAME IS NULL THEN 'Normal'
ELSE F.SOURCENAME
END AS MERGEDATA_FLAG_MKT --mark ins and pack by markettable
,
CASE
WHEN A.DTP_FLAG = '1' THEN 1
WHEN A.DATA_SOURCE = 'IQVIA-CHPA(Monthly)' THEN 1
ELSE 0
END AS MERGEDATA_FLAG_CHPA_DTP --mark chap and dtp data
,
CASE
WHEN A.DTP_FLAG = '1'
AND A.AUDIT_COD = 'ROC' THEN 0
ELSE 1
END AS MERGEDATA_FLAG_DTP_NOROC --mark dtp roc with 0
,
CASE WHEN A.DATA_SOURCE<>'AIA(Monthly)' THEN 'Y'
WHEN G.AUDIT_COD IS NULL THEN 'N'
ELSE 'Y'
END AS TARGET_INS
FROM
ALL_EXTERNAL_SALES AS A
LEFT JOIN DM.DM_AIA_HP_FLAG AS B ON A.INST_CODE = B.INST_CODE
AND A.DATA_SOURCE = B.DATA_SOURCE
LEFT JOIN dm.dm_ext_aia_data_remove_flag D ON A.market = D.market
AND A.audit_cod = D.ins_code
LEFT JOIN dws.dws_td_ext_merge_data_config F ON A.DATA_SOURCE = F.data_source
AND A.AUDIT_COD = F.audit_cod
AND A.MARKET = F.market
LEFT JOIN tmp.tmp_tf_ext_aia_target_inst G ON
A.MARKET = G.MARKET AND A.AUDIT_COD = G.AUDIT_COD AND A.DATA_SOURCE ='AIA(Monthly)'
-- COMMAND ----------
-- delete from dm.DM_TF_EXT_UNIONALL_SALES_MAPPING where DATA_SOURCE IN ('Retail(Quarterly)', 'EC(Monthly)')
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TF_EXT_UNIONALL_SALES_MAPPING
-- INSERT INTO TABLE DM.DM_TF_EXT_UNIONALL_SALES_MAPPING
SELECT
MARKET,
KEY_COMPETITOR,
CLASS,
YYYYMM,
PACK_COD,
CORP_COD,
AUDIT_COD,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
DATA_SOURCE,
CAST(PACK_FLAG AS INT) AS PACK_FLAG,
CAST(PROD_FLAG AS INT) AS PROD_FLAG,
CAST(DTP_FLAG AS INT) AS DTP_FLAG,
CAST(CMPS_FLAG AS INT) AS CMPS_FLAG,
NEW_CODE,
INST_CODE,
CAST(AIA_HP_FLAG AS INT) AS AIA_HP_FLAG,
DEPT_NAME,
CAST(H_LEVEL AS INT) AS H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_TYPE,
CAST(VA AS decimal(35,10)) AS VA,
CAST(UT AS decimal(35,10) ) AS UT,
CAST(CU AS decimal(35,10) ) AS CU,
CAST(PT AS decimal(35,10) ) AS PT,
CAST(SALES_UNIT_CAL AS decimal(35,10) ) AS SALES_UNIT_CAL,
CAST(SALES_UNIT_CAL_LY AS decimal(35,10) ) AS SALES_UNIT_CAL_LY,
CAST(SALES_VALUE_CAL AS decimal(35,10) ) AS SALES_VALUE_CAL,
CAST(SALES_VALUE_CAL_LY AS decimal(35,10) ) AS SALES_VALUE_CAL_LY,
CAST(CONUTING_UNIT AS decimal(35,10) ) AS CONUTING_UNIT,
CAST(CONUTING_UNIT_LY AS decimal(35,10) ) AS CONUTING_UNIT_LY,
CAST(PDOT AS decimal(35,10) ) AS PDOT,
CAST(PDOT_LY AS decimal(35,10) ) AS PDOT_LY,
CAST(PRESCRIPTION AS INT) AS PRESCRIPTION,
CAST(PRESCRIPTION_LY AS INT) AS PRESCRIPTION_LY,
CAST(AIA_REMOVE AS INT) AS AIA_REMOVE,
null MERGEDATA_FLAG,
MERGEDATA_FLAG_MKT MERGEDATA_FLAG_MKT,
CAST(MERGEDATA_FLAG_CHPA_DTP AS INT) AS MERGEDATA_FLAG_CHPA_DTP,
CAST(MERGEDATA_FLAG_DTP_NOROC AS INT) AS MERGEDATA_FLAG_DTP_NOROC,
null AS MERGED_DATA_DTP_OTHERS,
TARGET_INS
-- ,
-- FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
-- FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM
DM_TF_EXT_UNIONALL_SALES_MAPPING
-- WHERE DATA_SOURCE IN ('Retail(Quarterly)', 'EC(Monthly)')

112
UNIONALL/DM_TF_EXT_UNIONALL_SALES_MAPPING_NIAD.sql Normal file
View File

@@ -0,0 +1,112 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TF_EXT_UNIONALL_SALES_MAPPING_NIAD (
-- MARKET STRING,
-- KEY_COMPETITOR STRING,
-- CLASS STRING,
-- YYYYMM STRING,
-- PACK_COD STRING,
-- AUDIT_COD STRING,
-- DATA_SOURCE STRING,
-- VA decimal(35,10),
-- UT decimal(35,10),
-- CU decimal(35,10),
-- PT decimal(35,10),
-- SALES_UNIT_CAL decimal(35,10),
-- SALES_UNIT_CAL_LY decimal(35,10),
-- SALES_VALUE_CAL decimal(35,10),
-- SALES_VALUE_CAL_LY decimal(35,10),
-- CONUTING_UNIT decimal(35,10),
-- CONUTING_UNIT_LY decimal(35,10),
-- PDOT decimal(35,10),
-- PDOT_LY decimal(35,10),
-- PRESCRIPTION INT,
-- PRESCRIPTION_LY INT,
-- PACK_SOURCE STRING,
-- AUDIT_SOURCE STRING,
-- MARKET_SOURCE STRING,
-- CLASS_SOURCE STRING,
-- DATE_KEY DATE,
-- PLATFORM STRING,
-- STORE_NAME STRING,
-- STORE_TYPE STRING)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales_mapping_niad';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales_mapping_niad';
-- COMMAND ----------
-- DBTITLE 1,NIAD单独页面
-- ****************************************************
-- 功能说明：提取NIAD（非胰岛素类抗糖尿病药物）市场的销售数据并创建映射表
-- 数据源：EC(Monthly)月度电商数据 + Retail(Quarterly)季度零售数据
-- 创建日期：2026-04-02
-- 修改日期：2026-04-02
-- ****************************************************
INSERT OVERWRITE DM.DM_TF_EXT_UNIONALL_SALES_MAPPING_NIAD
WITH
-- CTE 1：获取NIAD市场的产品编码集合
-- 注：直接下推双重过滤条件，减少扫描数据量
niad_pack_cods AS (
SELECT DISTINCT PACK_COD
FROM DM.DM_TF_EXT_UNIONALL_MARKET_SALES
WHERE MARKET = 'NIAD (Non-Insulin Anti-Diabetic) Market'
AND DATA_SOURCE IN ('EC(Monthly)', 'Retail(Quarterly)')
)
-- 主查询：提取所有关联市场的完整销售数据
-- BROADCAST 小表 niad_markets，避免大表 shuffle
SELECT /*+ BROADCAST(m) */
-- 业务维度
t.MARKET,
t.KEY_COMPETITOR,
t.CLASS,
t.YYYYMM,
-- 产品标识
t.PACK_COD,
t.AUDIT_COD,
-- 数据来源
t.DATA_SOURCE,
-- 调整系数
t.VA, -- Value Adjustment
t.UT, -- Unit Adjustment
t.CU, -- Counting Unit Adjustment
t.PT, -- PDOT Adjustment
-- 当期销售指标
t.SALES_UNIT_CAL,
t.SALES_VALUE_CAL,
t.CONUTING_UNIT, -- 注：字段名原始拼写保持不变
t.PDOT,
-- 去年同期指标
t.SALES_UNIT_CAL_LY,
t.SALES_VALUE_CAL_LY,
t.CONUTING_UNIT_LY,
t.PDOT_LY,
-- 处方指标
t.PRESCRIPTION,
t.PRESCRIPTION_LY,
-- 衍生组合键
CONCAT(t.PACK_COD, t.DATA_SOURCE) AS PACK_SOURCE,
CONCAT(t.AUDIT_COD, t.DATA_SOURCE) AS AUDIT_SOURCE,
CONCAT(t.MARKET, t.DATA_SOURCE) AS MARKET_SOURCE,
CONCAT(t.MARKET,t.CLASS, t.DATA_SOURCE) AS CLASS_SOURCE,
-- 时间维度：优化为 TO_DATE 直接解析 yyyyMM 格式，避免字符串拼接
TO_DATE(t.YYYYMM, 'yyyyMM') AS DATE_KEY,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE
FROM DM.DM_TF_EXT_UNIONALL_MARKET_SALES t
LEFT SEMI JOIN niad_pack_cods m ON t.PACK_COD = m.PACK_COD
WHERE t.DATA_SOURCE IN ('EC(Monthly)', 'Retail(Quarterly)')