chenwu/MarketAnalysis-ETL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

county2026年切修改初稿

Browse Source 
修改geo和pack2market
This commit is contained in:
zhuchenwu@chenwuzhu.cn
2026-05-20 10:11:13 +00:00
parent 2984fe7195
commit 31a612a902
6 changed files with 516 additions and 831 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1105
09 dm_td_external_geo_type.sql
View File

File diff suppressed because it is too large Load Diff

11
AIA/11 DM_TD_EXT_AIA_TARGET_INST.sql
View File

@@ -1,11 +0,0 @@
-- Databricks notebook source
-- MAGIC %run ../../../Common/config
-- COMMAND ----------
-- MAGIC %python
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dim_product_wide`').createOrReplaceTempView('cdw_dwd_dim_product_wide')
-- COMMAND ----------

20
UNIONALL/DM_TD_EXT_UNIONALL_MARKET_PACK_MAPPING.sql
View File

@@ -280,7 +280,7 @@ UNION ALL
SELECT
PACK_CODE,
'ALL Market' AS MARKET,
'Others' AS CLASS,
'OTHERS' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
@@ -309,7 +309,7 @@ UNION ALL
SELECT
PACK_CODE,
'ALL Market' AS MARKET,
'Others' AS CLASS,
'OTHERS' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
@@ -338,7 +338,7 @@ UNION ALL
SELECT
PACK_CODE,
'ALL Market'MARKET,
'Others' AS CLASS,
'OTHERS' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
@@ -367,7 +367,7 @@ UNION ALL
SELECT
PACK_CODE,
'ALL Market'MARKET,
'Others' AS CLASS,
'OTHERS' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
@@ -396,7 +396,7 @@ UNION ALL
SELECT
PACK_CODE,
'ALL Market'MARKET,
'Others' AS CLASS,
'OTHERS' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
@@ -425,7 +425,7 @@ UNION ALL
SELECT
PACK_CODE,
'ALL Market'MARKET,
'Others' AS CLASS,
'OTHERS' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
@@ -454,7 +454,7 @@ UNION ALL
SELECT
PACK_CODE,
'ALL Market'MARKET,
'Others' AS CLASS,
'OTHERS' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
@@ -483,7 +483,7 @@ UNION ALL
SELECT
PACK_CODE,
'ALL Market'MARKET,
'Others' AS CLASS,
'OTHERS' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
@@ -512,7 +512,7 @@ UNION ALL
SELECT
PACK_CODE,
'ALL Market'MARKET,
'Others' AS CLASS,
'OTHERS' AS CLASS,
'OTHERS' AS KEY_COMPETITOR,
1 AS MARKET_RATIO,
200001 AS STARTTIME,
@@ -538,4 +538,4 @@ GROUP BY
-- COMMAND ----------
update DM.DM_TD_EXT_UNIONALL_MARKET_PACK_MAPPING set class = 'Others' where lower(CLASS) = 'others' or CLASS IS null
update DM.DM_TD_EXT_UNIONALL_MARKET_PACK_MAPPING set class = 'OTHERS' where lower(CLASS) = 'others' or CLASS IS null

2
county/04 DM_TD_EXT_COUNTY_PACK_PROPERTY.sql
View File

@@ -75,7 +75,7 @@
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_CHC_PACK_PROPERTY
INSERT OVERWRITE TABLE DM.DM_TD_EXT_COUNTY_PACK_PROPERTY
SELECT
-- MARKET_PACK_KEY,
PACK_COD AS PACK_CODE,

118
county/10 DM_TD_EXT_COUNTY_PACK2MARKET.sql
View File

@@ -45,6 +45,28 @@
-- COMMAND ----------
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_COUNTY_PACK2MARKET_KEYCOMPETITOR (
-- KEY_COMPETITOR STRING,
-- MARKET STRING,
-- PACK_CODE STRING)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_county_pack2market_keycompetitor';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_county_pack2market_keycompetitor';
-- COMMAND ----------
-- CREATE or REPLACE TABLE DM.DM_TD_EXT_COUNTY_PACK2MARKET_CLASS (
-- MARKET STRING,
-- PACK_CODE STRING,
-- CLASS STRING)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_county_pack2market_class';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_county_pack2market_class';
-- COMMAND ----------
-- 使用 EXCEPT（集合差操作）简化逻辑
INSERT OVERWRITE TABLE DM.DM_TD_EXT_COUNTY_PACK2MARKET
-- 包含规则
@@ -206,3 +228,99 @@ LEFT JOIN (
AND (T2.STRENGTH IS NULL OR T1.STGH_DESC = T2.STRENGTH)
AND (T2.MOLECULE_CODE IS NULL OR T1.CMPS_CODE = T2.MOLECULE_CODE)
WHERE T2.MARKET IS NOT NULL;
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_COUNTY_PACK2MARKET_KEYCOMPETITOR
WITH kc AS (
SELECT
keycompetitor, no,
ROW_NUMBER() OVER (ORDER BY
CASE
WHEN ATC1_Code IS NOT NULL THEN 1
WHEN ATC2_Code IS NOT NULL THEN 2
WHEN ATC3_Code IS NOT NULL THEN 3
WHEN ATC4_Code IS NOT NULL THEN 4
WHEN Molecule_Code IS NOT NULL THEN 5
WHEN Product_Code IS NOT NULL THEN 6
WHEN Pack_Code IS NOT NULL THEN 7
ELSE 999
END,
CASE WHEN NFC1_CODE IS NULL THEN 0 ELSE 1 END,
CASE WHEN NFC2_CODE IS NULL THEN 0 ELSE 1 END,
CASE WHEN NFC3_CODE IS NULL THEN 0 ELSE 1 END
) AS no1,
ATC1_Code, ATC2_Code, ATC3_Code, ATC4_Code,
NFC1_Code, NFC2_Code, NFC3_Code,
corporation_code, Manufacturer_Code,
Product_Code, Pack_Code, Strength, Molecule_Code, market
FROM dwd.dwd_gnd_ims_tblkeycompetitor
),
joined AS (
SELECT
/*+ BROADCAST(kc) */
COALESCE(kc.keycompetitor, 'OTHERS') AS Key_Competitor, -- ✅ UPDATE 逻辑内联
t1.market, t1.PACK_CODE
/*bug 调试区
,
kc.no,
kc.no1,
t1.PROD_COD, t1.CMPS_COD, t1.CMPS_DES,
t1.ATC1_COD, t1.ATC2_COD, t1.ATC3_COD, t1.ATC4_COD,
t1.APP1_COD, t1.APP2_COD, t1.APP3_COD,
t1.BIO_DESC, t1.GENE_ORIG_DESC, t1.ETH_OTC_DESC,
t1.NRDL_DESC, t1.NRDL_Entry_Date, t1.EDL_DESC, t1.TCM_DESC,
t1.PAED_DESC, t1.GQCE_DESC, t1.VBP_DESC,
t1.MANU_COD, t1.MANU_DES, t1.MNFL_COD, t1.MNFL_DES,
t1.CORP_COD, t1.CORP_DES, t1.BrandType, t1.bu, t1.Market_Ratio
*/
,ROW_NUMBER() OVER (
PARTITION BY t1.market, t1.PACK_CODE, t1.PROD_CODE
ORDER BY kc.no1 DESC, kc.no DESC
) AS _rn
FROM DM.DM_TD_EXT_COUNTY_PACK2MARKET t1
LEFT JOIN kc
ON (kc.ATC1_Code IS NULL OR t1.ATC1_CODE = kc.ATC1_Code)
AND (kc.ATC2_Code IS NULL OR t1.ATC2_CODE = kc.ATC2_Code)
AND (kc.ATC3_Code IS NULL OR t1.ATC3_CODE = kc.ATC3_Code)
AND (kc.ATC4_Code IS NULL OR t1.ATC4_CODE = kc.ATC4_Code)
AND (kc.NFC1_Code IS NULL OR t1.APP1_CODE = kc.NFC1_Code)
AND (kc.NFC2_Code IS NULL OR t1.APP2_CODE = kc.NFC2_Code)
AND (kc.NFC3_Code IS NULL OR t1.APP3_CODE = kc.NFC3_Code)
AND (kc.corporation_code IS NULL OR t1.CORP_CODE = kc.corporation_code)
AND (kc.Manufacturer_Code IS NULL OR t1.MANU_CODE = kc.Manufacturer_Code)
AND (kc.Product_Code IS NULL OR t1.PROD_CODE = kc.Product_Code)
AND (kc.Pack_Code IS NULL OR t1.PACK_CODE = kc.Pack_Code)
AND (kc.Strength IS NULL OR t1.STGH_DESC = kc.Strength)
AND (kc.Molecule_Code IS NULL OR t1.CMPS_CODE = kc.Molecule_Code)
AND (kc.market IS NULL OR t1.market = kc.market)
)
SELECT distinct Key_Competitor,market,pack_code
FROM joined
WHERE _rn = 1;
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_COUNTY_PACK2MARKET_CLASS
SELECT
/*+ BROADCAST(T2) */
DISTINCT
T1.MARKET,
T1.PACK_CODE,
COALESCE( T2.CLASS, 'OTHERS') AS CLASS
FROM DM.DM_TD_EXT_COUNTY_PACK2MARKET T1
LEFT JOIN DWD.DWD_GND_TBLCLASS T2
ON (T2.MARKET IS NULL OR T1.MARKET = T2.MARKET)
AND (T2.PACK_CODE IS NULL OR T1.PACK_CODE = T2.PACK_CODE)
AND (T2.PRODUCT_CODE IS NULL OR T1.PROD_CODE = T2.PRODUCT_CODE)
AND (T2.MOLECULE_CODE IS NULL OR T1.CMPS_CODE = T2.MOLECULE_CODE)
AND (T2.CORPORATION_CODE IS NULL OR T1.CORP_CODE = T2.CORPORATION_CODE)
AND (T2.MANUFACTURER_CODE IS NULL OR T1.MANU_CODE = T2.MANUFACTURER_CODE)
AND (T2.ATC1_CODE IS NULL OR T1.ATC1_CODE = T2.ATC1_CODE)
AND (T2.ATC2_CODE IS NULL OR T1.ATC2_CODE = T2.ATC2_CODE)
AND (T2.ATC3_CODE IS NULL OR T1.ATC3_CODE = T2.ATC3_CODE)
AND (T2.ATC4_CODE IS NULL OR T1.ATC4_CODE = T2.ATC4_CODE)
AND (T2.NFC1_CODE IS NULL OR T1.APP1_CODE = T2.NFC1_CODE)
AND (T2.NFC2_CODE IS NULL OR T1.APP2_CODE = T2.NFC2_CODE)
AND (T2.NFC3_CODE IS NULL OR T1.APP3_CODE = T2.NFC3_CODE)
AND (T2.STRENGTH IS NULL OR T1.STGH_DESC = T2.STRENGTH)

67
county/13 DM_TD_EXT_COUNTY_MARKET_PACK_MAPPING.sql
View File

@@ -31,32 +31,41 @@
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_COUNTY_MARKET_PACK_MAPPING
SELECT DISTINCT
T1.PACK_CODE,
T1.MARKET,
T1.CLASS,
T1.KEY_COMPETITOR,
T2.MARKET_RATIO,
T2.STARTTIME,
T2.ENDTIME,
T3.VALUE_BRAND_RATIO,
T3.VALUE_BRAND_RATIO_START,
T3.VALUE_BRAND_RATIO_END,
T3.UNIT_BRAND_RATIO,
T3.UNIT_BRAND_START,
T3.UNIT_BRAND_END,
T3.COUNTINGUNIT_BRAND_RATIO,
T3.COUNTINGUNIT_BRAND_START,
T3.COUNTINGUNIT_BRAND_END,
T3.PDOT_BRAND_RATIO,
T3.PDOT_BRAND_START,
T3.PDOT_BRAND_END,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM
DM.DM_TD_EXT_COUNTY_PACK_PROPERTY T1
LEFT JOIN DM.DM_TD_EXT_COUNTY_MARKET_RATIO T2
ON T1.MARKET = T2.MARKET AND T1.PACK_CODE = T2.PACK_CODE
LEFT JOIN DM.DM_TD_EXT_COUNTY_MARKET_BRAND_RATIO T3
ON T1.MARKET = T3.MARKET AND T1.PACK_CODE = T3.PACK_CODE
WHERE UPPER(T1.MARKET) NOT LIKE "%ALL%"
SELECT DISTINCT
T1.PACK_CODE,
T1.MARKET,
T4.CLASS,
T5.KEY_COMPETITOR,
T2.MARKET_RATIO,
T2.STARTTIME,
T2.ENDTIME,
T3.VALUE_BRAND_RATIO,
T3.VALUE_BRAND_RATIO_START,
T3.VALUE_BRAND_RATIO_END,
T3.UNIT_BRAND_RATIO,
T3.UNIT_BRAND_START,
T3.UNIT_BRAND_END,
T3.COUNTINGUNIT_BRAND_RATIO,
T3.COUNTINGUNIT_BRAND_START,
T3.COUNTINGUNIT_BRAND_END,
T3.PDOT_BRAND_RATIO,
T3.PDOT_BRAND_START,
T3.PDOT_BRAND_END,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
FROM
DM.DM_TD_EXT_COUNTY_PACK2MARKET T1
LEFT JOIN DM.DM_TD_EXT_COUNTY_MARKET_RATIO T2
ON T1.MARKET = T2.MARKET
AND T1.PACK_CODE = T2.PACK_CODE
LEFT JOIN DM.DM_TD_EXT_COUNTY_MARKET_BRAND_RATIO T3
ON T1.MARKET = T3.MARKET
AND T1.PACK_CODE = T3.PACK_CODE
LEFT JOIN DM.DM_TD_EXT_COUNTY_PACK2MARKET_CLASS T4
ON T1.PACK_CODE = T4.PACK_CODE
AND T1.MARKET = T4.MARKET
LEFT JOIN DM.DM_TD_EXT_COUNTY_PACK2MARKET_KEYCOMPETITOR T5
ON T1.PACK_CODE = T5.PACK_CODE
AND T1.MARKET = T5.MARKET
WHERE
UPPER(T1.MARKET) NOT LIKE "%ALL%"