chenwu/MarketAnalysis-ETL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260511chenwu定版

Browse Source 
ec的store name 修复
retail的niad直取数据修复，niadfree report 页面的表直接拿dwd数据
nida free report页面数据错列问题修复
This commit is contained in:
zhuchenwu@chenwuzhu.cn
2026-05-11 15:18:43 +00:00
parent 4689311dc0
commit db99577102
7 changed files with 91 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
EC/03 ec_load_data.ipynb
View File

@@ -1097,12 +1097,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778479043473,
"finishTime": 1778490270242,
"inputWidgets": {},
"nuid": "2c5737c9-b8fe-4b8a-a4b6-f39be69af55c",
"showTitle": false,
"startTime": 1778479036064,
"submitTime": 1778478972102,
"startTime": 1778490255038,
"submitTime": 1778490254902,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1123,7 +1123,7 @@
" 'ROC' as audit_cod,\n",
" t1.platform,\n",
" case when t1.store_type = '第三方' then 'POP' when t1.store_type in ('平台店','品牌店','自营旗舰') then 'SO' end as store_type,\n",
" '' as store_name,\n",
" store_name,\n",
" 1 as PACK_FLAG,\n",
" 1 as PROD_FLAG,\n",
" '' as flag,\n",
@@ -1136,7 +1136,7 @@
" select distinct iqvia_pack_code, corp_cod\n",
" from tmp.tmp_zk_ec_pack_property_corp\n",
") t2 on t1.iqvia_pack_code = t2.iqvia_pack_code\n",
"group by t1.yyyymm, t1.pack_id, t1.iqvia_pack_code, t2.corp_cod, t1.platform, t1.store_type\n",
"group by t1.yyyymm, t1.pack_id, t1.iqvia_pack_code, t2.corp_cod, t1.platform, t1.store_type,t1.store_name\n",
"\n",
"-- ========== 原始倒减逻辑（已注释） ==========\n",
"-- insert overwrite table tmp.tmp_dm_zk_ec_sales \n",
@@ -1264,12 +1264,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778479050432,
"finishTime": 1778490290797,
"inputWidgets": {},
"nuid": "9b43d6a9-6fd4-4568-bc48-7d7cb7ffe266",
"showTitle": false,
"startTime": 1778479043477,
"submitTime": 1778478972108,
"startTime": 1778490270248,
"submitTime": 1778490266491,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1360,12 +1360,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778479055876,
"finishTime": 1778490296579,
"inputWidgets": {},
"nuid": "ebb41c4f-a6d7-4dbd-bdb1-304f0c5abae4",
"showTitle": false,
"startTime": 1778479050436,
"submitTime": 1778478972114,
"startTime": 1778490290806,
"submitTime": 1778490266501,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1473,12 +1473,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778479060300,
"finishTime": 1778490302260,
"inputWidgets": {},
"nuid": "25bd255c-597c-4bb0-9c7a-0577e28eefee",
"showTitle": false,
"startTime": 1778479055880,
"submitTime": 1778478972119,
"startTime": 1778490296584,
"submitTime": 1778490266506,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1586,12 +1586,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778479098308,
"finishTime": 1778490321928,
"inputWidgets": {},
"nuid": "d13a4ab2-9a37-4ac4-a886-3cf57cc9d41a",
"showTitle": false,
"startTime": 1778479060304,
"submitTime": 1778478972125,
"startTime": 1778490302264,
"submitTime": 1778490266512,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1738,12 +1738,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778479117659,
"finishTime": 1778490326046,
"inputWidgets": {},
"nuid": "8c861c10-773a-4449-a5dc-c0340e5ec97e",
"showTitle": false,
"startTime": 1778479098312,
"submitTime": 1778478972131,
"startTime": 1778490321933,
"submitTime": 1778490266518,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1765,12 +1765,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778479124771,
"finishTime": 1778490330473,
"inputWidgets": {},
"nuid": "e39d8c66-7469-46db-a2f4-79976266bec2",
"showTitle": false,
"startTime": 1778479117663,
"submitTime": 1778478972137,
"startTime": 1778490326057,
"submitTime": 1778490266523,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1832,12 +1832,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778479127669,
"finishTime": 1778490333948,
"inputWidgets": {},
"nuid": "bbe40a04-e787-4133-9901-5f331baf912a",
"showTitle": false,
"startTime": 1778479124776,
"submitTime": 1778478972142,
"startTime": 1778490330478,
"submitTime": 1778490266529,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1863,12 +1863,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778479127677,
"finishTime": 1778490333956,
"inputWidgets": {},
"nuid": "f8d0befe-3de4-48a8-ac53-cdc19c897d0f",
"showTitle": false,
"startTime": 1778479127673,
"submitTime": 1778478972148,
"startTime": 1778490333953,
"submitTime": 1778490266536,
"tableResultSettingsMap": {},
"title": ""
}
@@ -2118,12 +2118,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778479133948,
"finishTime": 1778490340199,
"inputWidgets": {},
"nuid": "9e6100db-d97c-41a6-8b30-4f7a774c1903",
"showTitle": false,
"startTime": 1778479127685,
"submitTime": 1778478972153,
"startTime": 1778490333962,
"submitTime": 1778490266542,
"tableResultSettingsMap": {},
"title": ""
}
@@ -2310,12 +2310,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778479145677,
"finishTime": 1778490344022,
"inputWidgets": {},
"nuid": "0c0fe2ec-f80e-4872-85bf-fd5d058cc8b8",
"showTitle": false,
"startTime": 1778479133960,
"submitTime": 1778478972160,
"startTime": 1778490340204,
"submitTime": 1778490266549,
"tableResultSettingsMap": {},
"title": ""
}
@@ -2403,12 +2403,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778479158042,
"finishTime": 1778490351847,
"inputWidgets": {},
"nuid": "6b591466-5af8-40ef-a9b9-c4f137843f18",
"showTitle": false,
"startTime": 1778479145698,
"submitTime": 1778478972175,
"startTime": 1778490344026,
"submitTime": 1778490266564,
"tableResultSettingsMap": {},
"title": ""
}
@@ -2516,12 +2516,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778479158640,
"finishTime": 1778490352755,
"inputWidgets": {},
"nuid": "20f3c0a6-f626-4e64-bce4-7653087adb16",
"showTitle": false,
"startTime": 1778479158046,
"submitTime": 1778478972181,
"startTime": 1778490351852,
"submitTime": 1778490266570,
"tableResultSettingsMap": {},
"title": ""
}

31
Retail/01 load_tmp_data.py
View File

@@ -423,12 +423,12 @@ LEVEL_MARKET_RD = 'RD Market'
# MAGIC when right(quarter,2) = 'Q4' THEN '12' END)
# MAGIC AS month,
# MAGIC quarter,
# MAGIC replace(price,',','' ),
# MAGIC replace(sales_unit,',','' ),
# MAGIC replace(sales_value,',','' ),
# MAGIC digital_spread_rate,
# MAGIC weighted_spread_rate,
# MAGIC counting_unit,
# MAGIC CAST ( avg( replace(price,',','' )) AS decimal(20,8) ) price,
# MAGIC CAST ( sum(replace(sales_unit,',','' )) as decimal(20,8)) sales_unit,
# MAGIC CAST ( sum(replace(sales_value,',','' )) as decimal(20,8)) sales_value,
# MAGIC CAST ( avg(digital_spread_rate) AS decimal(20,8)) digital_spread_rate,
# MAGIC CAST ( avg(weighted_spread_rate) as decimal(20,8)) weighted_spread_rate,
# MAGIC CAST ( max(counting_unit) as decimal(20,8)) ,
# MAGIC null,
# MAGIC null,
# MAGIC null,
@@ -443,6 +443,25 @@ LEVEL_MARKET_RD = 'RD Market'
# MAGIC 0,
# MAGIC null
# MAGIC from dwd.dwd_gnd_ext_retail_nataional_niad
# MAGIC GROUP BY
# MAGIC zk_product_id,
# MAGIC zk_region,
# MAGIC zk_rx_otc,
# MAGIC zk_medicine_type,
# MAGIC zk_medicine_tier1,
# MAGIC zk_medicine_tier2,
# MAGIC zk_medicine_tier3,
# MAGIC zk_medicine_tier4,
# MAGIC zk_common_name,
# MAGIC zk_dosage_form,
# MAGIC zk_user_type,
# MAGIC zk_category_name,
# MAGIC zk_product_name,
# MAGIC zk_brand_name,
# MAGIC zk_manu_des,
# MAGIC zk_corp_des,
# MAGIC zk_pack_des,
# MAGIC quarter
# COMMAND ----------

18
Retail/04 map_to_dws_table.ipynb
View File

@@ -9,12 +9,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778484351983,
"finishTime": 1778490019473,
"inputWidgets": {},
"nuid": "4d16488f-0327-4ced-b23f-41f960a90d2f",
"showTitle": false,
"startTime": 1778484351892,
"submitTime": 1778484351680,
"startTime": 1778490019349,
"submitTime": 1778490019146,
"tableResultSettingsMap": {},
"title": ""
}
@@ -35,12 +35,12 @@
"implicitDf": true,
"rowLimit": 10000
},
"finishTime": 1778484383881,
"finishTime": 1778490037824,
"inputWidgets": {},
"nuid": "1168666b-255b-44a3-968b-4156c93dad53",
"showTitle": false,
"startTime": 1778484351988,
"submitTime": 1778484351685,
"startTime": 1778490019478,
"submitTime": 1778490019152,
"tableResultSettingsMap": {
"0": {
"dataGridStateBlob": "{\"version\":1,\"tableState\":{\"columnPinning\":{\"left\":[\"#row_number#\"],\"right\":[]},\"columnSizing\":{},\"columnVisibility\":{}},\"settings\":{\"columns\":{}},\"syncTimestamp\":1777430717962}",
@@ -376,12 +376,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778484383981,
"finishTime": 1778490037922,
"inputWidgets": {},
"nuid": "efe5e7ee-82b5-46d9-85f7-650756dffbf8",
"showTitle": false,
"startTime": 1778484383903,
"submitTime": 1778484351693,
"startTime": 1778490037828,
"submitTime": 1778490019162,
"tableResultSettingsMap": {},
"title": ""
}

2
Retail/13 DM_TF_EXT_RETAIL_SALES.sql
View File

@@ -38,7 +38,7 @@
-- COMMAND ----------
-- CREATE TABLE catalog_e2_ngbi_qas.DM.DM_TF_EXT_RETAIL_NIAD_SALES (
-- CREATE TABLE DM.DM_TF_EXT_RETAIL_NIAD_SALES (
-- YYYYMM STRING,
-- PACK_CODE STRING,
-- CORP_CODE STRING,

1
UNIONALL/DM_TF_EXT_RETAIL_MARKET_SALES_NIAD.sql
View File

@@ -99,6 +99,7 @@ WITH
*
FROM
DM.DM_TF_EXT_RETAIL_NIAD_SALES
WHERE YYYYMM>=202001
),
mapping_cleaned AS (
SELECT

2
UNIONALL/DM_TF_EXT_UNIONALL_SALES_MAPPING.sql
View File

@@ -478,7 +478,7 @@ FROM
-- COMMAND ----------
delete from dm.DM_TF_EXT_UNIONALL_SALES_MAPPING where DATA_SOURCE IN ('Retail(Quarterly)', 'EC(Monthly)')
-- delete from dm.DM_TF_EXT_UNIONALL_SALES_MAPPING where DATA_SOURCE IN ('Retail(Quarterly)', 'EC(Monthly)')
-- COMMAND ----------

24
UNIONALL/DM_TF_EXT_UNIONALL_SALES_MAPPING_NIAD.sql
View File

@@ -83,14 +83,15 @@ SELECT
-- 当期销售指标
t.SALES_UNIT_CAL,
t.SALES_VALUE_CAL,
t.CONUTING_UNIT, -- 注：字段名原始拼写保持不变
t.PDOT,
-- 去年同期指标
t.SALES_UNIT_CAL_LY,
t.SALES_VALUE_CAL,
t.SALES_VALUE_CAL_LY,
t.CONUTING_UNIT, -- 注：字段名原始拼写保持不变
t.CONUTING_UNIT_LY,
t.PDOT,
t.PDOT_LY,
-- 处方指标
@@ -133,7 +134,7 @@ WITH
-- 注：直接下推双重过滤条件，减少扫描数据量
niad_pack_cods AS (
SELECT DISTINCT PACK_COD
FROM DM.DM_TF_EXT_UNIONALL_MARKET_SALES
FROM DM.DM_TF_EXT_RETAIL_MARKET_SALES
WHERE MARKET = 'NIAD (Non-Insulin Anti-Diabetic) Market'
AND DATA_SOURCE IN ('Retail(Quarterly)')
),
@@ -179,14 +180,15 @@ SELECT /*+ BROADCAST(m) */
-- 当期销售指标
t.SALES_UNIT_CAL,
t.SALES_VALUE_CAL,
t.CONUTING_UNIT, -- 注：字段名原始拼写保持不变
t.PDOT,
-- 去年同期指标
t.SALES_UNIT_CAL_LY,
t.SALES_VALUE_CAL,
t.SALES_VALUE_CAL_LY,
t.CONUTING_UNIT, -- 注：字段名原始拼写保持不变
t.CONUTING_UNIT_LY,
t.PDOT,
t.PDOT_LY,
-- 处方指标