chenwu/MarketAnalysis-ETL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260512chenwu最终版

Browse Source
This commit is contained in:
zhuchenwu@chenwuzhu.cn
2026-05-12 10:52:14 +00:00
parent 23755b4e1e
commit e1a1176ba5
3 changed files with 65 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
EC/03 ec_load_data.ipynb
View File

@@ -553,12 +553,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778479032113, "finishTime": 1778557198314,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "339d06b1-1c67-4249-a105-aaafb352bb38", "nuid": "339d06b1-1c67-4249-a105-aaafb352bb38",
"showTitle": false, "showTitle": false,
"startTime": 1778479009604, "startTime": 1778557189165,
"submitTime": 1778478972063, "submitTime": 1778557188224,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -574,50 +574,7 @@
" sum(sales_value) as sales_value,\n", " sum(sales_value) as sales_value,\n",
" sum(sales_unit) as sales_unit,\n", " sum(sales_unit) as sales_unit,\n",
" sum(counting_unit) as counting_unit\n", " sum(counting_unit) as counting_unit\n",
"from (\n", "from tmp.tmp_ec_source t1\n",
" -- NIAD: 保持月度粒度，保留 store_name\n",
" select\n",
" a.yyyymm,a.pack_id,a.iqvia_pack_code,a.platform,a.store_name,a.store_type,\n",
" a.sales_value,a.sales_unit,a.counting_unit\n",
" from tmp.tmp_ec_source a\n",
" inner join (\n",
" select distinct iqvia_pack_code\n",
" from tmp.tmp_ec_temp_market\n",
" where upper(market) = 'NIAD'\n",
" ) m on a.iqvia_pack_code = m.iqvia_pack_code\n",
"\n",
" union all\n",
"\n",
" -- 非 NIAD: 聚合为季度（季末月），store_name 置空\n",
" select\n",
" cast(\n",
" case when cast(substr(cast(a.yyyymm as string), 5, 2) as int) <= 3 then concat(substr(cast(a.yyyymm as string), 1, 4), '03')\n",
" when cast(substr(cast(a.yyyymm as string), 5, 2) as int) <= 6 then concat(substr(cast(a.yyyymm as string), 1, 4), '06')\n",
" when cast(substr(cast(a.yyyymm as string), 5, 2) as int) <= 9 then concat(substr(cast(a.yyyymm as string), 1, 4), '09')\n",
" else concat(substr(cast(a.yyyymm as string), 1, 4), '12')\n",
" end as int\n",
" ) as yyyymm,\n",
" first_value(a.pack_id) as pack_id,\n",
" a.iqvia_pack_code,a.platform,\n",
" a.store_name,\n",
" a.store_type,\n",
" sum(a.sales_value) as sales_value,\n",
" sum(a.sales_unit) as sales_unit,\n",
" sum(a.counting_unit) as counting_unit\n",
" from tmp.tmp_ec_source a\n",
" inner join (\n",
" select distinct iqvia_pack_code\n",
" from tmp.tmp_ec_temp_market\n",
" where upper(market) <> 'NIAD'\n",
" ) m on a.iqvia_pack_code = m.iqvia_pack_code\n",
" group by\n",
" case when cast(substr(cast(a.yyyymm as string), 5, 2) as int) <= 3 then concat(substr(cast(a.yyyymm as string), 1, 4), '03')\n",
" when cast(substr(cast(a.yyyymm as string), 5, 2) as int) <= 6 then concat(substr(cast(a.yyyymm as string), 1, 4), '06')\n",
" when cast(substr(cast(a.yyyymm as string), 5, 2) as int) <= 9 then concat(substr(cast(a.yyyymm as string), 1, 4), '09')\n",
" else concat(substr(cast(a.yyyymm as string), 1, 4), '12')\n",
" end,\n",
" a.iqvia_pack_code,a.platform,a.store_name,a.store_type\n",
") t1\n",
"group by\n", "group by\n",
" t1.yyyymm,t1.pack_id,t1.iqvia_pack_code,t1.platform,t1.store_name,t1.store_type" " t1.yyyymm,t1.pack_id,t1.iqvia_pack_code,t1.platform,t1.store_name,t1.store_type"
] ]
@@ -632,12 +589,12 @@
"rowLimit": 10000 "rowLimit": 10000
}, },
"collapsed": true, "collapsed": true,
"finishTime": 1778479032120, "finishTime": 1778557198321,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "c686ec31-8c18-4a4a-96ca-ce5a29f2f676", "nuid": "c686ec31-8c18-4a4a-96ca-ce5a29f2f676",
"showTitle": false, "showTitle": false,
"startTime": 1778479032117, "startTime": 1778557198319,
"submitTime": 1778478972070, "submitTime": 1778557188234,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -686,12 +643,12 @@
"rowLimit": 10000 "rowLimit": 10000
}, },
"collapsed": true, "collapsed": true,
"finishTime": 1778479032125, "finishTime": 1778557198329,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "7814d185-5883-4f00-b350-0d5cab1da7f5", "nuid": "7814d185-5883-4f00-b350-0d5cab1da7f5",
"showTitle": false, "showTitle": false,
"startTime": 1778479032122, "startTime": 1778557198326,
"submitTime": 1778478972075, "submitTime": 1778557188239,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -737,12 +694,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778479036048, "finishTime": 1778557203672,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "b576e313-2815-4991-a92e-82fe68cbaed9", "nuid": "b576e313-2815-4991-a92e-82fe68cbaed9",
"showTitle": false, "showTitle": false,
"startTime": 1778479032127, "startTime": 1778557198332,
"submitTime": 1778478972079, "submitTime": 1778557188244,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -781,12 +738,12 @@
"rowLimit": 10000 "rowLimit": 10000
}, },
"collapsed": true, "collapsed": true,
"finishTime": 1778479036056, "finishTime": 1778557203681,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "5fdbf78f-228d-4e26-94b8-94565feb2ddf", "nuid": "5fdbf78f-228d-4e26-94b8-94565feb2ddf",
"showTitle": false, "showTitle": false,
"startTime": 1778479036052, "startTime": 1778557203677,
"submitTime": 1778478972087, "submitTime": 1778557188250,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -973,12 +930,12 @@
"rowLimit": 10000 "rowLimit": 10000
}, },
"collapsed": true, "collapsed": true,
"finishTime": 1778479036062, "finishTime": 1778557203688,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "b1f69228-3a47-4a76-ab21-a10e52305a56", "nuid": "b1f69228-3a47-4a76-ab21-a10e52305a56",
"showTitle": false, "showTitle": false,
"startTime": 1778479036059, "startTime": 1778557203685,
"submitTime": 1778478972095, "submitTime": 1778557188256,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1097,12 +1054,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778490270242, "finishTime": 1778557219728,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "2c5737c9-b8fe-4b8a-a4b6-f39be69af55c", "nuid": "2c5737c9-b8fe-4b8a-a4b6-f39be69af55c",
"showTitle": false, "showTitle": false,
"startTime": 1778490255038, "startTime": 1778557203693,
"submitTime": 1778490254902, "submitTime": 1778557188260,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1264,12 +1221,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778490290797, "finishTime": 1778557241785,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "9b43d6a9-6fd4-4568-bc48-7d7cb7ffe266", "nuid": "9b43d6a9-6fd4-4568-bc48-7d7cb7ffe266",
"showTitle": false, "showTitle": false,
"startTime": 1778490270248, "startTime": 1778557219733,
"submitTime": 1778490266491, "submitTime": 1778557188268,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1360,12 +1317,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778490296579, "finishTime": 1778557246541,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "ebb41c4f-a6d7-4dbd-bdb1-304f0c5abae4", "nuid": "ebb41c4f-a6d7-4dbd-bdb1-304f0c5abae4",
"showTitle": false, "showTitle": false,
"startTime": 1778490290806, "startTime": 1778557241789,
"submitTime": 1778490266501, "submitTime": 1778557188277,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1473,12 +1430,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778490302260, "finishTime": 1778557250987,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "25bd255c-597c-4bb0-9c7a-0577e28eefee", "nuid": "25bd255c-597c-4bb0-9c7a-0577e28eefee",
"showTitle": false, "showTitle": false,
"startTime": 1778490296584, "startTime": 1778557246546,
"submitTime": 1778490266506, "submitTime": 1778557188283,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1586,12 +1543,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778490321928, "finishTime": 1778557268253,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "d13a4ab2-9a37-4ac4-a886-3cf57cc9d41a", "nuid": "d13a4ab2-9a37-4ac4-a886-3cf57cc9d41a",
"showTitle": false, "showTitle": false,
"startTime": 1778490302264, "startTime": 1778557250991,
"submitTime": 1778490266512, "submitTime": 1778557188292,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1738,12 +1695,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778490326046, "finishTime": 1778557271279,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "8c861c10-773a-4449-a5dc-c0340e5ec97e", "nuid": "8c861c10-773a-4449-a5dc-c0340e5ec97e",
"showTitle": false, "showTitle": false,
"startTime": 1778490321933, "startTime": 1778557268257,
"submitTime": 1778490266518, "submitTime": 1778557188299,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1765,12 +1722,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778490330473, "finishTime": 1778557275535,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "e39d8c66-7469-46db-a2f4-79976266bec2", "nuid": "e39d8c66-7469-46db-a2f4-79976266bec2",
"showTitle": false, "showTitle": false,
"startTime": 1778490326057, "startTime": 1778557271284,
"submitTime": 1778490266523, "submitTime": 1778557188306,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1832,12 +1789,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778490333948, "finishTime": 1778557278550,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "bbe40a04-e787-4133-9901-5f331baf912a", "nuid": "bbe40a04-e787-4133-9901-5f331baf912a",
"showTitle": false, "showTitle": false,
"startTime": 1778490330478, "startTime": 1778557275539,
"submitTime": 1778490266529, "submitTime": 1778557188311,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1863,12 +1820,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778490333956, "finishTime": 1778557278560,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "f8d0befe-3de4-48a8-ac53-cdc19c897d0f", "nuid": "f8d0befe-3de4-48a8-ac53-cdc19c897d0f",
"showTitle": false, "showTitle": false,
"startTime": 1778490333953, "startTime": 1778557278555,
"submitTime": 1778490266536, "submitTime": 1778557188317,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -2118,12 +2075,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778490340199, "finishTime": 1778557283201,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "9e6100db-d97c-41a6-8b30-4f7a774c1903", "nuid": "9e6100db-d97c-41a6-8b30-4f7a774c1903",
"showTitle": false, "showTitle": false,
"startTime": 1778490333962, "startTime": 1778557278565,
"submitTime": 1778490266542, "submitTime": 1778557188322,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -2310,12 +2267,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778490344022, "finishTime": 1778557285684,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "0c0fe2ec-f80e-4872-85bf-fd5d058cc8b8", "nuid": "0c0fe2ec-f80e-4872-85bf-fd5d058cc8b8",
"showTitle": false, "showTitle": false,
"startTime": 1778490340204, "startTime": 1778557283205,
"submitTime": 1778490266549, "submitTime": 1778557188328,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -2403,12 +2360,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778490351847, "finishTime": 1778557292166,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "6b591466-5af8-40ef-a9b9-c4f137843f18", "nuid": "6b591466-5af8-40ef-a9b9-c4f137843f18",
"showTitle": false, "showTitle": false,
"startTime": 1778490344026, "startTime": 1778557285688,
"submitTime": 1778490266564, "submitTime": 1778557188343,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -2516,12 +2473,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778490352755, "finishTime": 1778557292715,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "20f3c0a6-f626-4e64-bce4-7653087adb16", "nuid": "20f3c0a6-f626-4e64-bce4-7653087adb16",
"showTitle": false, "showTitle": false,
"startTime": 1778490351852, "startTime": 1778557292171,
"submitTime": 1778490266570, "submitTime": 1778557188353,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }

8
UNIONALL/DM_TF_EXT_UNIONALL_MARKET_SALES.sql
View File

@@ -63,7 +63,7 @@
-- COMMAND ---------- -- COMMAND ----------
-- DBTITLE 1,测试跑数据 -- DBTITLE 1,测试跑数据
-- delete from DM.DM_TF_EXT_UNIONALL_MARKET_SALES where DATA_SOURCE IN ('Retail(Quarterly)', 'EC(Monthly)') -- delete from DM.DM_TF_EXT_UNIONALL_MARKET_SALES where DATA_SOURCE IN ('Retail(Quarterly)') --,
-- COMMAND ---------- -- COMMAND ----------
@@ -83,9 +83,9 @@
-- -- 'CHC(Quarterly)', -- -- 'CHC(Quarterly)',
-- -- 'IQVIA-COUNTY(Quarterly)', -- -- 'IQVIA-COUNTY(Quarterly)',
-- -- 'THC(Quarterly)' -- -- 'THC(Quarterly)'
-- 'Retail(Quarterly)', -- 'Retail(Quarterly)'
-- -- 'DTP(Quarterly)', -- -- ,'DTP(Quarterly)',
-- 'EC(Monthly)' -- -- 'EC(Monthly)'
-- -- 'XH Data(Quarterly)' -- -- 'XH Data(Quarterly)'
-- ) -- 谓词下推，最先过滤 -- ) -- 谓词下推，最先过滤
-- ), -- ),

5
UNIONALL/DM_TF_EXT_UNIONALL_SALES_MAPPING.sql
View File

@@ -478,7 +478,7 @@ FROM
-- COMMAND ---------- -- COMMAND ----------
-- delete from dm.DM_TF_EXT_UNIONALL_SALES_MAPPING where DATA_SOURCE IN ('Retail(Quarterly)', 'EC(Monthly)') -- delete from dm.DM_TF_EXT_UNIONALL_SALES_MAPPING where DATA_SOURCE IN ( 'AIA(Monthly)')--'Retail(Quarterly)','EC(Monthly)'
-- COMMAND ---------- -- COMMAND ----------
@@ -536,4 +536,5 @@ SELECT
FROM FROM
DM_TF_EXT_UNIONALL_SALES_MAPPING DM_TF_EXT_UNIONALL_SALES_MAPPING
-- WHERE DATA_SOURCE IN ('Retail(Quarterly)', 'EC(Monthly)') -- WHERE DATA_SOURCE IN ( 'AIA(Monthly)')--'',
-- AND yyyymm>=202501