chenwu/MarketAnalysis-ETL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260513_上线DBR后版本

Browse Source
This commit is contained in:
chenwu
2026-05-13 12:38:21 +00:00
parent 5d4d312641
commit a9bbe0e95a
18 changed files with 128 additions and 2629 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
11 DM_TD_EXTERNAL_MARKET_NEW.sql
View File

@@ -10,9 +10,9 @@
-- ETL_INSERT_DT TIMESTAMP, -- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP) -- ETL_UPDATE_DT TIMESTAMP)
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market';
-- COMMAND ---------- -- COMMAND ----------
@@ -27,9 +27,9 @@
-- ETL_UPDATE_DT TIMESTAMP -- ETL_UPDATE_DT TIMESTAMP
-- ) -- )
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market_ta'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market_ta';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market_ta'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market_ta';
-- COMMAND ---------- -- COMMAND ----------

12
CHC/04 DM_TD_EXT_CHC_PACK2MARKET.sql
View File

@@ -40,9 +40,9 @@
-- ETL_UPDATE_DT TIMESTAMP -- ETL_UPDATE_DT TIMESTAMP
-- ) -- )
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market';
-- COMMAND ---------- -- COMMAND ----------
@@ -51,9 +51,9 @@
-- MARKET STRING, -- MARKET STRING,
-- PACK_CODE STRING) -- PACK_CODE STRING)
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market_keycompetitor'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market_keycompetitor';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market_keycompetitor'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market_keycompetitor';
-- COMMAND ---------- -- COMMAND ----------
@@ -62,9 +62,9 @@
-- PACK_CODE STRING, -- PACK_CODE STRING,
-- CLASS STRING) -- CLASS STRING)
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market_class'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market_class';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market_class'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market_class';
-- COMMAND ---------- -- COMMAND ----------

1
CHPA/02 tmp_ims_td_prod_tmp.sql
View File

@@ -1,7 +1,6 @@
-- Databricks notebook source -- Databricks notebook source
-- DBTITLE 1,产品挂中文名 -- DBTITLE 1,产品挂中文名
--Product中文名字,若有超过5个CORP或者MANU生产，则将其报告上排序的优先级调低 --Product中文名字,若有超过5个CORP或者MANU生产，则将其报告上排序的优先级调低
insert overwrite table tmp.tmp_ims_td_prod_tmp insert overwrite table tmp.tmp_ims_td_prod_tmp
SELECT DISTINCT PROD.PROD_COD SELECT DISTINCT PROD.PROD_COD

151
EC/03 ec_load_data.ipynb
View File

@@ -9,12 +9,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778478974793, "finishTime": 1778669598531,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "86f0c1e4-9672-4b69-b7d7-e972573aa69d", "nuid": "86f0c1e4-9672-4b69-b7d7-e972573aa69d",
"showTitle": false, "showTitle": false,
"startTime": 1778478973655, "startTime": 1778669598528,
"submitTime": 1778478972034, "submitTime": 1778669597681,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -38,12 +38,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778478975285, "finishTime": 1778669598678,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "f319d145-a050-425c-b86f-c73d2e6f41ad", "nuid": "f319d145-a050-425c-b86f-c73d2e6f41ad",
"showTitle": false, "showTitle": false,
"startTime": 1778478974798, "startTime": 1778669598536,
"submitTime": 1778478972040, "submitTime": 1778669597686,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -89,12 +89,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778478991545, "finishTime": 1778669612999,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "a11838c0-855f-4fe6-95d7-790959cb6739", "nuid": "a11838c0-855f-4fe6-95d7-790959cb6739",
"showTitle": false, "showTitle": false,
"startTime": 1778478975290, "startTime": 1778669598682,
"submitTime": 1778478972044, "submitTime": 1778669597690,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -477,12 +477,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778655803909, "finishTime": 1778669615065,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "85453796-9385-4a5a-bf20-6bfd459580ae", "nuid": "85453796-9385-4a5a-bf20-6bfd459580ae",
"showTitle": false, "showTitle": false,
"startTime": 1778655745740, "startTime": 1778669613004,
"submitTime": 1778655744617, "submitTime": 1778669597696,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -494,7 +494,7 @@
"select\n", "select\n",
"iqvia_lineno,iqvia_pack_code,prod_cod,iqvia_notes,new_pack_flag,is_exists_chpa,product_id,prescription_nature,medicine_type,zk_medicine_tier1,zk_medicine_tier2,zk_medicine_tier3,zk_medicine_tier4,common_name,dosage_form,user_type,category_name,product_name,brand_name,zk_manu_des,zk_corp_des,zk_pack_des,counting_unit_a,dosage_unit_a,unit_a,app1_cod,app1_des,app1_des_c,app2_cod,app2_des,app2_des_c,app3_cod,app3_des,app3_des_c,atc1_cod,atc1_des,atc1_des_c,atc2_cod,atc2_des,atc2_des_c,atc3_cod,atc3_des,atc3_des_c,atc4_cod,atc4_des,atc4_des_c,bio_desc,cmps_cod,cmps_des,cmps_des_c,corp_cod,corp_des,corp_des_c,edl_desc,eth_otc_desc,gene_orig_desc,gqce_desc,manu_cod,manu_des,manu_des_c,mnfl_cod,mnfl_des,nrdl_desc,pack_des,stgh_des,pack_lch,paed_desc,prod_des,prod_des_c,tcm_desc,vbp_desc,unit,counting_unit,dosage_unit,NRDL_ENTRY_DATE,t2.Market,t2.bu,nvl(t2.extend_market_ratio,1) as extend_market_ratio,is_az\n", "iqvia_lineno,iqvia_pack_code,prod_cod,iqvia_notes,new_pack_flag,is_exists_chpa,product_id,prescription_nature,medicine_type,zk_medicine_tier1,zk_medicine_tier2,zk_medicine_tier3,zk_medicine_tier4,common_name,dosage_form,user_type,category_name,product_name,brand_name,zk_manu_des,zk_corp_des,zk_pack_des,counting_unit_a,dosage_unit_a,unit_a,app1_cod,app1_des,app1_des_c,app2_cod,app2_des,app2_des_c,app3_cod,app3_des,app3_des_c,atc1_cod,atc1_des,atc1_des_c,atc2_cod,atc2_des,atc2_des_c,atc3_cod,atc3_des,atc3_des_c,atc4_cod,atc4_des,atc4_des_c,bio_desc,cmps_cod,cmps_des,cmps_des_c,corp_cod,corp_des,corp_des_c,edl_desc,eth_otc_desc,gene_orig_desc,gqce_desc,manu_cod,manu_des,manu_des_c,mnfl_cod,mnfl_des,nrdl_desc,pack_des,stgh_des,pack_lch,paed_desc,prod_des,prod_des_c,tcm_desc,vbp_desc,unit,counting_unit,dosage_unit,NRDL_ENTRY_DATE,t2.Market,t2.bu,nvl(t2.extend_market_ratio,1) as extend_market_ratio,is_az\n",
"from tmp.tmp_ec_temp_market t1\n", "from tmp.tmp_ec_temp_market t1\n",
"left join (select Market,BU,Extend_Market,MAX(Extend_Market_Ratio) from DWD.dwd_gnd_ec_tblmarket_bymonth where Extend_Market is not null group by 1,2)t2\n", "left join (select Market,BU,Extend_Market,MAX(Extend_Market_Ratio) as Extend_Market_Ratio from DWD.dwd_gnd_ec_tblmarket_bymonth where Extend_Market is not null group by 1,2,3)t2\n",
"on t1.Market = t2.Extend_Market\n", "on t1.Market = t2.Extend_Market\n",
"where t2.Market is not null" "where t2.Market is not null"
] ]
@@ -508,12 +508,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778479009600, "finishTime": 1778669628644,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "015ef590-887b-4dfa-bfa1-ebd05effc2cf", "nuid": "015ef590-887b-4dfa-bfa1-ebd05effc2cf",
"showTitle": false, "showTitle": false,
"startTime": 1778478993870, "startTime": 1778669615070,
"submitTime": 1778478972058, "submitTime": 1778669597701,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -553,12 +553,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778557198314, "finishTime": 1778669638691,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "339d06b1-1c67-4249-a105-aaafb352bb38", "nuid": "339d06b1-1c67-4249-a105-aaafb352bb38",
"showTitle": false, "showTitle": false,
"startTime": 1778557189165, "startTime": 1778669628648,
"submitTime": 1778557188224, "submitTime": 1778669597706,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -589,12 +589,12 @@
"rowLimit": 10000 "rowLimit": 10000
}, },
"collapsed": true, "collapsed": true,
"finishTime": 1778557198321, "finishTime": 1778669638698,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "c686ec31-8c18-4a4a-96ca-ce5a29f2f676", "nuid": "c686ec31-8c18-4a4a-96ca-ce5a29f2f676",
"showTitle": false, "showTitle": false,
"startTime": 1778557198319, "startTime": 1778669638695,
"submitTime": 1778557188234, "submitTime": 1778669597711,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -643,12 +643,12 @@
"rowLimit": 10000 "rowLimit": 10000
}, },
"collapsed": true, "collapsed": true,
"finishTime": 1778557198329, "finishTime": 1778669638704,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "7814d185-5883-4f00-b350-0d5cab1da7f5", "nuid": "7814d185-5883-4f00-b350-0d5cab1da7f5",
"showTitle": false, "showTitle": false,
"startTime": 1778557198326, "startTime": 1778669638701,
"submitTime": 1778557188239, "submitTime": 1778669597717,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -694,12 +694,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778557203672, "finishTime": 1778669642980,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "b576e313-2815-4991-a92e-82fe68cbaed9", "nuid": "b576e313-2815-4991-a92e-82fe68cbaed9",
"showTitle": false, "showTitle": false,
"startTime": 1778557198332, "startTime": 1778669638708,
"submitTime": 1778557188244, "submitTime": 1778669597721,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -738,12 +738,12 @@
"rowLimit": 10000 "rowLimit": 10000
}, },
"collapsed": true, "collapsed": true,
"finishTime": 1778557203681, "finishTime": 1778669642990,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "5fdbf78f-228d-4e26-94b8-94565feb2ddf", "nuid": "5fdbf78f-228d-4e26-94b8-94565feb2ddf",
"showTitle": false, "showTitle": false,
"startTime": 1778557203677, "startTime": 1778669642985,
"submitTime": 1778557188250, "submitTime": 1778669597728,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -930,12 +930,12 @@
"rowLimit": 10000 "rowLimit": 10000
}, },
"collapsed": true, "collapsed": true,
"finishTime": 1778557203688, "finishTime": 1778669642998,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "b1f69228-3a47-4a76-ab21-a10e52305a56", "nuid": "b1f69228-3a47-4a76-ab21-a10e52305a56",
"showTitle": false, "showTitle": false,
"startTime": 1778557203685, "startTime": 1778669642994,
"submitTime": 1778557188256, "submitTime": 1778669597734,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1054,12 +1054,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778557219728, "finishTime": 1778669657466,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "2c5737c9-b8fe-4b8a-a4b6-f39be69af55c", "nuid": "2c5737c9-b8fe-4b8a-a4b6-f39be69af55c",
"showTitle": false, "showTitle": false,
"startTime": 1778557203693, "startTime": 1778669643001,
"submitTime": 1778557188260, "submitTime": 1778669597739,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1221,12 +1221,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778557241785, "finishTime": 1778669677566,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "9b43d6a9-6fd4-4568-bc48-7d7cb7ffe266", "nuid": "9b43d6a9-6fd4-4568-bc48-7d7cb7ffe266",
"showTitle": false, "showTitle": false,
"startTime": 1778557219733, "startTime": 1778669657471,
"submitTime": 1778557188268, "submitTime": 1778669597744,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1317,12 +1317,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778557246541, "finishTime": 1778669682213,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "ebb41c4f-a6d7-4dbd-bdb1-304f0c5abae4", "nuid": "ebb41c4f-a6d7-4dbd-bdb1-304f0c5abae4",
"showTitle": false, "showTitle": false,
"startTime": 1778557241789, "startTime": 1778669677570,
"submitTime": 1778557188277, "submitTime": 1778669597750,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1430,12 +1430,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778557250987, "finishTime": 1778669686953,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "25bd255c-597c-4bb0-9c7a-0577e28eefee", "nuid": "25bd255c-597c-4bb0-9c7a-0577e28eefee",
"showTitle": false, "showTitle": false,
"startTime": 1778557246546, "startTime": 1778669682218,
"submitTime": 1778557188283, "submitTime": 1778669597756,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1543,12 +1543,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778557268253, "finishTime": 1778669699382,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "d13a4ab2-9a37-4ac4-a886-3cf57cc9d41a", "nuid": "d13a4ab2-9a37-4ac4-a886-3cf57cc9d41a",
"showTitle": false, "showTitle": false,
"startTime": 1778557250991, "startTime": 1778669686957,
"submitTime": 1778557188292, "submitTime": 1778669597761,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1695,12 +1695,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778557271279, "finishTime": 1778669702234,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "8c861c10-773a-4449-a5dc-c0340e5ec97e", "nuid": "8c861c10-773a-4449-a5dc-c0340e5ec97e",
"showTitle": false, "showTitle": false,
"startTime": 1778557268257, "startTime": 1778669699387,
"submitTime": 1778557188299, "submitTime": 1778669597766,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1722,12 +1722,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778557275535, "finishTime": 1778669705886,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "e39d8c66-7469-46db-a2f4-79976266bec2", "nuid": "e39d8c66-7469-46db-a2f4-79976266bec2",
"showTitle": false, "showTitle": false,
"startTime": 1778557271284, "startTime": 1778669702239,
"submitTime": 1778557188306, "submitTime": 1778669597770,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1789,12 +1789,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778557278550, "finishTime": 1778669709326,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "bbe40a04-e787-4133-9901-5f331baf912a", "nuid": "bbe40a04-e787-4133-9901-5f331baf912a",
"showTitle": false, "showTitle": false,
"startTime": 1778557275539, "startTime": 1778669705892,
"submitTime": 1778557188311, "submitTime": 1778669597775,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -1820,12 +1820,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778557278560, "finishTime": 1778669709335,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "f8d0befe-3de4-48a8-ac53-cdc19c897d0f", "nuid": "f8d0befe-3de4-48a8-ac53-cdc19c897d0f",
"showTitle": false, "showTitle": false,
"startTime": 1778557278555, "startTime": 1778669709331,
"submitTime": 1778557188317, "submitTime": 1778669597779,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -2075,12 +2075,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778557283201, "finishTime": 1778669714239,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "9e6100db-d97c-41a6-8b30-4f7a774c1903", "nuid": "9e6100db-d97c-41a6-8b30-4f7a774c1903",
"showTitle": false, "showTitle": false,
"startTime": 1778557278565, "startTime": 1778669709339,
"submitTime": 1778557188322, "submitTime": 1778669597785,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -2267,12 +2267,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778557285684, "finishTime": 1778669717268,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "0c0fe2ec-f80e-4872-85bf-fd5d058cc8b8", "nuid": "0c0fe2ec-f80e-4872-85bf-fd5d058cc8b8",
"showTitle": false, "showTitle": false,
"startTime": 1778557283205, "startTime": 1778669714244,
"submitTime": 1778557188328, "submitTime": 1778669597790,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -2336,7 +2336,10 @@
"cell_type": "markdown", "cell_type": "markdown",
"metadata": { "metadata": {
"application/vnd.databricks.v1+cell": { "application/vnd.databricks.v1+cell": {
"cellMetadata": {}, "cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {}, "inputWidgets": {},
"nuid": "dc2bbed1-223b-4610-a55f-334e63050e68", "nuid": "dc2bbed1-223b-4610-a55f-334e63050e68",
"showTitle": false, "showTitle": false,
@@ -2357,12 +2360,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778557292166, "finishTime": 1778669722350,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "6b591466-5af8-40ef-a9b9-c4f137843f18", "nuid": "6b591466-5af8-40ef-a9b9-c4f137843f18",
"showTitle": false, "showTitle": false,
"startTime": 1778557285688, "startTime": 1778669717272,
"submitTime": 1778557188343, "submitTime": 1778669597803,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }
@@ -2470,12 +2473,12 @@
"byteLimit": 2048000, "byteLimit": 2048000,
"rowLimit": 10000 "rowLimit": 10000
}, },
"finishTime": 1778557292715, "finishTime": 1778669722953,
"inputWidgets": {}, "inputWidgets": {},
"nuid": "20f3c0a6-f626-4e64-bce4-7653087adb16", "nuid": "20f3c0a6-f626-4e64-bce4-7653087adb16",
"showTitle": false, "showTitle": false,
"startTime": 1778557292171, "startTime": 1778669722354,
"submitTime": 1778557188353, "submitTime": 1778669597808,
"tableResultSettingsMap": {}, "tableResultSettingsMap": {},
"title": "" "title": ""
} }

4
EC/1 (ec)blob_to_dwd.sql
View File

@@ -15,9 +15,9 @@
-- MAGIC ) -- MAGIC )
-- MAGIC # 基础路径 -- MAGIC # 基础路径
-- MAGIC # 测试环境 -- MAGIC # 测试环境
-- MAGIC base_path_0 = f"abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/ODS/GND/UserUpload/" -- MAGIC # base_path_0 = f"abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/ODS/GND/UserUpload/"
-- MAGIC # 生产环境 -- MAGIC # 生产环境
-- MAGIC # base_path_0 = f"abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/ODS/GND/UserUpload/" -- MAGIC base_path_0 = f"abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/ODS/GND/UserUpload/"
-- MAGIC base_path = base_path_0 + today_path -- MAGIC base_path = base_path_0 + today_path
-- MAGIC print(base_path) -- MAGIC print(base_path)
-- MAGIC -- MAGIC

4
Retail/18 DM_TD_EXT_DTP_MARKET_RATIO.sql
View File

@@ -9,9 +9,9 @@
-- ETL_INSERT_DT TIMESTAMP, -- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP) -- ETL_UPDATE_DT TIMESTAMP)
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_dtp_market_ratio'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_dtp_market_ratio';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_dtp_market_ratio'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_dtp_market_ratio';
-- COMMAND ---------- -- COMMAND ----------

4
Retail/19 DM_TD_EXT_DTP_MARKET_BRAND_RATIO.sql
View File

@@ -17,9 +17,9 @@
-- ETL_INSERT_DT TIMESTAMP, -- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP) -- ETL_UPDATE_DT TIMESTAMP)
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_dtp_market_brand_ratio'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_dtp_market_brand_ratio';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_dtp_market_brand_ratio'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_dtp_market_brand_ratio';
-- COMMAND ---------- -- COMMAND ----------

4
Retail/20 DM_TD_EXT_DTP_MARKET_PACK_MAPPING.sql
View File

@@ -24,9 +24,9 @@
-- ETL_UPDATE_DT TIMESTAMP -- ETL_UPDATE_DT TIMESTAMP
-- ) -- )
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_dtp_market_pack_mapping'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_dtp_market_pack_mapping';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_dtp_market_pack_mapping'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_dtp_market_pack_mapping';
-- COMMAND ---------- -- COMMAND ----------

12
THC/06 DM_TD_EXT_THC_PACK2MARKET.sql
View File

@@ -39,9 +39,9 @@
-- ETL_INSERT_DT TIMESTAMP, -- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP) -- ETL_UPDATE_DT TIMESTAMP)
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_thc_pack2market'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_thc_pack2market';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_thc_pack2market'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_thc_pack2market';
-- COMMAND ---------- -- COMMAND ----------
@@ -50,9 +50,9 @@
-- MARKET STRING, -- MARKET STRING,
-- PACK_CODE STRING) -- PACK_CODE STRING)
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_thc_pack2market_keycompetitor'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_thc_pack2market_keycompetitor';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_thc_pack2market_keycompetitor'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_thc_pack2market_keycompetitor';
-- COMMAND ---------- -- COMMAND ----------
@@ -61,9 +61,9 @@
-- PACK_CODE STRING, -- PACK_CODE STRING,
-- CLASS STRING) -- CLASS STRING)
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_thc_pack2market_class'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_thc_pack2market_class';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_thc_pack2market_class'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_thc_pack2market_class';
-- COMMAND ---------- -- COMMAND ----------

4
UNIONALL/DM_TF_EXT_RETAIL_MARKET_SALES_NIAD.sql
View File

@@ -43,9 +43,9 @@
-- ) -- )
-- USING delta -- USING delta
-- PARTITIONED BY (DATA_SOURCE) -- PARTITIONED BY (DATA_SOURCE)
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_retail_market_sales'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_retail_market_sales';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_retail_market_sales'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_retail_market_sales';
-- COMMAND ---------- -- COMMAND ----------

4
UNIONALL/DM_TF_EXT_UNIONALL_MARKET_SALES.sql
View File

@@ -44,9 +44,9 @@
-- ) -- )
-- USING delta -- USING delta
-- PARTITIONED BY (DATA_SOURCE) -- PARTITIONED BY (DATA_SOURCE)
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_market_sales'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_market_sales';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_market_sales'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_market_sales';
-- COMMAND ---------- -- COMMAND ----------

4
UNIONALL/DM_TF_EXT_UNIONALL_SALES.sql
View File

@@ -32,9 +32,9 @@
-- ETL_INSERT_DT TIMESTAMP, -- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP) -- ETL_UPDATE_DT TIMESTAMP)
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales';
-- COMMAND ---------- -- COMMAND ----------

7
UNIONALL/DM_TF_EXT_UNIONALL_SALES_MAPPING.sql
View File

@@ -51,9 +51,9 @@
-- -- ETL_UPDATE_DT TIMESTAMP -- -- ETL_UPDATE_DT TIMESTAMP
-- ) -- )
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales_mapping'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales_mapping';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales_mapping'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales_mapping';
-- COMMAND ---------- -- COMMAND ----------
@@ -341,9 +341,6 @@ WHERE
-- COMMAND ---------- -- COMMAND ----------
create or replace table tmp.tmp_tf_ext_aia_target_inst create or replace table tmp.tmp_tf_ext_aia_target_inst
-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/TMP/tmp_tf_ext_aia_target_inst';
-- 上面是生产环境location，下面是测试环境location
LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/TMP/tmp_tf_ext_aia_target_inst'
as as
select distinct select distinct
MARKET, MARKET,

4
UNIONALL/DM_TF_EXT_UNIONALL_SALES_MAPPING_NIAD.sql
View File

@@ -32,9 +32,9 @@
-- DIGITAL_SPREAD_RATE STRING, -- DIGITAL_SPREAD_RATE STRING,
-- WEIGHTED_SPREAD_RATE STRING) -- WEIGHTED_SPREAD_RATE STRING)
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales_mapping_niad'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales_mapping_niad';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales_mapping_niad'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales_mapping_niad';
-- COMMAND ---------- -- COMMAND ----------

4
XIEHE/07 DM_TD_EXT_XIEHE_PACK_PROPERTY.sql
View File

@@ -69,9 +69,9 @@
-- ETL_UPDATE_DT TIMESTAMP -- ETL_UPDATE_DT TIMESTAMP
-- ) -- )
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_pack_property'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_pack_property';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_pack_property'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_pack_property';
-- COMMAND ---------- -- COMMAND ----------

12
XIEHE/08 DM_TD_EXT_XIEHE_PACK2MARKET.sql
View File

@@ -39,9 +39,9 @@
-- ETL_INSERT_DT TIMESTAMP, -- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP) -- ETL_UPDATE_DT TIMESTAMP)
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_pack2market'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_pack2market';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_pack2market'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_pack2market';
-- COMMAND ---------- -- COMMAND ----------
@@ -50,9 +50,9 @@
-- MARKET STRING, -- MARKET STRING,
-- PACK_CODE STRING) -- PACK_CODE STRING)
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_pack2market_keycompetitor'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_pack2market_keycompetitor';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_pack2market_keycompetitor'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_pack2market_keycompetitor';
-- COMMAND ---------- -- COMMAND ----------
@@ -61,9 +61,9 @@
-- PACK_CODE STRING, -- PACK_CODE STRING,
-- CLASS STRING) -- CLASS STRING)
-- USING delta -- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_pack2market_class'; -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_pack2market_class';
-- -- 上面是生产环境location，下面是测试环境location -- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_pack2market_class'; -- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_pack2market_class';
-- COMMAND ---------- -- COMMAND ----------

2461
上线脚本.sql
View File

File diff suppressed because it is too large Load Diff

39
数据验证.sql
View File

@@ -1,39 +0,0 @@
-- Databricks notebook source
desc dm.dm_tf_ext_aia_sales
-- COMMAND ----------
select DATA_SOURCE,DTP_FLAG from dm.DM_TF_EXTERANL_SALES_MERGED_DATA_DTP_OTHERS_bymonth
group by DATA_SOURCE,DTP_FLAG
-- COMMAND ----------
select DTP_FLAG,DATA_SOURCE from DM.DM_TF_EXT_RETAIL_DTP_SALES group by DTP_FLAG,DATA_SOURCE
-- order by DATA_SOURCE
-- COMMAND ----------
select DATA_SOURCE,DTP_FLAG from DM.DM_TF_EXT_UNIONALL_SALES group by DATA_SOURCE,DTP_FLAG
order by DATA_SOURCE,DTP_FLAG
-- COMMAND ----------
select DATA_SOURCE,DTP_FLAG,count(*) from DM.dm_tf_ext_unionall_sales_mapping
group BY DATA_SOURCE,DTP_FLAG
ORDER BY DATA_SOURCE,DTP_FLAG
-- COMMAND ----------
select * from DM.DM_TF_EXT_UNIONALL_SALES_MAPPING
limit 4
-- COMMAND ----------
select AUDIT_CODE from DM.DM_TF_EXT_CHPA_SALES group by AUDIT_CODE
-- limit 5
-- COMMAND ----------
select * from DM.DM_TD_EXT_CHPA_PACK2MARKET
limit 5