chenwu/MarketAnalysis-ETL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260513_上线DBR后版本

Browse Source
This commit is contained in:
chenwu
2026-05-13 12:38:21 +00:00
parent 5d4d312641
commit a9bbe0e95a
18 changed files with 128 additions and 2629 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
EC/03 ec_load_data.ipynb
View File

@@ -9,12 +9,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778478974793,
"finishTime": 1778669598531,
"inputWidgets": {},
"nuid": "86f0c1e4-9672-4b69-b7d7-e972573aa69d",
"showTitle": false,
"startTime": 1778478973655,
"submitTime": 1778478972034,
"startTime": 1778669598528,
"submitTime": 1778669597681,
"tableResultSettingsMap": {},
"title": ""
}
@@ -38,12 +38,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778478975285,
"finishTime": 1778669598678,
"inputWidgets": {},
"nuid": "f319d145-a050-425c-b86f-c73d2e6f41ad",
"showTitle": false,
"startTime": 1778478974798,
"submitTime": 1778478972040,
"startTime": 1778669598536,
"submitTime": 1778669597686,
"tableResultSettingsMap": {},
"title": ""
}
@@ -89,12 +89,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778478991545,
"finishTime": 1778669612999,
"inputWidgets": {},
"nuid": "a11838c0-855f-4fe6-95d7-790959cb6739",
"showTitle": false,
"startTime": 1778478975290,
"submitTime": 1778478972044,
"startTime": 1778669598682,
"submitTime": 1778669597690,
"tableResultSettingsMap": {},
"title": ""
}
@@ -477,12 +477,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778655803909,
"finishTime": 1778669615065,
"inputWidgets": {},
"nuid": "85453796-9385-4a5a-bf20-6bfd459580ae",
"showTitle": false,
"startTime": 1778655745740,
"submitTime": 1778655744617,
"startTime": 1778669613004,
"submitTime": 1778669597696,
"tableResultSettingsMap": {},
"title": ""
}
@@ -494,7 +494,7 @@
"select\n",
"iqvia_lineno,iqvia_pack_code,prod_cod,iqvia_notes,new_pack_flag,is_exists_chpa,product_id,prescription_nature,medicine_type,zk_medicine_tier1,zk_medicine_tier2,zk_medicine_tier3,zk_medicine_tier4,common_name,dosage_form,user_type,category_name,product_name,brand_name,zk_manu_des,zk_corp_des,zk_pack_des,counting_unit_a,dosage_unit_a,unit_a,app1_cod,app1_des,app1_des_c,app2_cod,app2_des,app2_des_c,app3_cod,app3_des,app3_des_c,atc1_cod,atc1_des,atc1_des_c,atc2_cod,atc2_des,atc2_des_c,atc3_cod,atc3_des,atc3_des_c,atc4_cod,atc4_des,atc4_des_c,bio_desc,cmps_cod,cmps_des,cmps_des_c,corp_cod,corp_des,corp_des_c,edl_desc,eth_otc_desc,gene_orig_desc,gqce_desc,manu_cod,manu_des,manu_des_c,mnfl_cod,mnfl_des,nrdl_desc,pack_des,stgh_des,pack_lch,paed_desc,prod_des,prod_des_c,tcm_desc,vbp_desc,unit,counting_unit,dosage_unit,NRDL_ENTRY_DATE,t2.Market,t2.bu,nvl(t2.extend_market_ratio,1) as extend_market_ratio,is_az\n",
"from tmp.tmp_ec_temp_market t1\n",
"left join (select Market,BU,Extend_Market,MAX(Extend_Market_Ratio) from DWD.dwd_gnd_ec_tblmarket_bymonth where Extend_Market is not null group by 1,2)t2\n",
"left join (select Market,BU,Extend_Market,MAX(Extend_Market_Ratio) as Extend_Market_Ratio from DWD.dwd_gnd_ec_tblmarket_bymonth where Extend_Market is not null group by 1,2,3)t2\n",
"on t1.Market = t2.Extend_Market\n",
"where t2.Market is not null"
]
@@ -508,12 +508,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778479009600,
"finishTime": 1778669628644,
"inputWidgets": {},
"nuid": "015ef590-887b-4dfa-bfa1-ebd05effc2cf",
"showTitle": false,
"startTime": 1778478993870,
"submitTime": 1778478972058,
"startTime": 1778669615070,
"submitTime": 1778669597701,
"tableResultSettingsMap": {},
"title": ""
}
@@ -553,12 +553,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778557198314,
"finishTime": 1778669638691,
"inputWidgets": {},
"nuid": "339d06b1-1c67-4249-a105-aaafb352bb38",
"showTitle": false,
"startTime": 1778557189165,
"submitTime": 1778557188224,
"startTime": 1778669628648,
"submitTime": 1778669597706,
"tableResultSettingsMap": {},
"title": ""
}
@@ -589,12 +589,12 @@
"rowLimit": 10000
},
"collapsed": true,
"finishTime": 1778557198321,
"finishTime": 1778669638698,
"inputWidgets": {},
"nuid": "c686ec31-8c18-4a4a-96ca-ce5a29f2f676",
"showTitle": false,
"startTime": 1778557198319,
"submitTime": 1778557188234,
"startTime": 1778669638695,
"submitTime": 1778669597711,
"tableResultSettingsMap": {},
"title": ""
}
@@ -643,12 +643,12 @@
"rowLimit": 10000
},
"collapsed": true,
"finishTime": 1778557198329,
"finishTime": 1778669638704,
"inputWidgets": {},
"nuid": "7814d185-5883-4f00-b350-0d5cab1da7f5",
"showTitle": false,
"startTime": 1778557198326,
"submitTime": 1778557188239,
"startTime": 1778669638701,
"submitTime": 1778669597717,
"tableResultSettingsMap": {},
"title": ""
}
@@ -694,12 +694,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778557203672,
"finishTime": 1778669642980,
"inputWidgets": {},
"nuid": "b576e313-2815-4991-a92e-82fe68cbaed9",
"showTitle": false,
"startTime": 1778557198332,
"submitTime": 1778557188244,
"startTime": 1778669638708,
"submitTime": 1778669597721,
"tableResultSettingsMap": {},
"title": ""
}
@@ -738,12 +738,12 @@
"rowLimit": 10000
},
"collapsed": true,
"finishTime": 1778557203681,
"finishTime": 1778669642990,
"inputWidgets": {},
"nuid": "5fdbf78f-228d-4e26-94b8-94565feb2ddf",
"showTitle": false,
"startTime": 1778557203677,
"submitTime": 1778557188250,
"startTime": 1778669642985,
"submitTime": 1778669597728,
"tableResultSettingsMap": {},
"title": ""
}
@@ -930,12 +930,12 @@
"rowLimit": 10000
},
"collapsed": true,
"finishTime": 1778557203688,
"finishTime": 1778669642998,
"inputWidgets": {},
"nuid": "b1f69228-3a47-4a76-ab21-a10e52305a56",
"showTitle": false,
"startTime": 1778557203685,
"submitTime": 1778557188256,
"startTime": 1778669642994,
"submitTime": 1778669597734,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1054,12 +1054,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778557219728,
"finishTime": 1778669657466,
"inputWidgets": {},
"nuid": "2c5737c9-b8fe-4b8a-a4b6-f39be69af55c",
"showTitle": false,
"startTime": 1778557203693,
"submitTime": 1778557188260,
"startTime": 1778669643001,
"submitTime": 1778669597739,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1221,12 +1221,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778557241785,
"finishTime": 1778669677566,
"inputWidgets": {},
"nuid": "9b43d6a9-6fd4-4568-bc48-7d7cb7ffe266",
"showTitle": false,
"startTime": 1778557219733,
"submitTime": 1778557188268,
"startTime": 1778669657471,
"submitTime": 1778669597744,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1317,12 +1317,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778557246541,
"finishTime": 1778669682213,
"inputWidgets": {},
"nuid": "ebb41c4f-a6d7-4dbd-bdb1-304f0c5abae4",
"showTitle": false,
"startTime": 1778557241789,
"submitTime": 1778557188277,
"startTime": 1778669677570,
"submitTime": 1778669597750,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1430,12 +1430,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778557250987,
"finishTime": 1778669686953,
"inputWidgets": {},
"nuid": "25bd255c-597c-4bb0-9c7a-0577e28eefee",
"showTitle": false,
"startTime": 1778557246546,
"submitTime": 1778557188283,
"startTime": 1778669682218,
"submitTime": 1778669597756,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1543,12 +1543,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778557268253,
"finishTime": 1778669699382,
"inputWidgets": {},
"nuid": "d13a4ab2-9a37-4ac4-a886-3cf57cc9d41a",
"showTitle": false,
"startTime": 1778557250991,
"submitTime": 1778557188292,
"startTime": 1778669686957,
"submitTime": 1778669597761,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1695,12 +1695,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778557271279,
"finishTime": 1778669702234,
"inputWidgets": {},
"nuid": "8c861c10-773a-4449-a5dc-c0340e5ec97e",
"showTitle": false,
"startTime": 1778557268257,
"submitTime": 1778557188299,
"startTime": 1778669699387,
"submitTime": 1778669597766,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1722,12 +1722,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778557275535,
"finishTime": 1778669705886,
"inputWidgets": {},
"nuid": "e39d8c66-7469-46db-a2f4-79976266bec2",
"showTitle": false,
"startTime": 1778557271284,
"submitTime": 1778557188306,
"startTime": 1778669702239,
"submitTime": 1778669597770,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1789,12 +1789,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778557278550,
"finishTime": 1778669709326,
"inputWidgets": {},
"nuid": "bbe40a04-e787-4133-9901-5f331baf912a",
"showTitle": false,
"startTime": 1778557275539,
"submitTime": 1778557188311,
"startTime": 1778669705892,
"submitTime": 1778669597775,
"tableResultSettingsMap": {},
"title": ""
}
@@ -1820,12 +1820,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778557278560,
"finishTime": 1778669709335,
"inputWidgets": {},
"nuid": "f8d0befe-3de4-48a8-ac53-cdc19c897d0f",
"showTitle": false,
"startTime": 1778557278555,
"submitTime": 1778557188317,
"startTime": 1778669709331,
"submitTime": 1778669597779,
"tableResultSettingsMap": {},
"title": ""
}
@@ -2075,12 +2075,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778557283201,
"finishTime": 1778669714239,
"inputWidgets": {},
"nuid": "9e6100db-d97c-41a6-8b30-4f7a774c1903",
"showTitle": false,
"startTime": 1778557278565,
"submitTime": 1778557188322,
"startTime": 1778669709339,
"submitTime": 1778669597785,
"tableResultSettingsMap": {},
"title": ""
}
@@ -2267,12 +2267,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778557285684,
"finishTime": 1778669717268,
"inputWidgets": {},
"nuid": "0c0fe2ec-f80e-4872-85bf-fd5d058cc8b8",
"showTitle": false,
"startTime": 1778557283205,
"submitTime": 1778557188328,
"startTime": 1778669714244,
"submitTime": 1778669597790,
"tableResultSettingsMap": {},
"title": ""
}
@@ -2336,7 +2336,10 @@
"cell_type": "markdown",
"metadata": {
"application/vnd.databricks.v1+cell": {
"cellMetadata": {},
"cellMetadata": {
"byteLimit": 2048000,
"rowLimit": 10000
},
"inputWidgets": {},
"nuid": "dc2bbed1-223b-4610-a55f-334e63050e68",
"showTitle": false,
@@ -2357,12 +2360,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778557292166,
"finishTime": 1778669722350,
"inputWidgets": {},
"nuid": "6b591466-5af8-40ef-a9b9-c4f137843f18",
"showTitle": false,
"startTime": 1778557285688,
"submitTime": 1778557188343,
"startTime": 1778669717272,
"submitTime": 1778669597803,
"tableResultSettingsMap": {},
"title": ""
}
@@ -2470,12 +2473,12 @@
"byteLimit": 2048000,
"rowLimit": 10000
},
"finishTime": 1778557292715,
"finishTime": 1778669722953,
"inputWidgets": {},
"nuid": "20f3c0a6-f626-4e64-bce4-7653087adb16",
"showTitle": false,
"startTime": 1778557292171,
"submitTime": 1778557188353,
"startTime": 1778669722354,
"submitTime": 1778669597808,
"tableResultSettingsMap": {},
"title": ""
}

6
EC/1 (ec)blob_to_dwd.sql
View File

@@ -15,9 +15,9 @@
-- MAGIC )
-- MAGIC # 基础路径
-- MAGIC # 测试环境
-- MAGIC base_path_0 = f"abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/ODS/GND/UserUpload/"
-- MAGIC # base_path_0 = f"abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/ODS/GND/UserUpload/"
-- MAGIC # 生产环境
-- MAGIC # base_path_0 = f"abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/ODS/GND/UserUpload/"
-- MAGIC base_path_0 = f"abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/ODS/GND/UserUpload/"
-- MAGIC base_path = base_path_0 + today_path
-- MAGIC print(base_path)
-- MAGIC
@@ -100,4 +100,4 @@
-- MAGIC else:
-- MAGIC print("未找到批次或路径不存在。")
-- MAGIC
-- MAGIC
-- MAGIC