chenwu/MarketAnalysis-ETL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
3a03ba1ad2c18b4464572c32f08529a039a96abb
MarketAnalysis-ETL/Retail/07 split_dtp_pack_data.py
chenwu 832c7bcd69 new file: 01 dm_tf_external_sales.sql 
new file:   02 dm_td_external_keycompatitor.sql
	new file:   03 dm_td_external_brand_market.sql
	new file:   04 dm_td_external_calendar.sql
	new file:   05 dm_td_external_exchangerate.sql
	new file:   07 dm_td_external_packinfo.sql
	new file:   08 dm_td_external_corp.sql
	new file:   09 dm_td_external_geo_type.sql
	new file:   11 DM_TD_EXTERNAL_MARKET_NEW.sql
	new file:   12 dm_td_external_org.sql
	new file:   13 external auth.sql
	new file:   14 dm_tf_external_retail_special_bkp.sql
	new file:   AIA/01 dm_aia_pack_property.sql
	new file:   AIA/02 dm_ext_aia_sales.sql
	new file:   AIA/02 dm_ext_aia_sales_bakup_20230327.sql
	new file:   AIA/03 dm_aia_flag.sql
	new file:   AIA/04 dm_aia_provided_flag.sql
	new file:   AIA/06 DM_TD_EXT_AIA_PACK_PROPERTY.sql
	new file:   AIA/07 DM_TD_EXT_AIA_PACK2MARKET.sql
	new file:   AIA/08 DM_TD_EXT_AIA_MARKET_RATIO.sql
	new file:   AIA/09 DM_TD_EXT_AIA_MARKET_BRAND_RATIO.sql
	new file:   AIA/10 DM_TD_EXT_AIA_MARKET_PACK_MAPPING.sql
	new file:   AIA/11 DM_TD_EXT_AIA_TARGET_INST.sql
	new file:   AIA/11 dm_aia_targethp_flag.sql
	new file:   "AIA/z_03 dm_aia_flag_\345\244\207\344\273\275\347\224\250.sql"
	new file:   CHC/01 dm_chc_pack_property.sql
	new file:   CHC/02 DM_TF_EXT_CHC_SALES.sql
	new file:   CHC/03 DM_TD_EXT_CHC_PACK_PROPERTY.sql
	new file:   CHC/04 DM_TD_EXT_CHC_PACK2MARKET.sql
	new file:   CHC/05 DM_TD_EXT_CHC_MARKET_RATIO.sql
	new file:   CHC/06 DM_TD_EXT_CHC_MARKET_BRAND_RATIO.sql
	new file:   CHC/07 DM_TD_EXT_CHC_MARKET_PACK_MAPPING.sql
	new file:   CHPA/01 dwd_ims_atc_hierarchy.sql
	new file:   CHPA/01 dwd_ims_nfc_hierarchy.sql
	new file:   CHPA/01 dwd_ims_td_manufacturer_corp.sql
	new file:   CHPA/01 dwd_ims_td_pack_property.sql
	new file:   CHPA/01 dwd_update.sql
	new file:   CHPA/01_FB_BLOB_TO_DWD.sql
	new file:   CHPA/02 DWS_IMS_TD_GEO.sql
	new file:   CHPA/02 dws_ims_td_atc_cn.sql
	new file:   CHPA/02 dws_ims_td_corp_cn.sql
	new file:   CHPA/02 dws_ims_td_date.sql
	new file:   CHPA/02 dws_ims_td_manu_cn.sql
	new file:   CHPA/02 dws_ims_td_market.sql
	new file:   CHPA/02 dws_ims_td_market_ta.sql
	new file:   CHPA/02 dws_ims_td_nfc_cn.sql
	new file:   CHPA/02 dws_ims_td_prod_cn.sql
	new file:   CHPA/02 tmp_ims_td_prod_tmp.sql
	new file:   CHPA/02 tmp_ims_tf_fact_sales.sql
	new file:   CHPA/03 DM_TD_EXT_CHPA_MARKET_BRAND_RATIO.sql
	new file:   CHPA/03 DM_TD_EXT_CHPA_MARKET_PACK_MAPPING.sql
	new file:   CHPA/03 DM_TD_EXT_CHPA_MARKET_RATIO.sql
	new file:   CHPA/03 DM_TD_EXT_CHPA_PACK2MARKET.sql
	new file:   CHPA/03 DM_TD_EXT_CHPA_PACK_PROPERTY.sql
	new file:   CHPA/03 DM_TF_EXT_CHPA_SALES.sql
	new file:   CHPA/03 dm_ims_td_calendar.sql
	new file:   CHPA/03 dm_ims_td_geo.sql
	new file:   CHPA/03 dm_ims_td_market_property.sql
	new file:   CHPA/03 dm_ims_td_org.sql
	new file:   CHPA/03 dm_ims_td_org_hvh.sql
	new file:   CHPA/03 dm_ims_td_pack_property.sql
	new file:   CHPA/03 dm_ims_tf_sales.sql
	new file:   CHPA/03 dm_td_chpa_market_definition.sql
	new file:   CHPA/03 dm_td_ims_city_mapping.sql
	new file:   EC/03 ec_load_data.sql
	new file:   EC/04 DM_TD_EXT_EC_PACK_PROPERTY.sql
	new file:   EC/05 DM_TF_EXT_EC_SALES.sql
	new file:   EC/06 DM_TD_EXT_EC_PACK2MARKET.sql
	new file:   EC/07 DM_TD_EXT_EC_MARKET_RATIO.sql
	new file:   EC/08 DM_TD_EXT_EC_MARKET_BRAND_RATIO.sql
	new file:   EC/09 DM_TD_EXT_EC_MARKET_PACK_MAPPING.sql
	new file:   EC/1 (ec)blob_to_dwd.sql
	new file:   EC/2 dwd_inc_gnd_ext_ec_nationnal_pack_union_all.py
	new file:   Merged_Data/Merged_Data_Config_table_bkp.sql
	new file:   Merged_Data/Merged_Data_Config_table_bymonth.sql
	new file:   Merged_Data/dm_tf_exteranl_sales_merged_data_dtp_others_bkp.sql
	new file:   Merged_Data/dm_tf_exteranl_sales_merged_data_dtp_others_bymonth_bkp.sql
	new file:   ORG/DM_TD_EXT_AIA_ORG.sql
	new file:   ORG/DM_TD_EXT_CHC_ORG.sql
	new file:   ORG/DM_TD_EXT_CHPA_ORG.sql
	new file:   ORG/DM_TD_EXT_COUNTY_ORG.sql
	new file:   ORG/DM_TD_EXT_EC_ORG.sql
	new file:   ORG/DM_TD_EXT_RETAIL_ORG.sql
	new file:   ORG/DM_TD_EXT_THC_ORG.sql
	new file:   ORG/DM_TD_EXT_XIEHE_ORG.sql
	new file:   OTHERS/01 dm_td_report_url.sql
	new file:   OTHERS/02 dws_ext_email_warning.sql
	new file:   OTHERS/external_triggered_email.py
	new file:   Retail/01 load_tmp_data.py
	new file:   Retail/02 split_brand_data.py
	new file:   Retail/03 split_pack_data.py
	new file:   Retail/04 map_to_dws_table.py
	new file:   Retail/05 load_dtp_temp_data.py
	new file:   Retail/06 split_dtp_brand_data.py
	new file:   Retail/07 split_dtp_pack_data.py
	new file:   Retail/08 map_to_dtp_dws_table.py
	new file:   Retail/09 dwd_inc_gnd_ext_retail_nataional.py
	new file:   Retail/10 map_to_retail_dm_table.py
	new file:   Retail/11 map_to_overview_dm_table.py
	new file:   Retail/12 dws_tf_external_retail_dtp_special.sql
	new file:   Retail/13 DM_TF_EXT_RETAIL_SALES.sql
	new file:   Retail/14 DM_TF_EXT_RETAIL_DTP_SALES.sql
	new file:   Retail/15 DM_TD_EXT_RETAIL_PACK_PROPERTY.sql
	new file:   Retail/16 DM_TD_EXT_RETAIL_DTP_PACK_PROPERTY.sql
	new file:   Retail/17 DM_TD_EXT_DTP_PACK2MARKET.sql
	new file:   Retail/17 DM_TD_EXT_RETAIL_PACK2MARKET.sql
	new file:   Retail/18 DM_TD_EXT_DTP_MARKET_RATIO.sql
	new file:   Retail/18 DM_TD_EXT_RETAIL_MARKET_RATIO.sql
	new file:   Retail/19 DM_TD_EXT_DTP_MARKET_BRAND_RATIO.sql
	new file:   Retail/19 DM_TD_EXT_RETAIL_MARKET_BRAND_RATIO.sql
	new file:   Retail/20 DM_TD_EXT_DTP_MARKET_PACK_MAPPING.sql
	new file:   Retail/20 DM_TD_EXT_RETAIL_MARKET_PACK_MAPPING.sql
	new file:   "Retail/z1 dwd_inc_gnd_ext_retail_nataional_\344\275\234\345\272\237.py"
	new file:   "Retail/z2 retail_load_data_\344\275\234\345\272\237.sql"
	new file:   "Retail/z3 retail_overview_data_\344\275\234\345\272\237.sql"
	new file:   THC/01 dm_thc_pack_property.sql
	new file:   THC/02 dm_ext_thc_sales.sql
	new file:   THC/02 dm_ext_thc_sales_bakup_20260327.sql
	new file:   THC/03 DM_TF_EXT_THC_MARKET_SALES_CHT.sql
	new file:   THC/04 dm_tf_external_sales_thc.sql
	new file:   THC/05 DM_TD_EXT_THC_PACK_PROPERTY.sql
	new file:   THC/06 DM_TD_EXT_THC_PACK2MARKET.sql
	new file:   THC/07 DM_TD_EXT_THC_MARKET_RATIO.sql
	new file:   THC/08 DM_TD_EXT_THC_MARKET_BRAND_RATIO.sql
	new file:   THC/09 DM_TD_EXT_THC_MARKET_PACK_MAPPING.sql
	new file:   UNIONALL/DM_TD_EXT_UNIONALL_MARKET_PACK_MAPPING.sql
	new file:   UNIONALL/DM_TD_EXT_UNIONALL_PACKINFO.sql
	new file:   UNIONALL/DM_TF_EXT_UNIONALL_MARKET_SALES.sql
	new file:   UNIONALL/DM_TF_EXT_UNIONALL_MARKET_SALES_v1.0.sql
	new file:   UNIONALL/DM_TF_EXT_UNIONALL_SALES.sql
	new file:   UNIONALL/DM_TF_EXT_UNIONALL_SALES_MAPPING.sql
	new file:   UNIONALL/DM_TF_EXT_UNIONALL_SALES_MAPPING_NIAD.sql
	new file:   XIEHE/01 xiehe_blob_to_dwd.py
	new file:   XIEHE/02 dm_xiehe_pack_property.sql
	new file:   XIEHE/03 dm_ext_xiehe_geo.sql
	new file:   XIEHE/04 dm_ext_xiehe_sales.sql
	new file:   XIEHE/05 dm_td_xiehe_core_dept.sql
	new file:   XIEHE/06 DM_TF_EXT_XIEHE_SALES.sql
	new file:   XIEHE/07 DM_TD_EXT_XIEHE_PACK_PROPERTY.sql
	new file:   XIEHE/08 DM_TD_EXT_XIEHE_PACK2MARKET.sql
	new file:   XIEHE/09 DM_TD_EXT_XIEHE_MARKET_RATIO.sql
	new file:   XIEHE/10 DM_TD_EXT_XIEHE_MARKET_BRAND_RATIO.sql
	new file:   XIEHE/11 DM_TD_EXT_XIEHE_MARKET_PACK_MAPPING.sql
	new file:   XIEHE/bkp_01 xiehe_blob2dwd.py
	new file:   XIEHE/bkp_02 dm_ext_xiehe_sales.sql
	new file:   XIEHE/bkp_03 dm_ext_xiehe_pack_property.sql
	new file:   county/01 tmp_ims_county_fact_sales_sum.sql
	new file:   county/02 tmp_imscounty_Result.sql
	new file:   county/03 dm_ims_td_county_geo.sql
	new file:   county/04 dws_ext_county_tf_sales.sql
	new file:   county/05 dm_ext_county_td_pack_property.sql
	new file:   county/06 dm_td_county_pack_region.sql
	new file:   county/07 dm_ext_county_tf_sales_region.sql
	new file:   county/08 DM_TD_EXT_COUNTY_PACK_PROPERTY.sql
	new file:   county/09 DM_TF_EXT_COUNTY_SALES.sql
	new file:   county/10 DM_TD_EXT_COUNTY_PACK2MARKET.sql
	new file:   county/11 DM_TD_EXT_COUNTY_MARKET_RATIO.sql
	new file:   county/12 DM_TD_EXT_COUNTY_MARKET_BRAND_RATIO.sql
	new file:   county/13 DM_TD_EXT_COUNTY_MARKET_PACK_MAPPING.sql
	new file:   for_AIA_Dashboard/01 dm_td_aia_inst_mkt.sql
	new file:   for_AIA_Dashboard/02 dm_td_aia_auth_sales.sql
	new file:   for_AIA_Dashboard/03 dm_td_aia_original_col.sql
	new file:   for_AIA_Dashboard/04 dm_td_aia_nosales_inst.sql
	new file:   for_AIA_Dashboard/05 dm_td_aia_is_eagle_flag.sql
	new file:   for_AIA_Dashboard/06 dm_td_aia_rank.sql
	new file:   for_AIA_Dashboard/07 dm_ext_aia_data_remove_flag.sql
	new file:   for_AIA_Dashboard/07 dm_td_aia_remove_special_ins_bkp.py
	new file:   for_AIA_Dashboard/08 dm_ext_aia_data_quality_flag.sql
	new file:   z 01 dm_tf_external_sales.sql
	new file:   "z 10 dm_td_external_market_pack_mapping_\344\275\234\345\272\237.sql"
	new file:   "z 11 dm_td_external_market_\344\275\234\345\272\237.sql"
	new file:   "\344\270\212\347\272\277\350\204\232\346\234\254.sql"
	new file:   "\346\225\260\346\215\256\351\252\214\350\257\201.sql"
2026-04-27 15:48:38 +08:00

1107 lines
41 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# Databricks notebook source
# MAGIC %sql
# MAGIC -- CREATE OR REPLACE TABLE tmp.tmp_zk_retail_dtp_market (
# MAGIC -- -- product_id STRING,
# MAGIC -- -- region_type STRING,
# MAGIC -- -- higher_level_region STRING,
# MAGIC -- market STRING,
# MAGIC -- iqvia_pack_code STRING,
# MAGIC -- iqvia_prod_code STRING,
# MAGIC -- iqvia_notes STRING,
# MAGIC -- new_pack_flag STRING,
# MAGIC -- is_exists_chpa STRING,
# MAGIC -- -- prescription_nature STRING,
# MAGIC -- -- common_name STRING,
# MAGIC -- -- dosage_form STRING,
# MAGIC -- -- category_name STRING,
# MAGIC -- -- product_name STRING,
# MAGIC -- -- brand_name STRING,
# MAGIC -- -- zk_corp_des STRING,
# MAGIC -- -- zk_pack_des STRING,
# MAGIC -- -- counting_unit_a STRING,
# MAGIC -- app1_cod STRING,
# MAGIC -- app1_des STRING,
# MAGIC -- app1_des_c STRING,
# MAGIC -- app2_cod STRING,
# MAGIC -- app2_des STRING,
# MAGIC -- app2_des_c STRING,
# MAGIC -- app3_cod STRING,
# MAGIC -- app3_des STRING,
# MAGIC -- app3_des_c STRING,
# MAGIC -- atc1_cod STRING,
# MAGIC -- atc1_des STRING,
# MAGIC -- atc1_des_c STRING,
# MAGIC -- atc2_cod STRING,
# MAGIC -- atc2_des STRING,
# MAGIC -- atc2_des_c STRING,
# MAGIC -- atc3_cod STRING,
# MAGIC -- atc3_des STRING,
# MAGIC -- atc3_des_c STRING,
# MAGIC -- atc4_cod STRING,
# MAGIC -- atc4_des STRING,
# MAGIC -- atc4_des_c STRING,
# MAGIC -- bio_desc STRING,
# MAGIC -- cmps_cod STRING,
# MAGIC -- cmps_des STRING,
# MAGIC -- cmps_des_c STRING,
# MAGIC -- corp_cod STRING,
# MAGIC -- corp_des STRING,
# MAGIC -- corp_des_c STRING,
# MAGIC -- edl_desc STRING,
# MAGIC -- eth_otc_desc STRING,
# MAGIC -- gene_orig_desc STRING,
# MAGIC -- gqce_desc STRING,
# MAGIC -- manu_cod STRING,
# MAGIC -- manu_des STRING,
# MAGIC -- manu_des_c STRING,
# MAGIC -- mnfl_cod STRING,
# MAGIC -- mnfl_des STRING,
# MAGIC -- nrdl_desc STRING,
# MAGIC -- pack_des STRING,
# MAGIC -- stgh_des STRING,
# MAGIC -- pack_lch STRING,
# MAGIC -- paed_desc STRING,
# MAGIC -- prod_des STRING,
# MAGIC -- prod_des_c STRING,
# MAGIC -- tcm_desc STRING,
# MAGIC -- vbp_desc STRING,
# MAGIC -- unit STRING,
# MAGIC -- counting_unit STRING,
# MAGIC -- dosage_unit STRING,
# MAGIC -- NRDL_ENTRY_DATE STRING,
# MAGIC -- bu STRING,
# MAGIC -- Market_Ratio STRING)
# MAGIC -- USING delta
# MAGIC -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/TMP/tmp_zk_retail_dtp_market'
# MAGIC -- ;
# COMMAND ----------
# MAGIC %sql
# MAGIC -- CREATE OR REPLACE TABLE tmp.tmp_zk_retail_dtp_market_del (
# MAGIC -- -- product_id STRING,
# MAGIC -- -- region_type STRING,
# MAGIC -- -- higher_level_region STRING,
# MAGIC -- market STRING,
# MAGIC -- iqvia_pack_code STRING,
# MAGIC -- iqvia_prod_code STRING,
# MAGIC -- iqvia_notes STRING,
# MAGIC -- new_pack_flag STRING,
# MAGIC -- is_exists_chpa STRING,
# MAGIC -- -- prescription_nature STRING,
# MAGIC -- -- common_name STRING,
# MAGIC -- -- dosage_form STRING,
# MAGIC -- -- category_name STRING,
# MAGIC -- -- product_name STRING,
# MAGIC -- -- brand_name STRING,
# MAGIC -- -- zk_corp_des STRING,
# MAGIC -- -- zk_pack_des STRING,
# MAGIC -- -- counting_unit_a STRING,
# MAGIC -- app1_cod STRING,
# MAGIC -- app1_des STRING,
# MAGIC -- app1_des_c STRING,
# MAGIC -- app2_cod STRING,
# MAGIC -- app2_des STRING,
# MAGIC -- app2_des_c STRING,
# MAGIC -- app3_cod STRING,
# MAGIC -- app3_des STRING,
# MAGIC -- app3_des_c STRING,
# MAGIC -- atc1_cod STRING,
# MAGIC -- atc1_des STRING,
# MAGIC -- atc1_des_c STRING,
# MAGIC -- atc2_cod STRING,
# MAGIC -- atc2_des STRING,
# MAGIC -- atc2_des_c STRING,
# MAGIC -- atc3_cod STRING,
# MAGIC -- atc3_des STRING,
# MAGIC -- atc3_des_c STRING,
# MAGIC -- atc4_cod STRING,
# MAGIC -- atc4_des STRING,
# MAGIC -- atc4_des_c STRING,
# MAGIC -- bio_desc STRING,
# MAGIC -- cmps_cod STRING,
# MAGIC -- cmps_des STRING,
# MAGIC -- cmps_des_c STRING,
# MAGIC -- corp_cod STRING,
# MAGIC -- corp_des STRING,
# MAGIC -- corp_des_c STRING,
# MAGIC -- edl_desc STRING,
# MAGIC -- eth_otc_desc STRING,
# MAGIC -- gene_orig_desc STRING,
# MAGIC -- gqce_desc STRING,
# MAGIC -- manu_cod STRING,
# MAGIC -- manu_des STRING,
# MAGIC -- manu_des_c STRING,
# MAGIC -- mnfl_cod STRING,
# MAGIC -- mnfl_des STRING,
# MAGIC -- nrdl_desc STRING,
# MAGIC -- pack_des STRING,
# MAGIC -- stgh_des STRING,
# MAGIC -- pack_lch STRING,
# MAGIC -- paed_desc STRING,
# MAGIC -- prod_des STRING,
# MAGIC -- prod_des_c STRING,
# MAGIC -- tcm_desc STRING,
# MAGIC -- vbp_desc STRING,
# MAGIC -- unit STRING,
# MAGIC -- counting_unit STRING,
# MAGIC -- dosage_unit STRING,
# MAGIC -- NRDL_ENTRY_DATE STRING,
# MAGIC -- bu STRING,
# MAGIC -- Market_Ratio STRING)
# MAGIC -- USING delta
# MAGIC -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/TMP/tmp_zk_retail_dtp_market_del'
# MAGIC -- ;
# COMMAND ----------
# MAGIC %sql
# MAGIC -- CREATE OR REPLACE TABLE tmp.tmp_zk_retail_dtp_market_corp (
# MAGIC -- -- product_id STRING,
# MAGIC -- -- region_type STRING,
# MAGIC -- -- higher_level_region STRING,
# MAGIC -- market STRING,
# MAGIC -- iqvia_pack_code STRING,
# MAGIC -- iqvia_prod_code STRING,
# MAGIC -- iqvia_notes STRING,
# MAGIC -- new_pack_flag STRING,
# MAGIC -- is_exists_chpa STRING,
# MAGIC -- -- prescription_nature STRING,
# MAGIC -- -- common_name STRING,
# MAGIC -- -- dosage_form STRING,
# MAGIC -- -- category_name STRING,
# MAGIC -- -- product_name STRING,
# MAGIC -- -- brand_name STRING,
# MAGIC -- -- zk_corp_des STRING,
# MAGIC -- -- zk_pack_des STRING,
# MAGIC -- -- counting_unit_a STRING,
# MAGIC -- app1_cod STRING,
# MAGIC -- app1_des STRING,
# MAGIC -- app1_des_c STRING,
# MAGIC -- app2_cod STRING,
# MAGIC -- app2_des STRING,
# MAGIC -- app2_des_c STRING,
# MAGIC -- app3_cod STRING,
# MAGIC -- app3_des STRING,
# MAGIC -- app3_des_c STRING,
# MAGIC -- atc1_cod STRING,
# MAGIC -- atc1_des STRING,
# MAGIC -- atc1_des_c STRING,
# MAGIC -- atc2_cod STRING,
# MAGIC -- atc2_des STRING,
# MAGIC -- atc2_des_c STRING,
# MAGIC -- atc3_cod STRING,
# MAGIC -- atc3_des STRING,
# MAGIC -- atc3_des_c STRING,
# MAGIC -- atc4_cod STRING,
# MAGIC -- atc4_des STRING,
# MAGIC -- atc4_des_c STRING,
# MAGIC -- bio_desc STRING,
# MAGIC -- cmps_cod STRING,
# MAGIC -- cmps_des STRING,
# MAGIC -- cmps_des_c STRING,
# MAGIC -- corp_cod STRING,
# MAGIC -- corp_des STRING,
# MAGIC -- corp_des_c STRING,
# MAGIC -- edl_desc STRING,
# MAGIC -- eth_otc_desc STRING,
# MAGIC -- gene_orig_desc STRING,
# MAGIC -- gqce_desc STRING,
# MAGIC -- manu_cod STRING,
# MAGIC -- manu_des STRING,
# MAGIC -- manu_des_c STRING,
# MAGIC -- mnfl_cod STRING,
# MAGIC -- mnfl_des STRING,
# MAGIC -- nrdl_desc STRING,
# MAGIC -- pack_des STRING,
# MAGIC -- stgh_des STRING,
# MAGIC -- pack_lch STRING,
# MAGIC -- paed_desc STRING,
# MAGIC -- prod_des STRING,
# MAGIC -- prod_des_c STRING,
# MAGIC -- tcm_desc STRING,
# MAGIC -- vbp_desc STRING,
# MAGIC -- unit STRING,
# MAGIC -- counting_unit STRING,
# MAGIC -- dosage_unit STRING,
# MAGIC -- NRDL_ENTRY_DATE STRING,
# MAGIC -- bu STRING,
# MAGIC -- Market_Ratio STRING)
# MAGIC -- USING delta
# MAGIC -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/TMP/tmp_zk_retail_dtp_market_corp'
# MAGIC -- ;
# COMMAND ----------
### constant
DATA_TYPE_MARKET = 'MARKET'
DATA_TYPE_MOLECULE = 'MOLECULE'
DATA_TYPE_BRAND = 'BRAND'
DATA_TYPE_MULTI_MARKET = 'MULTI_MARKET'
DATA_TYPE_MULTI_MOLECULE = 'MULTI_MOLECULE'
DATA_TYPE_MULTI_BRAND = 'MULTI_BRAND'
DATA_TYPE_DUPLICATE = 'DUPLICATE'
DATA_TYPE_HEDGE_TA = 'HEDGE_TA'
DATA_TYPE_HEDGE_MARKET = 'HEDGE_MARKET'
DATA_TYPE_HEDGE_MOLECULE = 'HEDGE_MOLECULE'
DATA_TYPE_HEDGE_BRAND = 'HEDGE_BRAND'
DATA_FLAG_RAW = 0 ### final pack_flag = 0
DATA_FLAG_SUB = 1 ### final pack_flag = 1
DATA_FLAG_ADD = 2 ### final pack_flag = 1
DATA_FLAG_RATIO = 3 ### final pack_flag = 1
DATA_FLAG_AVG = 4 ### final pack_flag = 1
DATA_FLAG_HEDGE = 5 ### final pack_flag = 1
BRAND_FLAG_NATIVE = 1 ### final brand_flag = 1
BRAND_FLAG_CALCULATE = 2 ### final brand_flag = 2
BRAND_FLAG_NONE = 3 ### final brand_flag = 2
DATA_OTHER_TA = 'OTHER_TA'
DATA_OTHER_MARKET = 'OTHER_MARKET'
DATA_OTHER_MOLECULE = 'OTHER_MOLECULE'
DATA_OTHER_BRAND = 'OTHER_BRAND'
DATA_AREA_TYPE_ROC = 'ROC'
DATA_LABEL_AREA_ALL = '全国'
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-1: dtp 维度表处理
# COMMAND ----------
############################################################START##############################################################
### STEP-1: dtp 维度表处理
### STEP-2: update the pack data extra column
### STEP-3: split regional data by brand ratio when there is no regional data and there is a brand ratio
### STEP-4: substract ROC data
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: dtp 维度表处理
# MAGIC -- 1.1 先处理 Extend_Market 和 NOT_IN_FLAG 都为空的数据，代表有明确定义得市场
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert overwrite table tmp.tmp_zk_retail_dtp_market
# MAGIC select
# MAGIC distinct
# MAGIC -- t1.region_type
# MAGIC -- ,t1.higher_level_region
# MAGIC -- t1.product_id,
# MAGIC t2.market
# MAGIC ,case when length(trim(t1.iqvia_pack_code)) < 12 and trim(t1.iqvia_pack_code) REGEXP '^[0-9]' then right(concat('000000000000',trim(t1.iqvia_pack_code)),12) else trim(t1.iqvia_pack_code) end as iqvia_pack_code
# MAGIC ,case when length(t1.iqvia_prod_code) < 9 then right(concat('000000000',t1.iqvia_prod_code),9) else t1.iqvia_prod_code end as iqvia_prod_code
# MAGIC ,t1.iqvia_notes
# MAGIC ,t1.new_pack_flag
# MAGIC ,t1.is_exists_chpa
# MAGIC -- ,t1.prescription_nature
# MAGIC -- ,t1.common_name
# MAGIC -- ,t1.dosage_form
# MAGIC -- ,t1.category_name
# MAGIC -- ,t1.product_name
# MAGIC -- ,t1.brand_name
# MAGIC -- ,t1.zk_corp_des
# MAGIC -- ,t1.zk_pack_des
# MAGIC -- ,t1.counting_unit_a
# MAGIC ,t1.app1_cod
# MAGIC ,t1.app1_des
# MAGIC ,t1.app1_des_c
# MAGIC ,t1.app2_cod
# MAGIC ,t1.app2_des
# MAGIC ,t1.app2_des_c
# MAGIC ,t1.app3_cod
# MAGIC ,t1.app3_des
# MAGIC ,t1.app3_des_c
# MAGIC ,t1.atc1_cod
# MAGIC ,t1.atc1_des
# MAGIC ,t1.atc1_des_c
# MAGIC ,t1.atc2_cod
# MAGIC ,t1.atc2_des
# MAGIC ,t1.atc2_des_c
# MAGIC ,t1.atc3_cod
# MAGIC ,t1.atc3_des
# MAGIC ,t1.atc3_des_c
# MAGIC ,t1.atc4_cod
# MAGIC ,t1.atc4_des
# MAGIC ,t1.atc4_des_c
# MAGIC ,t1.bio_desc
# MAGIC , right(concat('000000', t1.CMPS_COD ),6 ) cmps_cod
# MAGIC ,t1.cmps_des
# MAGIC ,t1.cmps_des_c
# MAGIC ,t1.corp_cod
# MAGIC ,t1.corp_des
# MAGIC ,t1.corp_des_c
# MAGIC ,t1.edl_desc
# MAGIC ,t1.eth_otc_desc
# MAGIC ,t1.gene_orig_desc
# MAGIC ,t1.gqce_desc
# MAGIC ,t1.manu_cod
# MAGIC ,t1.manu_des
# MAGIC ,t1.manu_des_c
# MAGIC ,t1.mnfl_cod
# MAGIC ,t1.mnfl_des
# MAGIC ,t1.nrdl_desc
# MAGIC ,t1.pack_des
# MAGIC ,t1.stgh_des
# MAGIC ,t1.pack_lch
# MAGIC ,t1.paed_desc
# MAGIC ,t1.prod_des
# MAGIC ,t1.prod_des_c
# MAGIC ,t1.tcm_desc
# MAGIC ,t1.vbp_desc
# MAGIC ,t1.unit
# MAGIC ,t1.counting_unit
# MAGIC ,t1.dosage_unit
# MAGIC ,nvl(MOLE.NRDL_Entry_Date,'') AS NRDL_ENTRY_DATE
# MAGIC ,t2.bu,CASE WHEN t2.extend_market_ratio IS NULL THEN '1' ELSE t2.extend_market_ratio END AS Market_Ratio
# MAGIC from dwd.dwd_gnd_ext_dtp_pack_property t1
# MAGIC LEFT JOIN dwd.dwd_ims_td_pack_additional_attribute MOLE
# MAGIC ----------------------------------------------------------------------------------
# MAGIC -- format iqvia_pack_code
# MAGIC ON
# MAGIC case when length(trim(t1.iqvia_pack_code)) < 12 and trim(t1.iqvia_pack_code) REGEXP '^[0-9]'
# MAGIC then right(concat('000000000000',trim(t1.iqvia_pack_code)),12)
# MAGIC else trim(t1.iqvia_pack_code)
# MAGIC end
# MAGIC =
# MAGIC case when length(trim(MOLE.Pack_Code)) < 12 and trim(MOLE.Pack_Code) REGEXP '^[0-9]'
# MAGIC then right(concat('000000000000',trim(MOLE.Pack_Code)),12)
# MAGIC else trim(MOLE.Pack_Code)
# MAGIC end
# MAGIC ----------------------------------------------------------------------------------
# MAGIC left join (select * from dwd.dwd_gnd_dtp_tblmarket WHERE Extend_Market IS NULL AND ( NOT_IN_FLAG IS NULL or NOT_IN_FLAG = '1' )) t2
# MAGIC -- on t1.cmps_cod=right(concat('000000', Molecule_Code),6 )
# MAGIC on t1.ATC1_COD = case when t2.ATC1_Code is null then t1.ATC1_COD else t2.ATC1_Code end
# MAGIC and t1.ATC2_COD = case when t2.ATC2_Code is null then t1.ATC2_COD else t2.ATC2_Code end
# MAGIC and t1.ATC3_COD = case when t2.ATC3_Code is null then t1.ATC3_COD else t2.ATC3_Code end
# MAGIC and t1.ATC4_COD = case when t2.ATC4_Code is null then t1.ATC4_COD else t2.ATC4_Code end
# MAGIC and t1.APP1_COD = case when t2.NFC1_Code is null then t1.APP1_COD else t2.NFC1_Code end
# MAGIC and t1.APP2_COD = case when t2.NFC2_Code is null then t1.APP2_COD else t2.NFC2_Code end
# MAGIC and t1.APP3_COD = case when t2.NFC3_Code is null then t1.APP3_COD else t2.NFC3_Code end
# MAGIC and t1.CORP_COD = case when t2.corporation_code is null then t1.CORP_COD else t2.corporation_code end
# MAGIC and t1.MANU_COD = case when t2.Manufacturer_Code is null then t1.MANU_COD else t2.Manufacturer_Code end
# MAGIC ----------------------------------------------------------------------------------
# MAGIC -- format iqvia_prod_code
# MAGIC and
# MAGIC case when length(t1.iqvia_prod_code) < 9 then right(concat('000000000',t1.iqvia_prod_code),9) else t1.iqvia_prod_code end
# MAGIC =
# MAGIC case when t2.Product_Code is null
# MAGIC then case when length(t1.iqvia_prod_code) < 9 then right(concat('000000000',t1.iqvia_prod_code),9) else t1.iqvia_prod_code end
# MAGIC else case when length(t2.Product_Code) < 9 then right(concat('000000000',t2.Product_Code),9) else t2.Product_Code end
# MAGIC end
# MAGIC ----------------------------------------------------------------------------------
# MAGIC -- format iqvia_pack_code
# MAGIC and
# MAGIC case when length(trim(t1.iqvia_pack_code)) < 12 and trim(t1.iqvia_pack_code) REGEXP '^[0-9]'
# MAGIC then right(concat('000000000000',trim(t1.iqvia_pack_code)),12)
# MAGIC else trim(t1.iqvia_pack_code)
# MAGIC end
# MAGIC =
# MAGIC case when t2.Pack_Code is null
# MAGIC then
# MAGIC case when length(trim(t1.iqvia_pack_code)) < 12 and trim(t1.iqvia_pack_code) REGEXP '^[0-9]'
# MAGIC then right(concat('000000000000',trim(t1.iqvia_pack_code)),12)
# MAGIC else trim(t1.iqvia_pack_code)
# MAGIC end
# MAGIC else
# MAGIC case when length(trim(t2.Pack_Code)) < 12 and trim(t2.Pack_Code) REGEXP '^[0-9]'
# MAGIC then right(concat('000000000000',trim(t2.Pack_Code)),12)
# MAGIC else trim(t2.Pack_Code)
# MAGIC end
# MAGIC end
# MAGIC ----------------------------------------------------------------------------------
# MAGIC and nvl(t1.STGH_DES,'') = case when t2.Strength is null then nvl(t1.STGH_DES,'') else t2.Strength end
# MAGIC and right(concat('000000', t1.CMPS_COD ),6 ) = case when t2.Molecule_Code is null then right(concat('000000', t1.CMPS_COD ),6 ) else right(concat('000000', Molecule_Code),6 ) end
# MAGIC where t2.market is not null
# MAGIC ;
# MAGIC
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: dtp 维度表处理
# MAGIC -- 1.2 反选规则
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert overwrite table tmp.tmp_zk_retail_dtp_market_del
# MAGIC select
# MAGIC distinct
# MAGIC -- t1.region_type
# MAGIC -- ,t1.higher_level_region
# MAGIC -- t1.product_id,
# MAGIC t2.market
# MAGIC ,case when length(trim(t1.iqvia_pack_code)) < 12 and trim(t1.iqvia_pack_code) REGEXP '^[0-9]' then right(concat('000000000000',trim(t1.iqvia_pack_code)),12) else trim(t1.iqvia_pack_code) end as iqvia_pack_code
# MAGIC ,case when length(t1.iqvia_prod_code) < 9 then right(concat('000000000',t1.iqvia_prod_code),9) else t1.iqvia_prod_code end as iqvia_prod_code
# MAGIC ,t1.iqvia_notes
# MAGIC ,t1.new_pack_flag
# MAGIC ,t1.is_exists_chpa
# MAGIC -- ,t1.prescription_nature
# MAGIC -- ,t1.common_name
# MAGIC -- ,t1.dosage_form
# MAGIC -- ,t1.category_name
# MAGIC -- ,t1.product_name
# MAGIC -- ,t1.brand_name
# MAGIC -- ,t1.zk_corp_des
# MAGIC -- ,t1.zk_pack_des
# MAGIC -- ,t1.counting_unit_a
# MAGIC ,t1.app1_cod
# MAGIC ,t1.app1_des
# MAGIC ,t1.app1_des_c
# MAGIC ,t1.app2_cod
# MAGIC ,t1.app2_des
# MAGIC ,t1.app2_des_c
# MAGIC ,t1.app3_cod
# MAGIC ,t1.app3_des
# MAGIC ,t1.app3_des_c
# MAGIC ,t1.atc1_cod
# MAGIC ,t1.atc1_des
# MAGIC ,t1.atc1_des_c
# MAGIC ,t1.atc2_cod
# MAGIC ,t1.atc2_des
# MAGIC ,t1.atc2_des_c
# MAGIC ,t1.atc3_cod
# MAGIC ,t1.atc3_des
# MAGIC ,t1.atc3_des_c
# MAGIC ,t1.atc4_cod
# MAGIC ,t1.atc4_des
# MAGIC ,t1.atc4_des_c
# MAGIC ,t1.bio_desc
# MAGIC , right(concat('000000', t1.CMPS_COD ),6 ) cmps_cod
# MAGIC ,t1.cmps_des
# MAGIC ,t1.cmps_des_c
# MAGIC ,t1.corp_cod
# MAGIC ,t1.corp_des
# MAGIC ,t1.corp_des_c
# MAGIC ,t1.edl_desc
# MAGIC ,t1.eth_otc_desc
# MAGIC ,t1.gene_orig_desc
# MAGIC ,t1.gqce_desc
# MAGIC ,t1.manu_cod
# MAGIC ,t1.manu_des
# MAGIC ,t1.manu_des_c
# MAGIC ,t1.mnfl_cod
# MAGIC ,t1.mnfl_des
# MAGIC ,t1.nrdl_desc
# MAGIC ,t1.pack_des
# MAGIC ,t1.stgh_des
# MAGIC ,t1.pack_lch
# MAGIC ,t1.paed_desc
# MAGIC ,t1.prod_des
# MAGIC ,t1.prod_des_c
# MAGIC ,t1.tcm_desc
# MAGIC ,t1.vbp_desc
# MAGIC ,t1.unit
# MAGIC ,t1.counting_unit
# MAGIC ,t1.dosage_unit
# MAGIC ,nvl(MOLE.NRDL_Entry_Date,'') AS NRDL_ENTRY_DATE
# MAGIC ,t2.bu,CASE WHEN t2.extend_market_ratio IS NULL THEN '1' ELSE t2.extend_market_ratio END AS Market_Ratio
# MAGIC from dwd.dwd_gnd_ext_dtp_pack_property t1
# MAGIC LEFT JOIN dwd.dwd_ims_td_pack_additional_attribute MOLE
# MAGIC ----------------------------------------------------------------------------------
# MAGIC -- format iqvia_pack_code
# MAGIC ON
# MAGIC case when length(trim(t1.iqvia_pack_code)) < 12 and trim(t1.iqvia_pack_code) REGEXP '^[0-9]'
# MAGIC then right(concat('000000000000',trim(t1.iqvia_pack_code)),12)
# MAGIC else trim(t1.iqvia_pack_code)
# MAGIC end
# MAGIC =
# MAGIC case when length(trim(MOLE.Pack_Code)) < 12 and trim(MOLE.Pack_Code) REGEXP '^[0-9]'
# MAGIC then right(concat('000000000000',trim(MOLE.Pack_Code)),12)
# MAGIC else trim(MOLE.Pack_Code)
# MAGIC end
# MAGIC ----------------------------------------------------------------------------------
# MAGIC left join (select * from dwd.dwd_gnd_dtp_tblmarket WHERE Extend_Market IS NULL AND NOT_IN_FLAG = '0' ) t2
# MAGIC -- on t1.cmps_cod=right(concat('000000', Molecule_Code),6 )
# MAGIC on t1.ATC1_COD = case when t2.ATC1_Code is null then t1.ATC1_COD else t2.ATC1_Code end
# MAGIC and t1.ATC2_COD = case when t2.ATC2_Code is null then t1.ATC2_COD else t2.ATC2_Code end
# MAGIC and t1.ATC3_COD = case when t2.ATC3_Code is null then t1.ATC3_COD else t2.ATC3_Code end
# MAGIC and t1.ATC4_COD = case when t2.ATC4_Code is null then t1.ATC4_COD else t2.ATC4_Code end
# MAGIC and t1.APP1_COD = case when t2.NFC1_Code is null then t1.APP1_COD else t2.NFC1_Code end
# MAGIC and t1.APP2_COD = case when t2.NFC2_Code is null then t1.APP2_COD else t2.NFC2_Code end
# MAGIC and t1.APP3_COD = case when t2.NFC3_Code is null then t1.APP3_COD else t2.NFC3_Code end
# MAGIC and t1.CORP_COD = case when t2.corporation_code is null then t1.CORP_COD else t2.corporation_code end
# MAGIC and t1.MANU_COD = case when t2.Manufacturer_Code is null then t1.MANU_COD else t2.Manufacturer_Code end
# MAGIC ----------------------------------------------------------------------------------
# MAGIC -- format iqvia_prod_code
# MAGIC and
# MAGIC case when length(t1.iqvia_prod_code) < 9 then right(concat('000000000',t1.iqvia_prod_code),9) else t1.iqvia_prod_code end
# MAGIC =
# MAGIC case when t2.Product_Code is null
# MAGIC then case when length(t1.iqvia_prod_code) < 9 then right(concat('000000000',t1.iqvia_prod_code),9) else t1.iqvia_prod_code end
# MAGIC else case when length(t2.Product_Code) < 9 then right(concat('000000000',t2.Product_Code),9) else t2.Product_Code end
# MAGIC end
# MAGIC ----------------------------------------------------------------------------------
# MAGIC -- format iqvia_pack_code
# MAGIC and
# MAGIC case when length(trim(t1.iqvia_pack_code)) < 12 and trim(t1.iqvia_pack_code) REGEXP '^[0-9]'
# MAGIC then right(concat('000000000000',trim(t1.iqvia_pack_code)),12)
# MAGIC else trim(t1.iqvia_pack_code)
# MAGIC end
# MAGIC =
# MAGIC case when t2.Pack_Code is null
# MAGIC then
# MAGIC case when length(trim(t1.iqvia_pack_code)) < 12 and trim(t1.iqvia_pack_code) REGEXP '^[0-9]'
# MAGIC then right(concat('000000000000',trim(t1.iqvia_pack_code)),12)
# MAGIC else trim(t1.iqvia_pack_code)
# MAGIC end
# MAGIC else
# MAGIC case when length(trim(t2.Pack_Code)) < 12 and trim(t2.Pack_Code) REGEXP '^[0-9]'
# MAGIC then right(concat('000000000000',trim(t2.Pack_Code)),12)
# MAGIC else trim(t2.Pack_Code)
# MAGIC end
# MAGIC end
# MAGIC ----------------------------------------------------------------------------------
# MAGIC and t1.STGH_DES = case when t2.Strength is null then t1.STGH_DES else t2.Strength end
# MAGIC and right(concat('000000', t1.CMPS_COD ),6 ) = case when t2.Molecule_Code is null then right(concat('000000', t1.CMPS_COD ),6 ) else right(concat('000000', Molecule_Code),6 ) end
# MAGIC where t2.market is not null
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: dtp 维度表处理
# MAGIC -- 1.3 ！！！注意此段是反选的规则，故执行的是从已经定义好的TempMKT中删除数据
# MAGIC -------需要看TempMKT的字段结构，没有值的字段，在Value里面放个空值
# MAGIC -------------------------------------------------------------------------------------
# MAGIC MERGE INTO tmp.tmp_zk_retail_dtp_market AS t1
# MAGIC USING tmp.tmp_zk_retail_dtp_market_del AS t2
# MAGIC on t1.ATC1_COD = t2.ATC1_COD
# MAGIC and t1.ATC2_COD = t2.ATC2_COD
# MAGIC and t1.ATC3_COD = t2.ATC3_COD
# MAGIC and t1.ATC4_COD = t2.ATC4_COD
# MAGIC and t1.APP1_COD = t2.APP1_COD
# MAGIC and t1.APP2_COD = t2.APP2_COD
# MAGIC and t1.APP3_COD = t2.APP3_COD
# MAGIC and t1.CORP_COD = t2.CORP_COD
# MAGIC and t1.MANU_COD = t2.MANU_COD
# MAGIC and t1.iqvia_prod_code = t2.iqvia_prod_code
# MAGIC and t1.iqvia_pack_code = t2.iqvia_pack_code
# MAGIC and t1.STGH_DES = t2.STGH_DES
# MAGIC and t1.CMPS_COD = t2.CMPS_COD
# MAGIC and t1.market = t2.market
# MAGIC WHEN MATCHED THEN DELETE
# MAGIC ;
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: dtp 维度表处理
# MAGIC -- 1.4 将产品对应的公司重新归属
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert overwrite table tmp.tmp_zk_retail_dtp_market_corp
# MAGIC select
# MAGIC -- A.region_type
# MAGIC -- ,A.higher_level_region
# MAGIC -- A.product_id,
# MAGIC A.market
# MAGIC ,A.iqvia_pack_code
# MAGIC ,A.iqvia_prod_code
# MAGIC ,A.iqvia_notes
# MAGIC ,A.new_pack_flag
# MAGIC ,A.is_exists_chpa
# MAGIC -- ,A.prescription_nature
# MAGIC -- ,A.common_name
# MAGIC -- ,A.dosage_form
# MAGIC -- ,A.category_name
# MAGIC -- ,A.product_name
# MAGIC -- ,A.brand_name
# MAGIC -- ,A.zk_corp_des
# MAGIC -- ,A.zk_pack_des
# MAGIC -- ,A.counting_unit_a
# MAGIC ,A.app1_cod
# MAGIC ,A.app1_des
# MAGIC ,A.app1_des_c
# MAGIC ,A.app2_cod
# MAGIC ,A.app2_des
# MAGIC ,A.app2_des_c
# MAGIC ,A.app3_cod
# MAGIC ,A.app3_des
# MAGIC ,A.app3_des_c
# MAGIC ,A.atc1_cod
# MAGIC ,A.atc1_des
# MAGIC ,A.atc1_des_c
# MAGIC ,A.atc2_cod
# MAGIC ,A.atc2_des
# MAGIC ,A.atc2_des_c
# MAGIC ,A.atc3_cod
# MAGIC ,A.atc3_des
# MAGIC ,A.atc3_des_c
# MAGIC ,A.atc4_cod
# MAGIC ,A.atc4_des
# MAGIC ,A.atc4_des_c
# MAGIC ,A.bio_desc
# MAGIC ,A.cmps_cod
# MAGIC ,A.cmps_des
# MAGIC ,A.cmps_des_c
# MAGIC ,COALESCE(c.corp_cod,b.corp_cod,a.corp_cod) corp_cod
# MAGIC ,COALESCE(c.corp_des,b.corp_des,a.corp_des) corp_des
# MAGIC ,COALESCE(d.corp_des_c,a.corp_des_c ) corp_des_c
# MAGIC ,A.edl_desc
# MAGIC ,A.eth_otc_desc
# MAGIC ,A.gene_orig_desc
# MAGIC ,A.gqce_desc
# MAGIC ,COALESCE(c.manu_cod,b.manu_cod,a.manu_cod) manu_cod
# MAGIC ,COALESCE(c.manu_des,b.manu_des,a.manu_des) manu_des
# MAGIC ,COALESCE(e.manu_des_c,a.manu_des_c) manu_des_c
# MAGIC ,A.mnfl_cod
# MAGIC ,A.mnfl_des
# MAGIC ,A.nrdl_desc
# MAGIC ,A.pack_des
# MAGIC ,A.stgh_des
# MAGIC ,A.pack_lch
# MAGIC ,A.paed_desc
# MAGIC ,A.prod_des
# MAGIC ,A.prod_des_c
# MAGIC ,A.tcm_desc
# MAGIC ,A.vbp_desc
# MAGIC ,A.unit
# MAGIC ,A.counting_unit
# MAGIC ,A.dosage_unit
# MAGIC ,A.NRDL_ENTRY_DATE
# MAGIC ,A.bu
# MAGIC ,A.Market_Ratio
# MAGIC from tmp.tmp_zk_retail_dtp_market a
# MAGIC left join (
# MAGIC select
# MAGIC DISTINCT
# MAGIC ----------------------------------------------
# MAGIC -- format prod_cod
# MAGIC case when length(prod_cod) < 9 then right(concat('000000000',prod_cod),9) else prod_cod end as prod_cod,
# MAGIC ----------------------------------------------
# MAGIC --right(concat('00000',prod_cod ),5) prod_cod,
# MAGIC corp_cod,
# MAGIC corp_des,
# MAGIC right(concat('000000',manu_cod ),6) manu_cod,
# MAGIC manu_des
# MAGIC from dwd.dwd_gnd_tbl_corp_change
# MAGIC where pack_cod is null
# MAGIC ) b on a.iqvia_prod_code = b.prod_cod
# MAGIC left join (
# MAGIC select
# MAGIC DISTINCT
# MAGIC ----------------------------------------------
# MAGIC -- format pack_cod
# MAGIC case when length(trim(pack_cod)) < 12 and trim(pack_cod) REGEXP '^[0-9]' then right(concat('000000000000',trim(pack_cod)),12) else trim(pack_cod) end as pack_cod,
# MAGIC ----------------------------------------------
# MAGIC corp_cod,
# MAGIC corp_des,
# MAGIC right(concat('000000',manu_cod ),6) manu_cod,
# MAGIC manu_des
# MAGIC from dwd.dwd_gnd_tbl_corp_change
# MAGIC where pack_cod is not null
# MAGIC ) c on a.iqvia_pack_code = c.pack_cod
# MAGIC left join (select distinct corp_cod,corp_des_c from dwd.dwd_gnd_ext_dtp_pack_property) d
# MAGIC on coalesce(C.corp_cod,B.corp_cod ) = d.corp_cod
# MAGIC Left join (select distinct right(concat('000000',manu_cod ),6) manu_cod,manu_des,manu_des_c from dwd.dwd_gnd_ext_dtp_pack_property) e
# MAGIC On coalesce(C.manu_cod,B.manu_cod ) = e.manu_cod
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-2: update the pack data extra column
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-2: update the pack data extra column
# MAGIC -- 2.1 update pack_code, molecule_code, molecule_desc, product_code, product_desc, level_market
# MAGIC -------------------------------------------------------------------------------------
# MAGIC merge into tmp.tmp_retail_dtp_pack_rawdata a
# MAGIC using(
# MAGIC select
# MAGIC distinct
# MAGIC iqvia_pack_code,
# MAGIC -- common_name,
# MAGIC -- dosage_form,
# MAGIC -- category_name,
# MAGIC -- product_name,
# MAGIC -- brand_name,
# MAGIC -- zk_pack_des,
# MAGIC cmps_des,
# MAGIC cmps_des_c,
# MAGIC prod_des,
# MAGIC prod_des_c,
# MAGIC collect_list(market) as markets
# MAGIC from tmp.tmp_zk_retail_dtp_market_corp
# MAGIC group by 1,2,3,4,5
# MAGIC ) as b
# MAGIC on a.iqvia_pack_code = b.iqvia_pack_code
# MAGIC -- on a.common_name = b.common_name
# MAGIC -- and a.dosage_form = b.dosage_form
# MAGIC -- and a.category_name = b.category_name
# MAGIC -- and nvl(a.product_name, '') = nvl(b.product_name, '')
# MAGIC -- and a.brand_name = b.brand_name
# MAGIC -- and a.pack_des = b.zk_pack_des
# MAGIC when matched then
# MAGIC update set
# MAGIC a.pack_code = b.iqvia_pack_code,
# MAGIC a.molecule_code = b.cmps_des,
# MAGIC a.molecule_desc = b.cmps_des_c,
# MAGIC a.product_code = b.prod_des,
# MAGIC a.product_desc = b.prod_des_c,
# MAGIC --------------------* hard_code *-----------------------
# MAGIC -- 手动修正market映射市场
# MAGIC -- EGFR TKI Market -> EGFR TKI
# MAGIC -- Anti-HER2 mBC Market -> Anti-HER2 mBC
# MAGIC a.level_market =
# MAGIC case
# MAGIC when array_contains(markets, 'EGFR TKI Market') then 'EGFR TKI'
# MAGIC when array_contains(markets, 'Anti-HER2 mBC Market') then 'Anti-HER2 mBC'
# MAGIC else CONCAT_WS(',',markets)
# MAGIC end
# MAGIC --------------------* hard_code *-----------------------
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-2: update the pack data extra column
# MAGIC -- 2.2 generate tmp.tmp_retail_dtp_pack_mapping
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC with tmp_pack_mapping as (
# MAGIC select
# MAGIC distinct
# MAGIC a.level_ta,
# MAGIC a.level_market,
# MAGIC a.pack_code,
# MAGIC a.molecule_code,
# MAGIC a.molecule_desc,
# MAGIC a.product_code,
# MAGIC a.product_desc,
# MAGIC b.corp_cod,
# MAGIC CONCAT_WS(',',collect_set(b.market)) as market_desc
# MAGIC from tmp.tmp_retail_dtp_pack_rawdata a
# MAGIC inner join tmp.tmp_zk_retail_dtp_market_corp b
# MAGIC on a.pack_code = b.iqvia_pack_code
# MAGIC and a.molecule_code = b.cmps_des
# MAGIC and a.product_desc = b.prod_des_c
# MAGIC group by 1,2,3,4,5,6,7,8
# MAGIC ), tmp_level as (
# MAGIC select
# MAGIC distinct
# MAGIC ta,
# MAGIC market,
# MAGIC molecule,
# MAGIC brand
# MAGIC from tmp.tmp_retail_dtp_level_brand
# MAGIC ), tmp_molecule_brand as (
# MAGIC -----------------------ta matched & market matched & molecule matched & brand matched --------------
# MAGIC select
# MAGIC a.*,b.*,1 as brand_flag
# MAGIC from tmp_pack_mapping a
# MAGIC inner join tmp_level b
# MAGIC on a.level_ta = b.ta
# MAGIC and a.level_market = b.market
# MAGIC and a.molecule_desc = b.molecule
# MAGIC and a.product_desc = b.brand
# MAGIC ), tmp_other_brand as (
# MAGIC -----------------------ta matched & market matched & OTHER_MOLECULE & brand matched-----------------
# MAGIC select
# MAGIC a.*,b.*,1 as brand_flag
# MAGIC from tmp_pack_mapping a
# MAGIC inner join tmp_level b
# MAGIC on a.level_ta = b.ta
# MAGIC and a.level_market = b.market
# MAGIC and b.molecule = 'OTHER_MOLECULE'
# MAGIC and a.product_desc = b.brand
# MAGIC where a.pack_code not in (
# MAGIC select distinct pack_code from tmp_molecule_brand
# MAGIC )
# MAGIC ), tmp_molecule_other as(
# MAGIC -----------------------ta matched & market matched & molecule matched & OTHER_BRAND----------------
# MAGIC select
# MAGIC a.*,b.*,2 as brand_flag
# MAGIC from tmp_pack_mapping a
# MAGIC inner join tmp_level b
# MAGIC on a.level_ta = b.ta
# MAGIC and a.level_market = b.market
# MAGIC and a.molecule_desc = b.molecule
# MAGIC and b.brand = 'OTHER_BRAND'
# MAGIC where a.pack_code not in (
# MAGIC select distinct pack_code from tmp_molecule_brand
# MAGIC union all
# MAGIC select distinct pack_code from tmp_other_brand
# MAGIC )
# MAGIC ), tmp_other_other as (
# MAGIC -----------------------ta matched & market matched & OTHER_MOLECULE& OTHER_BRAND---------------------
# MAGIC select
# MAGIC a.*,b.*,2 as brand_flag
# MAGIC from tmp_pack_mapping a
# MAGIC inner join tmp_level b
# MAGIC on a.level_ta = b.ta
# MAGIC and a.level_market = b.market
# MAGIC and b.molecule = 'OTHER_MOLECULE'
# MAGIC and b.brand = 'OTHER_BRAND'
# MAGIC where a.pack_code not in (
# MAGIC select distinct pack_code from tmp_molecule_brand
# MAGIC union all
# MAGIC select distinct pack_code from tmp_other_brand
# MAGIC union all
# MAGIC select distinct pack_code from tmp_molecule_other
# MAGIC )
# MAGIC ), tmp_brand_mapping as (
# MAGIC select * from tmp_molecule_brand
# MAGIC union all
# MAGIC select * from tmp_other_brand
# MAGIC union all
# MAGIC select * from tmp_molecule_other
# MAGIC union all
# MAGIC select * from tmp_other_other
# MAGIC )
# MAGIC
# MAGIC insert overwrite table tmp.tmp_retail_dtp_pack_mapping
# MAGIC
# MAGIC select
# MAGIC level_ta,
# MAGIC level_market,
# MAGIC molecule as level_molecule,
# MAGIC brand as level_brand,
# MAGIC pack_code,
# MAGIC molecule_code,
# MAGIC molecule_desc,
# MAGIC product_code,
# MAGIC product_desc,
# MAGIC corp_cod,
# MAGIC market_desc,
# MAGIC brand_flag
# MAGIC from tmp_brand_mapping
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-2: update the pack data extra column
# MAGIC -- 2.3 update level_molecule, level_brand ,brand_flag
# MAGIC -------------------------------------------------------------------------------------
# MAGIC merge into tmp.tmp_retail_dtp_pack_rawdata a
# MAGIC using(
# MAGIC select
# MAGIC distinct pack_code,level_molecule,level_brand,brand_flag
# MAGIC from tmp.tmp_retail_dtp_pack_mapping
# MAGIC ) as b
# MAGIC on a.pack_code = b.pack_code
# MAGIC when matched then
# MAGIC update set
# MAGIC a.level_molecule = b.level_molecule,
# MAGIC a.level_brand = b.level_brand,
# MAGIC a.brand_flag = b.brand_flag
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-2: update the pack data extra column
# MAGIC -- 2.4 update the pack data brand flag to '1' which pack has regional data
# MAGIC -------------------------------------------------------------------------------------
# MAGIC with tmp_has_regional_data as (
# MAGIC select
# MAGIC distinct
# MAGIC time,
# MAGIC pack_code
# MAGIC from tmp.tmp_retail_dtp_pack_rawdata
# MAGIC where region <> '全国'
# MAGIC and data_flag = 0
# MAGIC )
# MAGIC
# MAGIC merge into tmp.tmp_retail_dtp_pack_rawdata a
# MAGIC using(
# MAGIC select * from tmp_has_regional_data
# MAGIC ) as b
# MAGIC on a.time = b.time
# MAGIC and a.pack_code = b.pack_code
# MAGIC when matched then
# MAGIC update set
# MAGIC a.brand_flag = 1
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-3: split regional data by brand ratio
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-3: split regional data by brand ratio when there is no regional data and there is a brand ratio
# MAGIC -- only split the pack data with brand ratio and without regional pack data
# MAGIC -------------------------------------------------------------------------------------
# MAGIC with tmp_has_regional_data as (
# MAGIC select
# MAGIC distinct
# MAGIC time as yyyymm,
# MAGIC pack_code
# MAGIC from tmp.tmp_retail_dtp_pack_rawdata
# MAGIC where region <> '全国'
# MAGIC ), tmp_has_brand_ratio as (
# MAGIC select
# MAGIC distinct
# MAGIC b.pack_code
# MAGIC from tmp.tmp_retail_dtp_level_brand a
# MAGIC inner join tmp.tmp_retail_dtp_pack_mapping b
# MAGIC on a.ta = b.level_ta
# MAGIC and a.market = b.level_market
# MAGIC and a.molecule = b.level_molecule
# MAGIC and a.brand = b.level_brand
# MAGIC ), tmp_pack_rawdata as (
# MAGIC select *
# MAGIC from tmp.tmp_retail_dtp_pack_rawdata a
# MAGIC where not exists(
# MAGIC select b.yyyymm,b.pack_code from tmp_has_regional_data b
# MAGIC where a.time = b.yyyymm
# MAGIC and a.pack_code = b.pack_code
# MAGIC ) and exists(
# MAGIC select c.pack_code from tmp_has_brand_ratio c
# MAGIC where a.pack_code = c.pack_code
# MAGIC )
# MAGIC )
# MAGIC ----------------------------- append split regional data ----------------------------
# MAGIC insert into table tmp.tmp_retail_dtp_pack_rawdata
# MAGIC
# MAGIC select
# MAGIC -- c.region_type,
# MAGIC c.iqvia_pack_code,
# MAGIC c.year,
# MAGIC c.time,
# MAGIC -- c.higher_level_region,
# MAGIC a.province_city as region,
# MAGIC -- c.prescription_nature,
# MAGIC -- c.medicine_attribute,
# MAGIC -- c.dosage_form,
# MAGIC -- c.object,
# MAGIC -- c.zk_classify1,
# MAGIC -- c.zk_classify2,
# MAGIC -- c.zk_classify3,
# MAGIC c.target_points,
# MAGIC -- c.common_name,
# MAGIC -- c.brand_name,
# MAGIC -- c.product_name,
# MAGIC -- c.category_name,
# MAGIC -- c.pack_des,
# MAGIC c.counting_unit,
# MAGIC -- c.factory,
# MAGIC -- c.corp_des,
# MAGIC c.average_price,
# MAGIC round(c.sales_amount * a.ratio_val ,10) as sales_amount,
# MAGIC round(c.sales_volume * a.ratio_vol ,10) as sales_volume,
# MAGIC c.counting_units_obversion,
# MAGIC c.pack_code,
# MAGIC c.molecule_code,
# MAGIC c.molecule_desc,
# MAGIC c.product_code,
# MAGIC c.product_desc,
# MAGIC c.level_ta,
# MAGIC c.level_market,
# MAGIC c.level_molecule,
# MAGIC c.level_brand,
# MAGIC a.ratio_val,
# MAGIC a.ratio_vol,
# MAGIC 3 as data_flag,
# MAGIC b.brand_flag
# MAGIC from tmp.tmp_retail_dtp_level_brand a
# MAGIC inner join tmp.tmp_retail_dtp_pack_mapping b
# MAGIC on a.ta = b.level_ta
# MAGIC and a.market = b.level_market
# MAGIC and a.molecule = b.level_molecule
# MAGIC and a.brand = b.level_brand
# MAGIC inner join tmp_pack_rawdata c
# MAGIC on c.pack_code = b.pack_code
# MAGIC and c.time = a.yyyymm
# MAGIC where a.province_city <> '全国'
# MAGIC
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-4: substract ROC data
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-4: substract ROC data
# MAGIC -- if there is regional pack data or splited regional pack data, ROC = '全国' - 'sum regional data'
# MAGIC -- if there is not regional pack data, ROC = '全国'
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC with tmp_area_group as (
# MAGIC select
# MAGIC pack_code,
# MAGIC time,
# MAGIC sum(sales_amount) as sales_amount,
# MAGIC sum(sales_volume) as sales_volume
# MAGIC from tmp.tmp_retail_dtp_pack_rawdata
# MAGIC where region <> '全国'
# MAGIC group by pack_code,time
# MAGIC ), tmp_has_regional_data as (
# MAGIC select
# MAGIC distinct
# MAGIC time,
# MAGIC pack_code
# MAGIC from tmp.tmp_retail_dtp_pack_rawdata
# MAGIC where region <> '全国'
# MAGIC and data_flag = 0
# MAGIC )
# MAGIC
# MAGIC ----------------------------- append ROC data ------------------------------------------
# MAGIC insert into table tmp.tmp_retail_dtp_pack_rawdata
# MAGIC
# MAGIC select
# MAGIC a.iqvia_pack_code,
# MAGIC -- a.region_type,
# MAGIC a.year,
# MAGIC a.time,
# MAGIC -- a.higher_level_region,
# MAGIC 'ROC' as region,
# MAGIC -- a.prescription_nature,
# MAGIC -- a.medicine_attribute,
# MAGIC -- a.dosage_form,
# MAGIC -- a.object,
# MAGIC -- a.zk_classify1,
# MAGIC -- a.zk_classify2,
# MAGIC -- a.zk_classify3,
# MAGIC a.target_points,
# MAGIC -- a.common_name,
# MAGIC -- a.brand_name,
# MAGIC -- a.product_name,
# MAGIC -- a.category_name,
# MAGIC -- a.pack_des,
# MAGIC a.counting_unit,
# MAGIC -- a.factory,
# MAGIC -- a.corp_des,
# MAGIC a.average_price,
# MAGIC a.sales_amount - ifnull(b.sales_amount,0) as sales_amount,
# MAGIC a.sales_volume - ifnull(b.sales_volume,0) as sales_volume,
# MAGIC a.counting_units_obversion,
# MAGIC a.pack_code,
# MAGIC a.molecule_code,
# MAGIC a.molecule_desc,
# MAGIC a.product_code,
# MAGIC a.product_desc,
# MAGIC a.level_ta,
# MAGIC a.level_market,
# MAGIC a.level_molecule,
# MAGIC a.level_brand,
# MAGIC a.ratio_val,
# MAGIC a.ratio_vol,
# MAGIC -------------------------------------------------------------------------------
# MAGIC -- 有省份pack的数（包括无品牌数据的数）计算出来的ROC所对应的pack_flag，也需要是1-原始值，不能是2-拆分的值
# MAGIC -- 所以这里针对这类有省份数据的ROC数据，data_flag赋值为0-原始值，其他的仍为1-倒减值
# MAGIC case when c.pack_code is not null then 0 else 1 end as data_flag,
# MAGIC -------------------------------------------------------------------------------
# MAGIC a.brand_flag
# MAGIC from tmp.tmp_retail_dtp_pack_rawdata a
# MAGIC inner join tmp_area_group b
# MAGIC on a.pack_code = b.pack_code
# MAGIC and a.time = b.time
# MAGIC left join tmp_has_regional_data c
# MAGIC on a.time = c.time
# MAGIC and a.pack_code = c.pack_code
# MAGIC where a.region = '全国'
# COMMAND ----------
############################################################END################################################################
Reference in New Issue View Git Blame Copy Permalink