chenwu/MarketAnalysis-ETL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
3e14e78af3b9ae914e8ea866e316ee6f7c34de98
MarketAnalysis-ETL/Retail/11 map_to_overview_dm_table.py
chenwu 3e14e78af3 update20260427
2026-04-27 11:04:09 +08:00

1000 lines
35 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# Databricks notebook source
############################################################START##############################################################
### STEP-10: map to dm.dm_zk_retail_overview_data
### STEP-11: map to dm.dm_zk_retail_product_info
### STEP-12: map to dm.dm_zk_retail_brand
### STEP-13: map to dm.dm_zk_retail_export
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-10: map to dm.dm_zk_retail_overview_data
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-10: map to dm.dm_zk_retail_overview_data
# MAGIC -- 10.1 将所有overview 数据整合在同一张表里
# MAGIC
# MAGIC -- -- 注意事项：
# MAGIC -- --rank1字段：Total行排名设为99，其他行如果数据源是null，则设为98。总之不能为null。
# MAGIC -- -- 每个数据源文件（file_name）判断是否Total行的方式不尽相同，具体参考代码。
# MAGIC -- --sales_quarter字段：数据源中有两种格式，24Q1以及2024Q1，因此需要统一成2024Q1这种格式。
# MAGIC -- -- 有些因为是YTD的值，源文件里sales_quarter是null，需将YTD字段中的时间拆出来作为sales_quarter字段的值
# MAGIC -- --最终表的source_name字段的内容是各个TA，RC字段是南区、北区等。而数据源文件中都叫source_name，每个源文件需按实际情况拆分成2列
# MAGIC -- --yyyymm字段：Q1是3月、Q2是6月、Q3是9月、Q4是12月。
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC insert overwrite table dws.dws_zk_retail_overview_data
# MAGIC (
# MAGIC rank1
# MAGIC ,product_desc
# MAGIC ,product_type
# MAGIC ,common_name
# MAGIC ,brand
# MAGIC ,corp
# MAGIC ,corporation
# MAGIC ,corp_type
# MAGIC ,sales_quarter
# MAGIC ,yyyymm
# MAGIC ,ytd
# MAGIC ,sales_amount
# MAGIC ,sales_amount_ly
# MAGIC ,sales_amount_total
# MAGIC ,sales_amount_ly_total
# MAGIC ,rx_type
# MAGIC ,top1_brand_val
# MAGIC ,top1_brand_gr
# MAGIC ,top2_brand_val
# MAGIC ,top2_brand_gr
# MAGIC ,top1_incremental_brand
# MAGIC ,top1_incremental_gr
# MAGIC ,top2_incremental_brand
# MAGIC ,top2_incremental_gr
# MAGIC ,ytd_gr
# MAGIC ,ytd_ms
# MAGIC ,ytd_delta_ms
# MAGIC ,dtp_name
# MAGIC ,file_name
# MAGIC ,source_name
# MAGIC ,rc
# MAGIC ,rx_otc
# MAGIC ,top_incre
# MAGIC ,etl_insert_dt
# MAGIC ,etl_update_dt
# MAGIC )
# MAGIC
# MAGIC
# MAGIC with overview as
# MAGIC (
# MAGIC select
# MAGIC case when replace(upper(corp_name),'-','') = 'DTP TOTAL' then '99'
# MAGIC when rank1 is null then '98'
# MAGIC else replace(rank1,' ','') end as rank1
# MAGIC ,null as product_desc
# MAGIC ,null as product_type
# MAGIC ,null as common_name
# MAGIC ,null as brand
# MAGIC ,replace(corp_name,'-','') as corp
# MAGIC ,null as corporation
# MAGIC ,null as corp_type
# MAGIC ,case when length(sales_quarter) = 4 then concat('20',sales_quarter)
# MAGIC else sales_quarter
# MAGIC end as sales_quarter
# MAGIC ,null as ytd
# MAGIC ,sales_amount
# MAGIC ,null as rx_type
# MAGIC ,null as top1_brand_val
# MAGIC ,null as top1_brand_gr
# MAGIC ,null as top2_brand_val
# MAGIC ,null as top2_brand_gr
# MAGIC ,null as top1_incremental_brand
# MAGIC ,null as top1_incremental_gr
# MAGIC ,null as top2_incremental_brand
# MAGIC ,null as top2_incremental_gr
# MAGIC ,null as ytd_gr
# MAGIC ,null as ytd_ms
# MAGIC ,null as ytd_delta_ms
# MAGIC ,null as dtp_name
# MAGIC ,'DTP TOP10企业排名' file_name
# MAGIC ,NULL as source_name
# MAGIC ,null rc
# MAGIC ,'Rx' as rx_otc
# MAGIC ,'TOP' as top_incre
# MAGIC from dwd.dwd_gnd_ext_retail_dtp_top_copd ---这个没有ytd 三个字段添加
# MAGIC union all
# MAGIC select
# MAGIC case when replace(upper(brand_name),'-','') = 'DTP TOTAL' then '99'
# MAGIC when rank1 is null then '98'
# MAGIC else replace(rank1,' ','') end as rank1
# MAGIC ,null as product_desc
# MAGIC ,null as product_type
# MAGIC ,null as common_name
# MAGIC ,brand_name as brand
# MAGIC ,replace(corp_name,'-','') as corp
# MAGIC ,null as corporation
# MAGIC ,null as corp_type
# MAGIC ,case when length(sales_quarter) = 4 then concat('20',sales_quarter)
# MAGIC else sales_quarter
# MAGIC end as sales_quarter
# MAGIC ,null as ytd
# MAGIC ,sales_amount
# MAGIC ,null as rx_type
# MAGIC ,null as top1_brand_val
# MAGIC ,null as top1_brand_gr
# MAGIC ,null as top2_brand_val
# MAGIC ,null as top2_brand_gr
# MAGIC ,null as top1_incremental_brand
# MAGIC ,null as top1_incremental_gr
# MAGIC ,null as top2_incremental_brand
# MAGIC ,null as top2_incremental_gr
# MAGIC ,null as ytd_gr
# MAGIC ,null as ytd_ms
# MAGIC ,null as ytd_delta_ms
# MAGIC ,null as dtp_name
# MAGIC ,'DTP TOP10品牌排名' file_name
# MAGIC ,NULL as source_name
# MAGIC ,null rc
# MAGIC ,'Rx' as rx_otc
# MAGIC ,'TOP' as top_incre
# MAGIC from dwd.dwd_gnd_ext_retail_dtp_top_brand ---这个没有ytd 三个字段添加
# MAGIC union all
# MAGIC
# MAGIC select
# MAGIC case when upper(replace(rank1,' ','')) = 'TOTAL' then '99'
# MAGIC when rank1 is null then '98'
# MAGIC else replace(rank1,' ','') end as rank1
# MAGIC ,null as product_desc
# MAGIC ,null as product_type
# MAGIC ,null as common_name
# MAGIC ,null as brand
# MAGIC ,corp_desc as corp
# MAGIC ,corporation
# MAGIC ,`type` as corp_type
# MAGIC ,case when length(sales_quarter) = 4 then concat('20',sales_quarter)
# MAGIC else sales_quarter
# MAGIC end as sales_quarter
# MAGIC ,null as ytd
# MAGIC ,sales_amount
# MAGIC ,null as rx_type
# MAGIC ,null as top1_brand_val
# MAGIC ,null as top1_brand_gr
# MAGIC ,null as top2_brand_val
# MAGIC ,null as top2_brand_gr
# MAGIC ,null as top1_incremental_brand
# MAGIC ,null as top1_incremental_gr
# MAGIC ,null as top2_incremental_brand
# MAGIC ,null as top2_incremental_gr
# MAGIC ,null as ytd_gr
# MAGIC ,null as ytd_ms
# MAGIC ,null as ytd_delta_ms
# MAGIC ,null as dtp_name
# MAGIC ,'全国-集团排名' file_name
# MAGIC ,case when ta = 'GI' then concat(ta,'-',rx_otc)
# MAGIC when ta = 'NIAD-excl. GLP1' then 'DM'
# MAGIC else ta end as source_name
# MAGIC ,null rc
# MAGIC ,rx_otc
# MAGIC ,case when upper(top_top_incre) = 'TOP SALES' then 'TOP' else 'Incre' end as top_incre
# MAGIC from dwd.dwd_gnd_retail_national_corp_rank
# MAGIC
# MAGIC union all
# MAGIC select
# MAGIC case when upper(replace(rank1,' ','')) = 'TOTAL' then '99'
# MAGIC when rank1 is null then '98'
# MAGIC else replace(rank1,' ','') end as rank1
# MAGIC ,prod_desc as product_desc
# MAGIC ,`type` as product_type
# MAGIC ,cmps_desc as common_name
# MAGIC ,brand
# MAGIC ,corp_desc as corp
# MAGIC ,corporation
# MAGIC ,null as corp_type
# MAGIC ,case when length(sales_quarter) = 4 then concat('20',sales_quarter)
# MAGIC else sales_quarter
# MAGIC end as sales_quarter
# MAGIC ,null as ytd
# MAGIC ,sales_amount
# MAGIC ,null as rx_type
# MAGIC ,null as top1_brand_val
# MAGIC ,null as top1_brand_gr
# MAGIC ,null as top2_brand_val
# MAGIC ,null as top2_brand_gr
# MAGIC ,null as top1_incremental_brand
# MAGIC ,null as top1_incremental_gr
# MAGIC ,null as top2_incremental_brand
# MAGIC ,null as top2_incremental_gr
# MAGIC ,null as ytd_gr
# MAGIC ,null as ytd_ms
# MAGIC ,null as ytd_delta_ms
# MAGIC ,null as dtp_name
# MAGIC ,'全国-产品排名' file_name
# MAGIC ,case when ta = 'GI' then concat(ta,'-',rx_otc)
# MAGIC when ta = 'NIAD-excl. GLP1' then 'DM'
# MAGIC else ta end as source_name
# MAGIC ,null rc
# MAGIC ,rx_otc
# MAGIC ,case when upper(top_top_incre) = 'TOP SALES' then 'TOP' else 'Incre' end as top_incre
# MAGIC from dwd.dwd_gnd_retail_national_prd_rank
# MAGIC union all
# MAGIC select
# MAGIC case when upper(replace(rank1,' ','')) = 'TOTAL' then '99'
# MAGIC when rank1 is null then '98'
# MAGIC else replace(rank1,' ','') end as rank1
# MAGIC ,null as product_desc
# MAGIC ,null as product_type
# MAGIC ,null as common_name
# MAGIC ,null as brand
# MAGIC ,corp_desc as corp
# MAGIC ,corporation
# MAGIC ,`type` as corp_type
# MAGIC ,case when length(sales_quarter) = 4 then concat('20',sales_quarter)
# MAGIC else sales_quarter
# MAGIC end as sales_quarter
# MAGIC ,null as ytd
# MAGIC ,sales_amount
# MAGIC ,null as rx_type
# MAGIC ,null as top1_brand_val
# MAGIC ,null as top1_brand_gr
# MAGIC ,null as top2_brand_val
# MAGIC ,null as top2_brand_gr
# MAGIC ,null as top1_incremental_brand
# MAGIC ,null as top1_incremental_gr
# MAGIC ,null as top2_incremental_brand
# MAGIC ,null as top2_incremental_gr
# MAGIC ,null as ytd_gr
# MAGIC ,null as ytd_ms
# MAGIC ,null as ytd_delta_ms
# MAGIC ,null as dtp_name
# MAGIC ,'大区-集团排名' file_name
# MAGIC ,case when ta = 'GI' then concat(ta,'-',rx_otc)
# MAGIC when ta = 'NIAD-excl. GLP1' then 'DM'
# MAGIC else ta end as source_name
# MAGIC ,region rc
# MAGIC ,rx_otc
# MAGIC ,case when upper(top_top_incre) = 'TOP SALES' then 'TOP' else 'Incre' end as top_incre
# MAGIC from dwd.dwd_gnd_retail_region_corp_rank
# MAGIC )
# MAGIC
# MAGIC -- --计算Total值，按文件、季度、TA、rx_otc、top_incre进行合计
# MAGIC -- --因为数据中本身就包含了Total行，因此直接筛选了rank1 = '99'(即筛选Total行)进行合计
# MAGIC ,overview_total as (
# MAGIC select file_name,
# MAGIC source_name,
# MAGIC rx_otc,
# MAGIC top_incre,
# MAGIC sales_quarter,
# MAGIC rc,
# MAGIC sum(sales_amount) sales_amount_total
# MAGIC from overview
# MAGIC where rank1 = '99'
# MAGIC group by file_name,sales_quarter,rx_otc,top_incre,source_name,rc
# MAGIC )
# MAGIC
# MAGIC -- --取到文件中的最大时间，在获取ly值的时候，用于限定时间范围
# MAGIC ,overview_maxyq as (
# MAGIC select max(sales_quarter) max_sales_quarter
# MAGIC from overview
# MAGIC where sales_quarter is not null
# MAGIC )
# MAGIC
# MAGIC -- --在数据源的基础上，增加yyyymm字段、sales_amount_ly、sales_amount_total、sales_amount_ly_total字段
# MAGIC select
# MAGIC case when max(rank1) < 10000 then max(rank1) else cast(max(rank1) as int) - 10000 end as rank1
# MAGIC ,product_desc
# MAGIC ,product_type
# MAGIC ,common_name
# MAGIC ,brand
# MAGIC ,corp
# MAGIC ,corporation
# MAGIC ,corp_type
# MAGIC ,sales_quarter
# MAGIC ,case when right(sales_quarter,1) = '1' then concat(left(sales_quarter,4),'03')
# MAGIC when right(sales_quarter,1) = '2' then concat(left(sales_quarter,4),'06')
# MAGIC when right(sales_quarter,1) = '3' then concat(left(sales_quarter,4),'09')
# MAGIC when right(sales_quarter,1) = '4' then concat(left(sales_quarter,4),'12')
# MAGIC end as yyyymm
# MAGIC ,max(ytd) ytd
# MAGIC ,sum(sales_amount) as sales_amount
# MAGIC ,sum(sales_amount_ly) as sales_amount_ly
# MAGIC ,sum(sales_amount_total) as sales_amount_total
# MAGIC ,sum(sales_amount_ly_total) as sales_amount_ly_total
# MAGIC ,rx_type
# MAGIC ,max(top1_brand_val) top1_brand_val
# MAGIC ,max(top1_brand_gr) top1_brand_gr
# MAGIC ,max(top2_brand_val) top2_brand_val
# MAGIC ,max(top2_brand_gr) top2_brand_gr
# MAGIC ,max(top1_incremental_brand) top1_incremental_brand
# MAGIC ,max(top1_incremental_gr) top1_incremental_gr
# MAGIC ,max(top2_incremental_brand) top2_incremental_brand
# MAGIC ,max(top2_incremental_gr) top2_incremental_gr
# MAGIC ,max(ytd_gr) ytd_gr
# MAGIC ,max(ytd_ms) ytd_ms
# MAGIC ,max(ytd_delta_ms) ytd_delta_ms
# MAGIC ,dtp_name
# MAGIC ,file_name
# MAGIC ,source_name
# MAGIC ,rc
# MAGIC ,rx_otc
# MAGIC ,top_incre
# MAGIC ,from_utc_timestamp(current_timestamp(),'UTC+8') etl_insert_dt
# MAGIC ,from_utc_timestamp(current_timestamp(),'UTC+8') etl_update_dt
# MAGIC from
# MAGIC (
# MAGIC select
# MAGIC cast(rank1 as int) + 10000 rank1
# MAGIC ,product_desc
# MAGIC ,product_type
# MAGIC ,common_name
# MAGIC ,brand
# MAGIC ,corp
# MAGIC ,corporation
# MAGIC ,corp_type
# MAGIC ,t1.sales_quarter
# MAGIC ,ytd
# MAGIC ,sales_amount
# MAGIC ,0 as sales_amount_ly
# MAGIC ,t2.sales_amount_total
# MAGIC ,0 as sales_amount_ly_total
# MAGIC ,rx_type
# MAGIC ,top1_brand_val
# MAGIC ,top1_brand_gr
# MAGIC ,top2_brand_val
# MAGIC ,top2_brand_gr
# MAGIC ,top1_incremental_brand
# MAGIC ,top1_incremental_gr
# MAGIC ,top2_incremental_brand
# MAGIC ,top2_incremental_gr
# MAGIC ,ytd_gr
# MAGIC ,ytd_ms
# MAGIC ,ytd_delta_ms
# MAGIC ,dtp_name
# MAGIC ,t1.file_name
# MAGIC ,t1.source_name
# MAGIC ,t1.rc
# MAGIC ,t1.rx_otc
# MAGIC ,t1.top_incre
# MAGIC from overview t1
# MAGIC left join overview_total t2
# MAGIC on t1.file_name = t2.file_name
# MAGIC and nvl(t1.sales_quarter,'') = nvl(t2.sales_quarter,'')
# MAGIC and t1.rx_otc = t2.rx_otc
# MAGIC and t1.top_incre = t2.top_incre
# MAGIC and NVL(t1.source_name,'') = NVL(t2.source_name,'')
# MAGIC and NVL(t1.rc,'') = NVL(t2.rc,'')
# MAGIC
# MAGIC union all
# MAGIC
# MAGIC select
# MAGIC cast(rank1 as int) rank1
# MAGIC ,product_desc
# MAGIC ,product_type
# MAGIC ,common_name
# MAGIC ,brand
# MAGIC ,corp
# MAGIC ,corporation
# MAGIC ,corp_type
# MAGIC ,concat(cast(left(t1.sales_quarter,4) as int) + 1,right(t1.sales_quarter,2)) as sales_quarter
# MAGIC ,null as ytd
# MAGIC ,0 as sales_amount
# MAGIC ,sales_amount as sales_amount_ly
# MAGIC ,0 as sales_amount_total
# MAGIC ,t2.sales_amount_total as sales_amount_ly_total
# MAGIC ,rx_type
# MAGIC ,null as top1_brand_val
# MAGIC ,null as top1_brand_gr
# MAGIC ,null as top2_brand_val
# MAGIC ,null as top2_brand_gr
# MAGIC ,null as top1_incremental_brand
# MAGIC ,null as top1_incremental_gr
# MAGIC ,null as top2_incremental_brand
# MAGIC ,null as top2_incremental_gr
# MAGIC ,null as ytd_gr
# MAGIC ,null as ytd_ms
# MAGIC ,null as ytd_delta_ms
# MAGIC ,dtp_name
# MAGIC ,t1.file_name
# MAGIC ,t1.source_name
# MAGIC ,t1.rc
# MAGIC ,t1.rx_otc
# MAGIC ,t1.top_incre
# MAGIC from overview t1
# MAGIC left join overview_total t2
# MAGIC on t1.file_name = t2.file_name
# MAGIC and nvl(t1.sales_quarter,'') = nvl(t2.sales_quarter,'')
# MAGIC and t1.rx_otc = t2.rx_otc
# MAGIC and t1.top_incre = t2.top_incre
# MAGIC and NVL(t1.source_name,'') = NVL(t2.source_name,'')
# MAGIC and NVL(t1.rc,'') = NVL(t2.rc,'')
# MAGIC left join overview_maxyq t3
# MAGIC on 1=1
# MAGIC where concat(cast(left(t1.sales_quarter,4) as int) + 1,right(t1.sales_quarter,2)) <= t3.max_sales_quarter
# MAGIC ) t
# MAGIC group by
# MAGIC product_desc
# MAGIC ,product_type
# MAGIC ,common_name
# MAGIC ,brand
# MAGIC ,corp
# MAGIC ,corporation
# MAGIC ,corp_type
# MAGIC ,sales_quarter
# MAGIC ,rx_type
# MAGIC ,dtp_name
# MAGIC ,file_name
# MAGIC ,source_name
# MAGIC ,rc
# MAGIC ,rx_otc
# MAGIC ,top_incre
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-10: map to dm.dm_zk_retail_overview_data
# MAGIC -- 10.2 将free report overview 导出的合并文件写入dm
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC insert overwrite table dm.dm_zk_retail_overview_data(
# MAGIC rank1,
# MAGIC product_desc,
# MAGIC product_type,
# MAGIC common_name,
# MAGIC brand,
# MAGIC corp,
# MAGIC corporation,
# MAGIC corp_type,
# MAGIC sales_quarter,
# MAGIC yyyymm,
# MAGIC ytd,
# MAGIC sales_amount,
# MAGIC sales_amount_ly,
# MAGIC sales_amount_total,
# MAGIC sales_amount_ly_total,
# MAGIC rx_type,
# MAGIC top1_brand_val,
# MAGIC top1_brand_gr,
# MAGIC top2_brand_val,
# MAGIC top2_brand_gr,
# MAGIC top1_incremental_brand,
# MAGIC top1_incremental_gr,
# MAGIC top2_incremental_brand,
# MAGIC top2_incremental_gr,
# MAGIC ytd_gr,
# MAGIC ytd_ms,
# MAGIC ytd_delta_ms,
# MAGIC dtp_name,
# MAGIC file_name,
# MAGIC source_name,
# MAGIC rc,
# MAGIC rx_otc,
# MAGIC top_incre,
# MAGIC etl_insert_dt,
# MAGIC etl_update_dt
# MAGIC )
# MAGIC select
# MAGIC trim(rank1) rank1
# MAGIC ,product_desc
# MAGIC ,product_type
# MAGIC ,common_name
# MAGIC ,brand
# MAGIC ,nvl(corp,'' ) corp
# MAGIC ,corporation
# MAGIC ,corp_type
# MAGIC ,sales_quarter
# MAGIC ,yyyymm
# MAGIC ,case when ytd is null then 'N' ELSE 'Y' END ytd
# MAGIC ,sales_amount
# MAGIC ,sales_amount_ly
# MAGIC ,sales_amount_total
# MAGIC ,sales_amount_ly_total
# MAGIC ,rx_type
# MAGIC ,top1_brand_val
# MAGIC ,cast(top1_brand_gr as double) top1_brand_gr
# MAGIC ,top2_brand_val
# MAGIC ,cast(top2_brand_gr as double) top2_brand_gr
# MAGIC ,top1_incremental_brand
# MAGIC ,cast(top1_incremental_gr as double) top1_incremental_gr
# MAGIC ,top2_incremental_brand
# MAGIC ,cast(top2_incremental_gr as double) top2_incremental_gr
# MAGIC ,cast(ytd_gr as double ) ytd_gr
# MAGIC ,cast(ytd_ms as double) ytd_ms
# MAGIC ,cast(ytd_delta_ms as double) ytd_delta_ms
# MAGIC ,dtp_name
# MAGIC ,file_name
# MAGIC ,trim(source_name) source_name
# MAGIC ,rc
# MAGIC ,rx_otc
# MAGIC ,top_incre
# MAGIC ,from_utc_timestamp(current_timestamp(),'UTC+8') etl_insert_dt
# MAGIC ,from_utc_timestamp(current_timestamp(),'UTC+8') etl_update_dt
# MAGIC from dws.dws_zk_retail_overview_data
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-11: map to dm.dm_zk_retail_product_info
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-11: map to dm.dm_zk_retail_product_info
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC ------------------------------retail/EC原始维度，从b2c配置表直取-----------------------------------------
# MAGIC insert overwrite table dws.dws_zk_retail_product_info
# MAGIC select distinct
# MAGIC product_id
# MAGIC -------------------------------------------------------------
# MAGIC -- format iqvia_pack_code
# MAGIC ,case when length(trim(iqvia_pack_code)) < 12 and trim(iqvia_pack_code) REGEXP '^[0-9]' then right(concat('000000000000',trim(iqvia_pack_code)),12) else trim(iqvia_pack_code) end as iqvia_pack_code
# MAGIC -------------------------------------------------------------
# MAGIC ,prescription_nature
# MAGIC ,medicine_type
# MAGIC ,zk_medicine_tier1
# MAGIC ,zk_medicine_tier2
# MAGIC ,zk_medicine_tier3
# MAGIC ,zk_medicine_tier4
# MAGIC ,common_name
# MAGIC ,dosage_form
# MAGIC ,user_type
# MAGIC ,category_name
# MAGIC ,product_name
# MAGIC ,brand_name
# MAGIC ,zk_manu_des
# MAGIC ,zk_corp_des
# MAGIC ,zk_pack_des
# MAGIC ,from_utc_timestamp(current_timestamp(),'UTC+8') etl_insert_dt
# MAGIC ,from_utc_timestamp(current_timestamp(),'UTC+8') etl_update_dt
# MAGIC from dwd.dwd_gnd_ext_retail_pack_property;
# MAGIC ------------------------------retail/EC原始维度，从b2c直取 写入dm-----------------------------------------
# MAGIC insert overwrite table dm.dm_zk_retail_product_info
# MAGIC select distinct
# MAGIC product_id
# MAGIC ,iqvia_pack_code
# MAGIC ,prescription_nature
# MAGIC ,medicine_type
# MAGIC ,zk_medicine_tier1
# MAGIC ,zk_medicine_tier2
# MAGIC ,zk_medicine_tier3
# MAGIC ,zk_medicine_tier4
# MAGIC ,common_name
# MAGIC ,dosage_form
# MAGIC ,user_type
# MAGIC ,category_name
# MAGIC ,product_name
# MAGIC ,brand_name
# MAGIC ,zk_manu_des
# MAGIC ,zk_corp_des
# MAGIC ,zk_pack_des
# MAGIC ,from_utc_timestamp(current_timestamp(),'UTC+8') etl_insert_dt
# MAGIC ,from_utc_timestamp(current_timestamp(),'UTC+8') etl_update_dt
# MAGIC from dws.dws_zk_retail_product_info;
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-12: map to dm.dm_zk_retail_brand
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-12: map to dm.dm_zk_retail_brand
# MAGIC -- retail 自有部分需要导出数据，从品牌数据报表直取,无特殊加工
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC ------------------------------retail 自有部分需要导出数据，从品牌数据报表直取,无特殊加工-----------------------------------------
# MAGIC insert overwrite table dws.dws_zk_retail_brand
# MAGIC with max_yq as (
# MAGIC select TA,market,rc_name_en,province_city,ranked_by,max(yq) as yq
# MAGIC from dwd.dwd_inc_gnd_ext_retail_nataional_brand_union_all
# MAGIC group by TA,market,rc_name_en,province_city,ranked_by
# MAGIC )
# MAGIC
# MAGIC select distinct
# MAGIC t1.TA,
# MAGIC t1.market,
# MAGIC t1.rc_name_en,
# MAGIC t1.province_city
# MAGIC ,key_brand_ytd
# MAGIC ,key_brand_rank_ytd
# MAGIC ,top_brand_ytd
# MAGIC ,top_brand_ms_ytd
# MAGIC ,top_brand_inc_ms_ytd
# MAGIC ,top_brand_gr_ytd
# MAGIC ,key_brand_qtd
# MAGIC ,key_brand_rank_qtd
# MAGIC ,top_brand_qtd
# MAGIC ,top_brand_ms_qtd
# MAGIC ,top_brand_inc_ms_qtd
# MAGIC ,top_brand_gr_qtd
# MAGIC ,t1.ranked_by
# MAGIC ,from_utc_timestamp(current_timestamp(),'UTC+8') etl_insert_dt
# MAGIC ,from_utc_timestamp(current_timestamp(),'UTC+8') etl_update_dt
# MAGIC from dwd.dwd_inc_gnd_ext_retail_nataional_brand_union_all t1
# MAGIC inner join max_yq t2
# MAGIC on t1.TA = t2.TA
# MAGIC and t1.market = t2.market
# MAGIC and t1.rc_name_en = t2.rc_name_en
# MAGIC and t1.province_city = t2.province_city
# MAGIC and t1.yq = t2.yq
# MAGIC and t1.ranked_by = t2.ranked_by
# MAGIC ;
# MAGIC
# MAGIC -- dm_zk_retail_brand
# MAGIC
# MAGIC ------------------------------retail 自有部分需要导出数据，写入dm-----------------------------------------
# MAGIC insert overwrite table dm.dm_zk_retail_brand
# MAGIC select distinct
# MAGIC TA,market,rc_name_en,province_city
# MAGIC ,key_brand_ytd
# MAGIC ,cast(key_brand_rank_ytd as double) key_brand_rank_ytd
# MAGIC ,top_brand_ytd
# MAGIC ,cast(top_brand_ms_ytd as double) top_brand_ms_ytd
# MAGIC ,cast(top_brand_inc_ms_ytd as double) top_brand_inc_ms_ytd
# MAGIC ,cast(top_brand_gr_ytd as double) top_brand_gr_ytd
# MAGIC ,key_brand_qtd
# MAGIC ,cast(key_brand_rank_qtd as double ) key_brand_rank_qtd
# MAGIC ,top_brand_qtd
# MAGIC ,cast(top_brand_ms_qtd as double ) top_brand_ms_qtd
# MAGIC ,cast(top_brand_inc_ms_qtd as double ) top_brand_inc_ms_qtd
# MAGIC ,cast(top_brand_gr_qtd as double ) top_brand_gr_qtd
# MAGIC ,ranked_by
# MAGIC from dws.dws_zk_retail_brand;
# MAGIC
# MAGIC
# MAGIC
# MAGIC
# MAGIC
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-13: map to dm.dm_zk_retail_export
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-13: map to dm.dm_zk_retail_export
# MAGIC
# MAGIC --修改时间：20240821
# MAGIC --修改人：FanXujia
# MAGIC --修改内容：
# MAGIC --导出模板整合，由原来的十几个文件改为3个文件
# MAGIC --《全国-集团排名》：dwd.dwd_gnd_retail_national_corp_rank
# MAGIC --《全国-产品排名》：dwd.dwd_gnd_retail_national_prd_rank
# MAGIC --《大区-集团排名》：dwd.dwd_gnd_retail_region_corp_rank
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC insert overwrite table dws.dws_zk_retail_export
# MAGIC (
# MAGIC ta,
# MAGIC rx_otc,
# MAGIC top_top_incre,
# MAGIC rank1,
# MAGIC corp_desc,
# MAGIC corporation,
# MAGIC type,
# MAGIC prod_desc,
# MAGIC cmps_desc,
# MAGIC brand,
# MAGIC region,
# MAGIC sales_quarter,
# MAGIC sales_amount,
# MAGIC file_name,
# MAGIC etl_insert_dt,
# MAGIC etl_update_dt
# MAGIC )
# MAGIC --《全国-集团排名》
# MAGIC select distinct
# MAGIC ta,
# MAGIC rx_otc,
# MAGIC top_top_incre,
# MAGIC case when upper(trim(rank1)) = 'TOTAL' then '99'
# MAGIC when rank1 is null then '98'
# MAGIC else trim(rank1) end as rank1,
# MAGIC corp_desc,
# MAGIC corporation,
# MAGIC type,
# MAGIC null as prod_desc,
# MAGIC null as cmps_desc,
# MAGIC null as brand,
# MAGIC null as region,
# MAGIC sales_quarter,
# MAGIC sales_amount,
# MAGIC '全国-集团排名' as file_name,
# MAGIC from_utc_timestamp(current_timestamp(),'UTC+8') as etl_insert_dt,
# MAGIC from_utc_timestamp(current_timestamp(),'UTC+8') as etl_update_dt
# MAGIC from dwd.dwd_gnd_retail_national_corp_rank
# MAGIC
# MAGIC union all
# MAGIC
# MAGIC --《全国-产品排名》
# MAGIC select distinct
# MAGIC ta,
# MAGIC rx_otc,
# MAGIC top_top_incre,
# MAGIC case when upper(trim(rank1)) = 'TOTAL' then '99'
# MAGIC when rank1 is null then '98'
# MAGIC else trim(rank1) end as rank1,
# MAGIC corp_desc,
# MAGIC corporation,
# MAGIC type,
# MAGIC prod_desc,
# MAGIC cmps_desc,
# MAGIC brand,
# MAGIC null as region,
# MAGIC sales_quarter,
# MAGIC sales_amount,
# MAGIC '全国-产品排名' as file_name,
# MAGIC from_utc_timestamp(current_timestamp(),'UTC+8') as etl_insert_dt,
# MAGIC from_utc_timestamp(current_timestamp(),'UTC+8') as etl_update_dt
# MAGIC from dwd.dwd_gnd_retail_national_prd_rank
# MAGIC
# MAGIC union all
# MAGIC
# MAGIC --《大区-集团排名》
# MAGIC select distinct
# MAGIC ta,
# MAGIC rx_otc,
# MAGIC top_top_incre,
# MAGIC case when upper(trim(rank1)) = 'TOTAL' then '99'
# MAGIC when rank1 is null then '98'
# MAGIC else trim(rank1) end as rank1,
# MAGIC corp_desc,
# MAGIC corporation,
# MAGIC type,
# MAGIC null as prod_desc,
# MAGIC null as cmps_desc,
# MAGIC null as brand,
# MAGIC region,
# MAGIC sales_quarter,
# MAGIC sales_amount,
# MAGIC '大区-集团排名' as file_name,
# MAGIC from_utc_timestamp(current_timestamp(),'UTC+8') as etl_insert_dt,
# MAGIC from_utc_timestamp(current_timestamp(),'UTC+8') as etl_update_dt
# MAGIC from dwd.dwd_gnd_retail_region_corp_rank;
# MAGIC
# MAGIC --写入dm表
# MAGIC insert overwrite table dm.dm_zk_retail_export
# MAGIC (
# MAGIC ta,
# MAGIC rx_otc,
# MAGIC top_top_incre,
# MAGIC rank1,
# MAGIC corp_desc,
# MAGIC corporation,
# MAGIC type,
# MAGIC prod_desc,
# MAGIC cmps_desc,
# MAGIC brand,
# MAGIC region,
# MAGIC sales_quarter,
# MAGIC sales_amount,
# MAGIC file_name,
# MAGIC etl_insert_dt,
# MAGIC etl_update_dt
# MAGIC )
# MAGIC select
# MAGIC ta,
# MAGIC rx_otc,
# MAGIC top_top_incre,
# MAGIC rank1,
# MAGIC corp_desc,
# MAGIC corporation,
# MAGIC type,
# MAGIC prod_desc,
# MAGIC cmps_desc,
# MAGIC brand,
# MAGIC region,
# MAGIC sales_quarter,
# MAGIC sales_amount,
# MAGIC file_name,
# MAGIC etl_insert_dt,
# MAGIC etl_update_dt
# MAGIC from dws.dws_zk_retail_export;
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-14: calculate niad date
# COMMAND ----------
# MAGIC %sql
# MAGIC CREATE OR REPLACE TABLE tmp.tmp_dm_td_ext_retail_niad_month as
# MAGIC WITH ALL_RETIAL_DATA (
# MAGIC select
# MAGIC max(
# MAGIC nvl(
# MAGIC month,
# MAGIC CONCAT(
# MAGIC SUBSTRING(quarter, 1, 4), -- 提取年份（前4位）
# MAGIC CASE
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '1' THEN '03' -- Q1 → 03月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '2' THEN '06' -- Q2 → 06月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '3' THEN '09' -- Q3 → 09月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '4' THEN '12' -- Q4 → 12月
# MAGIC END
# MAGIC )
# MAGIC )
# MAGIC ) as month,
# MAGIC 'NIAD' as datatype
# MAGIC from
# MAGIC dwd.dwd_gnd_ext_retail_nataional_niad
# MAGIC union all
# MAGIC select
# MAGIC max(
# MAGIC nvl(
# MAGIC month,
# MAGIC CONCAT(
# MAGIC SUBSTRING(quarter, 1, 4), -- 提取年份（前4位）
# MAGIC CASE
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '1' THEN '03' -- Q1 → 03月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '2' THEN '06' -- Q2 → 06月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '3' THEN '09' -- Q3 → 09月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '4' THEN '12' -- Q4 → 12月
# MAGIC END
# MAGIC )
# MAGIC )
# MAGIC ) as month,
# MAGIC 'NOTNIAD' as datatype
# MAGIC from
# MAGIC dwd.dwd_gnd_ext_retail_statin_xzk
# MAGIC union all
# MAGIC select
# MAGIC max(
# MAGIC nvl(
# MAGIC month,
# MAGIC CONCAT(
# MAGIC SUBSTRING(quarter, 1, 4), -- 提取年份（前4位）
# MAGIC CASE
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '1' THEN '03' -- Q1 → 03月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '2' THEN '06' -- Q2 → 06月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '3' THEN '09' -- Q3 → 09月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '4' THEN '12' -- Q4 → 12月
# MAGIC END
# MAGIC )
# MAGIC )
# MAGIC ) as month,
# MAGIC 'NOTNIAD' as datatype
# MAGIC from
# MAGIC dwd.dwd_gnd_ext_retail_nataional_oap
# MAGIC union all
# MAGIC select
# MAGIC max(
# MAGIC nvl(
# MAGIC month,
# MAGIC CONCAT(
# MAGIC SUBSTRING(quarter, 1, 4), -- 提取年份（前4位）
# MAGIC CASE
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '1' THEN '03' -- Q1 → 03月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '2' THEN '06' -- Q2 → 06月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '3' THEN '09' -- Q3 → 09月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '4' THEN '12' -- Q4 → 12月
# MAGIC END
# MAGIC )
# MAGIC )
# MAGIC ) as month,
# MAGIC 'NOTNIAD' as datatype
# MAGIC from
# MAGIC dwd.dwd_gnd_ext_retail_anti_asthma_copd
# MAGIC union all
# MAGIC select
# MAGIC max(
# MAGIC nvl(
# MAGIC month,
# MAGIC CONCAT(
# MAGIC SUBSTRING(quarter, 1, 4), -- 提取年份（前4位）
# MAGIC CASE
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '1' THEN '03' -- Q1 → 03月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '2' THEN '06' -- Q2 → 06月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '3' THEN '09' -- Q3 → 09月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '4' THEN '12' -- Q4 → 12月
# MAGIC END
# MAGIC )
# MAGIC )
# MAGIC ) as month,
# MAGIC 'NOTNIAD' as datatype
# MAGIC from
# MAGIC dwd.dwd_gnd_ext_retail_aagsa_ppi_oral
# MAGIC union all
# MAGIC select
# MAGIC max(
# MAGIC nvl(
# MAGIC month,
# MAGIC CONCAT(
# MAGIC SUBSTRING(quarter, 1, 4), -- 提取年份（前4位）
# MAGIC CASE
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '1' THEN '03' -- Q1 → 03月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '2' THEN '06' -- Q2 → 06月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '3' THEN '09' -- Q3 → 09月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '4' THEN '12' -- Q4 → 12月
# MAGIC END
# MAGIC )
# MAGIC )
# MAGIC ) as month,
# MAGIC 'NOTNIAD' as datatype
# MAGIC from
# MAGIC dwd.dwd_gnd_ext_retail_atomizer
# MAGIC union all
# MAGIC select
# MAGIC max(
# MAGIC nvl(
# MAGIC month,
# MAGIC CONCAT(
# MAGIC SUBSTRING(quarter, 1, 4), -- 提取年份（前4位）
# MAGIC CASE
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '1' THEN '03' -- Q1 → 03月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '2' THEN '06' -- Q2 → 06月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '3' THEN '09' -- Q3 → 09月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '4' THEN '12' -- Q4 → 12月
# MAGIC END
# MAGIC )
# MAGIC )
# MAGIC ) as month,
# MAGIC 'NOTNIAD' as datatype
# MAGIC from
# MAGIC dwd.dwd_gnd_ext_retail_nataional_rd
# MAGIC union all
# MAGIC select
# MAGIC max(
# MAGIC nvl(
# MAGIC month,
# MAGIC CONCAT(
# MAGIC SUBSTRING(quarter, 1, 4), -- 提取年份（前4位）
# MAGIC CASE
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '1' THEN '03' -- Q1 → 03月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '2' THEN '06' -- Q2 → 06月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '3' THEN '09' -- Q3 → 09月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '4' THEN '12' -- Q4 → 12月
# MAGIC END
# MAGIC )
# MAGIC )
# MAGIC ) as month,
# MAGIC 'NOTNIAD' as datatype
# MAGIC from
# MAGIC dwd.dwd_gnd_ext_retail_metoprolol_tartrat
# MAGIC union all
# MAGIC select
# MAGIC max(
# MAGIC nvl(
# MAGIC month,
# MAGIC CONCAT(
# MAGIC SUBSTRING(quarter, 1, 4), -- 提取年份（前4位）
# MAGIC CASE
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '1' THEN '03' -- Q1 → 03月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '2' THEN '06' -- Q2 → 06月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '3' THEN '09' -- Q3 → 09月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '4' THEN '12' -- Q4 → 12月
# MAGIC END
# MAGIC )
# MAGIC )
# MAGIC ) as month,
# MAGIC 'NOTNIAD' as datatype
# MAGIC from
# MAGIC dwd.dwd_gnd_ext_retail_htn
# MAGIC )
# MAGIC SELECT
# MAGIC MAX(t1.month) as NIAD_MONTH,
# MAGIC max(t2.month) OTHERS_MONTH
# MAGIC FROM
# MAGIC all_retial_data t1
# MAGIC LEFT JOIN (
# MAGIC SELECT
# MAGIC MAX(month) as month
# MAGIC FROM
# MAGIC all_retial_data
# MAGIC WHERE
# MAGIC datatype = 'NOTNIAD'
# MAGIC ) t2
# MAGIC WHERE
# MAGIC t1.datatype = 'NIAD'
# COMMAND ----------
############################################################END################################################################
Reference in New Issue View Git Blame Copy Permalink