chenwu/MarketAnalysis-ETL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new file: 01 dm_tf_external_sales.sql

Browse Source 
new file:   02 dm_td_external_keycompatitor.sql
	new file:   03 dm_td_external_brand_market.sql
	new file:   04 dm_td_external_calendar.sql
	new file:   05 dm_td_external_exchangerate.sql
	new file:   07 dm_td_external_packinfo.sql
	new file:   08 dm_td_external_corp.sql
	new file:   09 dm_td_external_geo_type.sql
	new file:   11 DM_TD_EXTERNAL_MARKET_NEW.sql
	new file:   12 dm_td_external_org.sql
	new file:   13 external auth.sql
	new file:   14 dm_tf_external_retail_special_bkp.sql
	new file:   AIA/01 dm_aia_pack_property.sql
	new file:   AIA/02 dm_ext_aia_sales.sql
	new file:   AIA/02 dm_ext_aia_sales_bakup_20230327.sql
	new file:   AIA/03 dm_aia_flag.sql
	new file:   AIA/04 dm_aia_provided_flag.sql
	new file:   AIA/06 DM_TD_EXT_AIA_PACK_PROPERTY.sql
	new file:   AIA/07 DM_TD_EXT_AIA_PACK2MARKET.sql
	new file:   AIA/08 DM_TD_EXT_AIA_MARKET_RATIO.sql
	new file:   AIA/09 DM_TD_EXT_AIA_MARKET_BRAND_RATIO.sql
	new file:   AIA/10 DM_TD_EXT_AIA_MARKET_PACK_MAPPING.sql
	new file:   AIA/11 DM_TD_EXT_AIA_TARGET_INST.sql
	new file:   AIA/11 dm_aia_targethp_flag.sql
	new file:   "AIA/z_03 dm_aia_flag_\345\244\207\344\273\275\347\224\250.sql"
	new file:   CHC/01 dm_chc_pack_property.sql
	new file:   CHC/02 DM_TF_EXT_CHC_SALES.sql
	new file:   CHC/03 DM_TD_EXT_CHC_PACK_PROPERTY.sql
	new file:   CHC/04 DM_TD_EXT_CHC_PACK2MARKET.sql
	new file:   CHC/05 DM_TD_EXT_CHC_MARKET_RATIO.sql
	new file:   CHC/06 DM_TD_EXT_CHC_MARKET_BRAND_RATIO.sql
	new file:   CHC/07 DM_TD_EXT_CHC_MARKET_PACK_MAPPING.sql
	new file:   CHPA/01 dwd_ims_atc_hierarchy.sql
	new file:   CHPA/01 dwd_ims_nfc_hierarchy.sql
	new file:   CHPA/01 dwd_ims_td_manufacturer_corp.sql
	new file:   CHPA/01 dwd_ims_td_pack_property.sql
	new file:   CHPA/01 dwd_update.sql
	new file:   CHPA/01_FB_BLOB_TO_DWD.sql
	new file:   CHPA/02 DWS_IMS_TD_GEO.sql
	new file:   CHPA/02 dws_ims_td_atc_cn.sql
	new file:   CHPA/02 dws_ims_td_corp_cn.sql
	new file:   CHPA/02 dws_ims_td_date.sql
	new file:   CHPA/02 dws_ims_td_manu_cn.sql
	new file:   CHPA/02 dws_ims_td_market.sql
	new file:   CHPA/02 dws_ims_td_market_ta.sql
	new file:   CHPA/02 dws_ims_td_nfc_cn.sql
	new file:   CHPA/02 dws_ims_td_prod_cn.sql
	new file:   CHPA/02 tmp_ims_td_prod_tmp.sql
	new file:   CHPA/02 tmp_ims_tf_fact_sales.sql
	new file:   CHPA/03 DM_TD_EXT_CHPA_MARKET_BRAND_RATIO.sql
	new file:   CHPA/03 DM_TD_EXT_CHPA_MARKET_PACK_MAPPING.sql
	new file:   CHPA/03 DM_TD_EXT_CHPA_MARKET_RATIO.sql
	new file:   CHPA/03 DM_TD_EXT_CHPA_PACK2MARKET.sql
	new file:   CHPA/03 DM_TD_EXT_CHPA_PACK_PROPERTY.sql
	new file:   CHPA/03 DM_TF_EXT_CHPA_SALES.sql
	new file:   CHPA/03 dm_ims_td_calendar.sql
	new file:   CHPA/03 dm_ims_td_geo.sql
	new file:   CHPA/03 dm_ims_td_market_property.sql
	new file:   CHPA/03 dm_ims_td_org.sql
	new file:   CHPA/03 dm_ims_td_org_hvh.sql
	new file:   CHPA/03 dm_ims_td_pack_property.sql
	new file:   CHPA/03 dm_ims_tf_sales.sql
	new file:   CHPA/03 dm_td_chpa_market_definition.sql
	new file:   CHPA/03 dm_td_ims_city_mapping.sql
	new file:   EC/03 ec_load_data.sql
	new file:   EC/04 DM_TD_EXT_EC_PACK_PROPERTY.sql
	new file:   EC/05 DM_TF_EXT_EC_SALES.sql
	new file:   EC/06 DM_TD_EXT_EC_PACK2MARKET.sql
	new file:   EC/07 DM_TD_EXT_EC_MARKET_RATIO.sql
	new file:   EC/08 DM_TD_EXT_EC_MARKET_BRAND_RATIO.sql
	new file:   EC/09 DM_TD_EXT_EC_MARKET_PACK_MAPPING.sql
	new file:   EC/1 (ec)blob_to_dwd.sql
	new file:   EC/2 dwd_inc_gnd_ext_ec_nationnal_pack_union_all.py
	new file:   Merged_Data/Merged_Data_Config_table_bkp.sql
	new file:   Merged_Data/Merged_Data_Config_table_bymonth.sql
	new file:   Merged_Data/dm_tf_exteranl_sales_merged_data_dtp_others_bkp.sql
	new file:   Merged_Data/dm_tf_exteranl_sales_merged_data_dtp_others_bymonth_bkp.sql
	new file:   ORG/DM_TD_EXT_AIA_ORG.sql
	new file:   ORG/DM_TD_EXT_CHC_ORG.sql
	new file:   ORG/DM_TD_EXT_CHPA_ORG.sql
	new file:   ORG/DM_TD_EXT_COUNTY_ORG.sql
	new file:   ORG/DM_TD_EXT_EC_ORG.sql
	new file:   ORG/DM_TD_EXT_RETAIL_ORG.sql
	new file:   ORG/DM_TD_EXT_THC_ORG.sql
	new file:   ORG/DM_TD_EXT_XIEHE_ORG.sql
	new file:   OTHERS/01 dm_td_report_url.sql
	new file:   OTHERS/02 dws_ext_email_warning.sql
	new file:   OTHERS/external_triggered_email.py
	new file:   Retail/01 load_tmp_data.py
	new file:   Retail/02 split_brand_data.py
	new file:   Retail/03 split_pack_data.py
	new file:   Retail/04 map_to_dws_table.py
	new file:   Retail/05 load_dtp_temp_data.py
	new file:   Retail/06 split_dtp_brand_data.py
	new file:   Retail/07 split_dtp_pack_data.py
	new file:   Retail/08 map_to_dtp_dws_table.py
	new file:   Retail/09 dwd_inc_gnd_ext_retail_nataional.py
	new file:   Retail/10 map_to_retail_dm_table.py
	new file:   Retail/11 map_to_overview_dm_table.py
	new file:   Retail/12 dws_tf_external_retail_dtp_special.sql
	new file:   Retail/13 DM_TF_EXT_RETAIL_SALES.sql
	new file:   Retail/14 DM_TF_EXT_RETAIL_DTP_SALES.sql
	new file:   Retail/15 DM_TD_EXT_RETAIL_PACK_PROPERTY.sql
	new file:   Retail/16 DM_TD_EXT_RETAIL_DTP_PACK_PROPERTY.sql
	new file:   Retail/17 DM_TD_EXT_DTP_PACK2MARKET.sql
	new file:   Retail/17 DM_TD_EXT_RETAIL_PACK2MARKET.sql
	new file:   Retail/18 DM_TD_EXT_DTP_MARKET_RATIO.sql
	new file:   Retail/18 DM_TD_EXT_RETAIL_MARKET_RATIO.sql
	new file:   Retail/19 DM_TD_EXT_DTP_MARKET_BRAND_RATIO.sql
	new file:   Retail/19 DM_TD_EXT_RETAIL_MARKET_BRAND_RATIO.sql
	new file:   Retail/20 DM_TD_EXT_DTP_MARKET_PACK_MAPPING.sql
	new file:   Retail/20 DM_TD_EXT_RETAIL_MARKET_PACK_MAPPING.sql
	new file:   "Retail/z1 dwd_inc_gnd_ext_retail_nataional_\344\275\234\345\272\237.py"
	new file:   "Retail/z2 retail_load_data_\344\275\234\345\272\237.sql"
	new file:   "Retail/z3 retail_overview_data_\344\275\234\345\272\237.sql"
	new file:   THC/01 dm_thc_pack_property.sql
	new file:   THC/02 dm_ext_thc_sales.sql
	new file:   THC/02 dm_ext_thc_sales_bakup_20260327.sql
	new file:   THC/03 DM_TF_EXT_THC_MARKET_SALES_CHT.sql
	new file:   THC/04 dm_tf_external_sales_thc.sql
	new file:   THC/05 DM_TD_EXT_THC_PACK_PROPERTY.sql
	new file:   THC/06 DM_TD_EXT_THC_PACK2MARKET.sql
	new file:   THC/07 DM_TD_EXT_THC_MARKET_RATIO.sql
	new file:   THC/08 DM_TD_EXT_THC_MARKET_BRAND_RATIO.sql
	new file:   THC/09 DM_TD_EXT_THC_MARKET_PACK_MAPPING.sql
	new file:   UNIONALL/DM_TD_EXT_UNIONALL_MARKET_PACK_MAPPING.sql
	new file:   UNIONALL/DM_TD_EXT_UNIONALL_PACKINFO.sql
	new file:   UNIONALL/DM_TF_EXT_UNIONALL_MARKET_SALES.sql
	new file:   UNIONALL/DM_TF_EXT_UNIONALL_MARKET_SALES_v1.0.sql
	new file:   UNIONALL/DM_TF_EXT_UNIONALL_SALES.sql
	new file:   UNIONALL/DM_TF_EXT_UNIONALL_SALES_MAPPING.sql
	new file:   UNIONALL/DM_TF_EXT_UNIONALL_SALES_MAPPING_NIAD.sql
	new file:   XIEHE/01 xiehe_blob_to_dwd.py
	new file:   XIEHE/02 dm_xiehe_pack_property.sql
	new file:   XIEHE/03 dm_ext_xiehe_geo.sql
	new file:   XIEHE/04 dm_ext_xiehe_sales.sql
	new file:   XIEHE/05 dm_td_xiehe_core_dept.sql
	new file:   XIEHE/06 DM_TF_EXT_XIEHE_SALES.sql
	new file:   XIEHE/07 DM_TD_EXT_XIEHE_PACK_PROPERTY.sql
	new file:   XIEHE/08 DM_TD_EXT_XIEHE_PACK2MARKET.sql
	new file:   XIEHE/09 DM_TD_EXT_XIEHE_MARKET_RATIO.sql
	new file:   XIEHE/10 DM_TD_EXT_XIEHE_MARKET_BRAND_RATIO.sql
	new file:   XIEHE/11 DM_TD_EXT_XIEHE_MARKET_PACK_MAPPING.sql
	new file:   XIEHE/bkp_01 xiehe_blob2dwd.py
	new file:   XIEHE/bkp_02 dm_ext_xiehe_sales.sql
	new file:   XIEHE/bkp_03 dm_ext_xiehe_pack_property.sql
	new file:   county/01 tmp_ims_county_fact_sales_sum.sql
	new file:   county/02 tmp_imscounty_Result.sql
	new file:   county/03 dm_ims_td_county_geo.sql
	new file:   county/04 dws_ext_county_tf_sales.sql
	new file:   county/05 dm_ext_county_td_pack_property.sql
	new file:   county/06 dm_td_county_pack_region.sql
	new file:   county/07 dm_ext_county_tf_sales_region.sql
	new file:   county/08 DM_TD_EXT_COUNTY_PACK_PROPERTY.sql
	new file:   county/09 DM_TF_EXT_COUNTY_SALES.sql
	new file:   county/10 DM_TD_EXT_COUNTY_PACK2MARKET.sql
	new file:   county/11 DM_TD_EXT_COUNTY_MARKET_RATIO.sql
	new file:   county/12 DM_TD_EXT_COUNTY_MARKET_BRAND_RATIO.sql
	new file:   county/13 DM_TD_EXT_COUNTY_MARKET_PACK_MAPPING.sql
	new file:   for_AIA_Dashboard/01 dm_td_aia_inst_mkt.sql
	new file:   for_AIA_Dashboard/02 dm_td_aia_auth_sales.sql
	new file:   for_AIA_Dashboard/03 dm_td_aia_original_col.sql
	new file:   for_AIA_Dashboard/04 dm_td_aia_nosales_inst.sql
	new file:   for_AIA_Dashboard/05 dm_td_aia_is_eagle_flag.sql
	new file:   for_AIA_Dashboard/06 dm_td_aia_rank.sql
	new file:   for_AIA_Dashboard/07 dm_ext_aia_data_remove_flag.sql
	new file:   for_AIA_Dashboard/07 dm_td_aia_remove_special_ins_bkp.py
	new file:   for_AIA_Dashboard/08 dm_ext_aia_data_quality_flag.sql
	new file:   z 01 dm_tf_external_sales.sql
	new file:   "z 10 dm_td_external_market_pack_mapping_\344\275\234\345\272\237.sql"
	new file:   "z 11 dm_td_external_market_\344\275\234\345\272\237.sql"
	new file:   "\344\270\212\347\272\277\350\204\232\346\234\254.sql"
	new file:   "\346\225\260\346\215\256\351\252\214\350\257\201.sql"
This commit is contained in:
chenwu
2026-04-27 15:48:38 +08:00
parent d2485035c0
commit 832c7bcd69
173 changed files with 62579 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
01 dm_tf_external_sales.sql Normal file
View File

@@ -0,0 +1,64 @@
-- Databricks notebook source
insert overwrite table dm.dm_tf_external_sales
(YYYYMM,
PACK_COD,
CORP_COD,
AUDIT_COD,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
DATA_SOURCE,
inst_code,
cmps_flag,
dept_name,
prescription,
prescription_ly,
new_code,
area,
h_level,
reimburse,
reimburse_type,
prescription_source
)
select
YYYYMM,
PACK_CODE AS PACK_COD,
CORP_CODE AS CORP_COD,
AUDIT_CODE AS AUDIT_COD,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
DATA_SOURCE,
inst_code,
cmps_flag,
dept_name,
prescription,
prescription_ly,
new_code,
area,
h_level,
reimburse,
reimburse_type,
prescription_source
from
dm.DM_TF_EXT_UNIONALL_SALES

42
02 dm_td_external_keycompatitor.sql Normal file
View File

@@ -0,0 +1,42 @@
-- Databricks notebook source
insert overwrite dm.dm_td_external_keycompatitor (
RNK,
BRAND_NAME,
BRAND_CODE,
IMSBRAND,
CompatitorCN,
CLASS,
CompatitorEN,
CORP,
Market,
VBP,
ValueType,
MARKETDESC,
Strategicbrand,
TA,
DEFAULTFLAG,
CompatitorMarket,
CompatitorType,
DATA_SOURCE
)
select
RNK,
BRAND_NAME,
BRAND_CODE,
IMSBRAND,
competitorcn,
CLASS,
competitoren,
CORP,
Market,
VBP,
ValueType,
MARKETDESC,
Strategicbrand,
TA,
int(DEFAULTFLAG),
competitormarket,
competitortype,
DATA_SOURCE
from
dwd.dwd_gnd_ims_compete_product_config

70
03 dm_td_external_brand_market.sql Normal file
View File

@@ -0,0 +1,70 @@
-- Databricks notebook source
-- drop table if exists dm.DM_TD_EXTERNAL_BRAND_MARKET;
-- create table if not exists dm.DM_TD_EXTERNAL_BRAND_MARKET
-- using parquet as
INSERT OVERWRITE dm.DM_TD_EXTERNAL_BRAND_MARKET (
TA_NAME,
RNK,
BRAND_CODE,
BRAND_NAME,
KEYCOMPATITORCN,
KEYCOMPATITOREN,
CLASS,
IMSBRAND,
CORP,
MARKET,
MARKETDESC,
DEFAULTFLAG,
VBP,
VBPFLAG,
VALUETYPE,
Strategicbrand,
CompatitorMarket,
CompatitorType,
TA_RN,
DATA_SOURCE
)
SELECT
A.TA TA_NAME,
CASE
WHEN A.RNK = 'null' then 99
ELSE A.RNK
END,
A.BRAND_CODE,
A.BRAND_NAME,
REPLACE(NVL(A.CompatitorCN, ''), 'null', '') KEYCOMPATITORCN,
REPLACE(NVL(A.CompatitorEN, ''), 'null', '') KEYCOMPATITOREN,
REPLACE(NVL(A.CLASS, ''), 'null', '') CLASS,
upper(A.IMSBRAND) IMSBRAND,
REPLACE(NVL(A.CORP, ''), 'null', '') CORP,
REPLACE(NVL(A.MARKET, ''), 'null', '') MARKET,
A.MARKETDESC,
A.DEFAULTFLAG,
CASE
WHEN REPLACE(NVL(A.VBP, ''), 'null', '') = '' THEN 'NonVBP'
ELSE REPLACE(NVL(A.VBP, ''), 'null', '')
END VBP,
CASE
WHEN REPLACE(NVL(A.VBP, ''), 'null', '') <> '' THEN 'VBP'
ELSE 'NonVBP'
END VBPFLAG,
A.VALUETYPE,
A.Strategicbrand,
REPLACE(NVL(A.CompatitorMarket, ''), 'null', '') CompatitorMarket,
REPLACE(NVL(A.CompatitorType, ''), 'null', '') CompatitorType,
CASE
WHEN A.TA = 'CV' THEN 2
WHEN A.TA = 'DM' THEN 1
WHEN A.TA = 'RD' THEN 3
WHEN A.TA = 'RE' THEN 4
WHEN A.TA = 'NEB' THEN 5
WHEN A.TA = 'GI' THEN 6
WHEN A.TA = 'ONC' THEN 7
WHEN A.TA = 'RARE' THEN 8
END TA_RN,
CASE
when A.DATA_SOURCE = 'IQVIA-County(Quarterly)' THEN 'IQVIA-COUNTY(Quarterly)'
ELSE A.DATA_SOURCE
END DATA_SOURCE
FROM
dm.DM_TD_EXTERNAL_KEYCOMPATITOR A

382
04 dm_td_external_calendar.sql Normal file
View File

@@ -0,0 +1,382 @@
-- Databricks notebook source
---------------------------------------------------------------------------
--修改时间：20241126
--修改人：Fanxujia
--修改内容：
--THC渠道分为14省的数据源以及全国的数据源两种，分在了2个事实表里，下面在维表在引用事实表时，需要将两个事实表union在一起
----------------------------------------------------------------------------
create or replace temporary view external_sales_union as
select YYYYMM,SALES_VALUE_CAL,DATA_SOURCE from dm.dm_tf_ext_unionall_sales
union all
select YYYYMM,SALES_VALUE_CAL,DATA_SOURCE from dm.dm_tf_external_sales_thc
-- COMMAND ----------
---------------------------------------------------------------------------
--修改时间：20260309
--修改人：chenwu
--修改内容：
--niad 的数据会比其他数据先入数据库，会产生月份差。
----------------------------------------------------------------------------
create or replace temporary view retail_niad_sales as
select
GREATEST(niad_month, OTHERS_MONTH) as YYYYMM,
NIAD_MONTH
from
(
select
max(niad_month) as niad_month,
max(OTHERS_MONTH) as OTHERS_MONTH
from
(
select
niad_month,
OTHERS_MONTH
from
tmp.tmp_dm_td_ext_retail_niad_month
union all
select
niad_month,
OTHERS_MONTH
from
tmp.tmp_dm_td_ext_ec_niad_month
)
)
-- COMMAND ----------
create or replace temporary view merged_date_calender as
WITH ALL_DATA_YYYY (
select
max(yyyymm) YYYYMM,
'AIA' DATATYPE
from
dm.dm_tf_ext_unionall_sales
where
DATA_SOURCE = 'AIA(Monthly)'
union all
select
max(yyyymm) YYYYMM,
'DTP' DATATYPE
from
dm.dm_tf_ext_unionall_sales
where
DATA_SOURCE = 'DTP(Quarterly)'
union all
select
max(yyyymm) YYYYMM,
'CHPA' DATATYPE
from
dm.dm_tf_ext_unionall_sales
where
DATA_SOURCE = 'IQVIA-CHPA(Monthly)'
)
SELECT
min(YYYYMM) AS MERGED_DATA_MONTH,
GREATEST(
'202401',
CAST((CAST(LEFT(min(yyyymm), 4) AS INT) - 3) * 100 + 1 AS VARCHAR(6)) --三年区间
) AS MERGED_DATA_MONTH_START
from
ALL_DATA_YYYY
-- COMMAND ----------
insert overwrite dm.dm_td_external_calendar (
date,
yyyymmdd,
yyyymm,
year,
quarter,
month,
month_cn,
day,
is_holiday,
work_days,
mtd_work_days,
left_work_days,
year_quater,
half_year,
dd,
YYYYMM_EXTERNAL,
MONTH_EXTERNAL,
YEAR_EXTERNAL,
YYYYMM_EXTERNAL_AIA,
YYYYMM_EXTERNAL_NIAD,
YYYYMM_MERGED_DATA
)
--公用的日历表最小日期不到2018年，因此需要自己构造一个日历表，日期从2018年起。
--31天
with day_31 as (select
explode(
array(
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
14,
15,
16,
17,
18,
19,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31
)
) as day
),
--30天
day_30 as (select
day
from
day_31
where
day <= 30
),
--29天（闰年的2月份）
day_29 as (select
day
from
day_31
where
day <= 29
),
--28天（非闰年的2月份）
day_28 as (select
day
from
day_31
where
day <= 28
),
--最后一天是31号的月份
month_31 as (select
explode(array(1, 3, 5, 7, 8, 10, 12)) as month
),
--最后一天是30号的月份
month_30 as (select
explode(array(4, 6, 9, 11)) as month
),
--2月份单拎出来
month_2 as (select
2 as month
),
--非闰年的月、日
mmdd_365 as (select
month,
day
from
month_31 cross join day_31
union all
select
month,
day
from
month_30 cross join day_30
union all
select
month,
day
from
month_2 cross join day_28
),
--闰年的月、日
mmdd_366 as (select
month,
day
from
month_31 cross join day_31
union all
select
month,
day
from
month_30 cross join day_30
union all
select
month,
day
from
month_2 cross join day_29
),
--注意事项：
--目前年份设计只到2040年，如果2041年依旧使用此套代码，这里需要做修改
--非闰年的年份
year_365 as (select
explode(
array(
2018,
2019,
2021,
2022,
2023,
2025,
2026,
2027,
2029,
2030,
2031,
2033,
2034,
2035,
2037,
2038,
2039
)
) as year
),
--闰年的年份
year_366 as (select
explode(array(2020, 2024, 2028, 2032, 2036, 2040)) as year
),
--从2018年1月1日至2040年12月31日的数据
--最终插入dm表时还会在此基础上框定时间范围
yyyymmdd as (select
year,
month,
day
from
year_365 cross join mmdd_365
union all
select
year,
month,
day
from
year_366 cross join mmdd_366
),
--补充其他需要的字段
tmp_calendar as (select
make_date(year, month, day) as `date`,
year * 10000 + month * 100 + day as yyyymmdd,
year * 100 + month as yyyymm,
concat(month, '月') as month_cn,
year,
month,
day,
case
when month <= 3 then 1
when month <= 6 then 2
when month <= 9 then 3
else 4
end as quarter,
case
when month <= 3 then concat(year, 'Q1')
when month <= 6 then concat(year, 'Q2')
when month <= 9 then concat(year, 'Q3')
else concat(year, 'Q4')
end as year_quater,
case
when month <= 6 then concat(year, 'H1')
else concat(year, 'H2')
end as half_year,
--下面4个字段实际在PBI上没有用到，因此直接设为0
0 as work_days,
0 as mtd_work_days,
0 as is_holiday,
0 as left_work_days
from
yyyymmdd
)
select
`date`,
yyyymmdd,
A.yyyymm,
`year`,
quarter,
`month`,
month_cn,
`day`,
is_holiday,
work_days,
mtd_work_days,
left_work_days,
year_quater,
half_year,
substring(yyyymmdd, 7, 2) dd,
CASE
WHEN A.yyyymm = B.YYYYMM THEN 'R'
ELSE A.yyyymm
END YYYYMM_EXTERNAL,
CASE
WHEN A.yyyymm = B.YYYYMM THEN 'R'
ELSE `month`
END MONTH_EXTERNAL,
CASE
WHEN A.`year` = LEFT(B.YYYYMM, 4) THEN 'R'
ELSE A.`year`
END YEAR_EXTERNAL,
-- from DM.DM_TD_CALENDAR A
----------------------------------------------------
--修改时间：20241211
--修改人：Fanxujia
--修改内容：
--给AIA报告增加单独的R月标记
----------------------------------------------------
case
when A.yyyymm = C.YYYYMM THEN 'R'
WHEN A.yyyymm >= C.min_yyyymm AND A.yyyymm <=C.yyyymm
THEN A.YYYYMM
ELSE NULL
end as YYYYMM_EXTERNAL_AIA,
case
when A.yyyymm = D.NIAD_MONTH THEN 'R'
else A.yyyymm
end as YYYYMM_EXTERNAL_NIAD,
case
when A.yyyymm = E.MERGED_DATA_MONTH THEN 'R'
when A.yyyymm >= E.MERGED_DATA_MONTH_START AND A.YYYYMM<= E.MERGED_DATA_MONTH THEN A.yyyymm
else null
end as YYYYMM_MERGED_DATA
from
tmp_calendar A
join (
select
MAX(YYYYMM) YYYYMM
from
external_sales_union
where
SALES_VALUE_CAL <> 0
) B
on 1 = 1
--获得AIA数据的最新月
cross join (
select
max(YYYYMM) as YYYYMM,min(yyyymm) AS min_yyyymm
from
dm.dm_tf_ext_unionall_sales
where
DATA_SOURCE = 'AIA(Monthly)'
and SALES_VALUE_CAL <> 0
) C
left join retail_niad_sales AS D
left join merged_date_calender as E
WHERE
A.yyyymm >= (
SELECT
MIN(YYYYMM)
FROM
external_sales_union
)
AND A.yyyymm <= (LEFT(D.YYYYMM, 4) * 100 + 12)
AND A.day = 1

26
05 dm_td_external_exchangerate.sql Normal file
View File

@@ -0,0 +1,26 @@
-- Databricks notebook source
/*
修改日期：20250311
修改人：chenwu
修改内容：取sales表中最大日期的年
*/
insert OVERWRITE dm.dm_td_external_exchangerate (
from_currency_id, to_currency_id, az_year, exchange_rate, etl_update_dt, etl_insert_dt
)
select
from_currency_id,
to_currency_id,
az_year,
exchange_rate,
etl_update_dt,
etl_insert_dt
from
dm.DM_TD_EXCHANGE_RATE
where
AZ_YEAR in (
select
left(max(YYYYMM), 4)
from
dm.dm_tf_ext_unionall_sales --取sales表中最大日期的年
)
AND FROM_CURRENCY_ID = 'USD'

2833
07 dm_td_external_packinfo.sql Normal file
View File

File diff suppressed because it is too large Load Diff

83
08 dm_td_external_corp.sql Normal file
View File

@@ -0,0 +1,83 @@
-- Databricks notebook source
----------------------------------------------------------------------------
--修改时间：20241126
--修改人：Fanxujia
--修改内容：
--补充THC渠道的全国的事实表中的pack_cod
----------------------------------------------------------------------------
----------------------------------------------------------------------------
--修改时间：20241129
--修改人：Fanxujia
--修改内容：
--补充Retail直取中DTP的手工文件里的pack_cod
----------------------------------------------------------------------------
with external_sales as (
select distinct PACK_COD
from dm.DM_TF_EXTERNAL_SALES
union
select distinct pack_cod
from dm.dm_tf_external_sales_thc
union
select distinct pack_code
from dm.dm_zk_retail_sales_kpi
where NVL(pack_code,'Others') <> 'Others'
and pack_code <> ''
)
--只存放有外部数据的公司信息数据
INSERT OVERWRITE dm.dm_td_external_corp(CORP_COD,CORP_DES,CORP_DES_C,CORP_TYPE)
SELECT A.CORP_COD,CASE WHEN MAX(CORP_DES) LIKE 'ASTRAZENECA%' THEN 'ASTRAZENECA' ELSE MAX(CORP_DES) END CORP_DES,MAX(CORP_DES_C) CORP_DES_C,
CASE WHEN MAX(UPPER(MNFL_DES)) = 'LOCAL' THEN 'LOCAL'
WHEN UPPER(A.CORP_COD) LIKE '%CORP_COD%' OR UPPER(A.CORP_COD) LIKE '%OTHERS%' THEN ''
ELSE 'MNC' END CORP_TYPE
FROM (select distinct PACK_COD,CORP_COD,CORP_DES,CORP_DES_C,MNFL_DES from DM.dm_td_external_packinfo_temp2) A
INNER JOIN external_sales B ON A.PACK_COD = B.PACK_COD
WHERE A.CORP_COD IS NOT NULL
GROUP BY A.CORP_COD
-- COMMAND ----------
------------ec补充缺少的Corp信息
insert into dm.dm_td_external_corp(CORP_COD,CORP_DES,CORP_DES_C,CORP_TYPE)
select
case when CORP_COD is null or CORP_COD = '' then 'CORP_COD_EC(Monthly)'
else CORP_COD end as CORP_COD,
max(CORP_DES) CORP_DES,
max(CORP_DES_C) CORP_DES,
max(case when upper(MNFL_DES) = 'LOCAL' then 'LOCAL' else 'MNC' end) CORP_TYPE
from dwd.dwd_inc_gnd_retail_b2c_label_total
where CORP_COD in (
select distinct
a.CORP_COD
from (select distinct corp_cod from dm.dm_zk_ec_sales_kpi ) a
where not exists(
select 1 from dm.dm_td_external_corp b
where a.CORP_COD = b.CORP_COD
)
)
group by 1
-- COMMAND ----------
------------Retail补充缺少的Corp信息
insert into dm.dm_td_external_corp(CORP_COD,CORP_DES,CORP_DES_C,CORP_TYPE)
select distinct
case when CORP_COD is null or CORP_COD = '' then 'CORP_COD_Retail(Quarterly)'
else CORP_COD end as CORP_COD,
max(CORP_DES) CORP_DES,
max(CORP_DES_C) CORP_DES,
max(case when upper(MNFL_DES) = 'LOCAL' then 'LOCAL' else 'MNC' end) CORP_TYPE
from dwd.dwd_inc_gnd_retail_b2c_label_total
where CORP_COD in (
select distinct
a.CORP_COD
from (select distinct corp_cod from dm.dm_zk_retail_sales_kpi ) a
where not exists(
select 1 from dm.dm_td_external_corp b
where a.CORP_COD = b.CORP_COD
)
)
group by 1

1968
09 dm_td_external_geo_type.sql Normal file
View File

File diff suppressed because it is too large Load Diff

393
11 DM_TD_EXTERNAL_MARKET_NEW.sql Normal file
View File

@@ -0,0 +1,393 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_UNIONALL_MARKET (
-- MARKET STRING,
-- DATA_SOURCE STRING,
-- TA STRING,
-- AZ_RELATED STRING,
-- MARKET_DESC STRING,
-- TA_RN INT,
-- PRD_FLAG STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market';
-- COMMAND ----------
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_UNIONALL_MARKET_TA (
-- MARKET STRING COMMENT '市场',
-- DATA_SOURCE STRING COMMENT '数据源',
-- TA STRING COMMENT '治疗领域',
-- TA_MAP STRING COMMENT '治疗领域匹配',
-- TA_RN INT COMMENT '治疗领域排名',
-- MARKET_DESC STRING COMMENT '市场描述',
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP
-- )
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market_ta';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_unionall_market_ta';
-- COMMAND ----------
-- DBTITLE 1,pbi页面展示ta标签
-- create or replace table tmp.tmp_td_ext_unionall_market_ta_mapping
-- (
-- market string,
-- ta string,
-- data_source string);
-- INSERT OVERWRITE tmp.tmp_td_ext_unionall_market_ta_mapping (market, ta, data_source) VALUES
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'DM', 'EC(Monthly)'),
-- ('GI Market', 'GI', 'EC(Monthly)'),
-- ('RE Market', 'RE', 'EC(Monthly)'),
-- ('ATTR_EPL', 'CV', 'DTP(Quarterly)'),
-- ('ONC Market', 'ONCO', 'DTP(Quarterly)'),
-- ('RARE Market', 'RARE', 'DTP(Quarterly)'),
-- ('RE Market', 'RE', 'Retail(Quarterly)'),
-- ('CV Market', 'CV', 'Retail(Quarterly)'),
-- ('RD Market', 'RD', 'Retail(Quarterly)'),
-- ('GI Market', 'GI', 'Retail(Quarterly)'),
-- ('DM Market', 'DM', 'THC(Quarterly)'),
-- ('GI Market', 'GI', 'THC(Quarterly)'),
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'THC(Quarterly)'),
-- ('RE Market', 'RE', 'THC(Quarterly)'),
-- ('ONC Market', 'ONCO', 'THC(Quarterly)'),
-- ('CV Market', 'CV', 'THC(Quarterly)'),
-- ('RE Market', 'RE', 'XH Data(Quarterly)'),
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'XH Data(Quarterly)'),
-- ('RD Market', 'RD', 'XH Data(Quarterly)'),
-- ('DM Market', 'DM', 'XH Data(Quarterly)'),
-- ('ONC Market', 'ONCO', 'XH Data(Quarterly)'),
-- ('RARE Market', 'RARE', 'XH Data(Quarterly)'),
-- ('CV Market', 'CV', 'XH Data(Quarterly)'),
-- ('DM Market', 'DM', 'IQVIA-COUNTY(Quarterly)'),
-- ('ONC Market', 'ONCO', 'IQVIA-COUNTY(Quarterly)'),
-- ('GI Market', 'GI', 'IQVIA-COUNTY(Quarterly)'),
-- ('RE Market', 'RE', 'IQVIA-COUNTY(Quarterly)'),
-- ('CV Market', 'CV', 'IQVIA-COUNTY(Quarterly)'),
-- ('RARE Market', 'RARE', 'IQVIA-COUNTY(Quarterly)'),
-- ('RD Market', 'RD', 'IQVIA-COUNTY(Quarterly)'),
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'IQVIA-COUNTY(Quarterly)'),
-- ('ONC Market', 'ONCO', 'IQVIA-CHPA(Monthly)'),
-- ('RE Market', 'RE', 'IQVIA-CHPA(Monthly)'),
-- ('RD Market', 'RD', 'IQVIA-CHPA(Monthly)'),
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'IQVIA-CHPA(Monthly)'),
-- ('RARE Market', 'RARE', 'IQVIA-CHPA(Monthly)'),
-- ('DM Market', 'DM', 'IQVIA-CHPA(Monthly)'),
-- ('GI Market', 'GI', 'IQVIA-CHPA(Monthly)'),
-- ('CV Market', 'CV', 'IQVIA-CHPA(Monthly)'),
-- ('DM Market', 'DM', 'AIA(Monthly)'),
-- ('CV Market', 'CV', 'AIA(Monthly)'),
-- ('GI Market', 'GI', 'AIA(Monthly)'),
-- ('RE Market', 'RE', 'AIA(Monthly)'),
-- ('RARE Market', 'RARE', 'AIA(Monthly)'),
-- ('ONC Market', 'ONCO', 'AIA(Monthly)'),
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'AIA(Monthly)'),
-- ('RD Market', 'RD', 'AIA(Monthly)'),
-- ('GI Market', 'GI', 'CHC(Quarterly)'),
-- ('DM Market', 'DM', 'CHC(Quarterly)'),
-- ('RE Market', 'RE', 'CHC(Quarterly)'),
-- ('CV Market', 'CV', 'CHC(Quarterly)'),
-- ('NIAD (Non-Insulin Anti-Diabetic) Market', 'NIAD', 'CHC(Quarterly)'),
-- ('CVRM Market', 'CVRM', 'Retail(Quarterly)'),
-- ('CVRM Market', 'CVRM', 'EC(Monthly)'),
-- ('CVRM Market', 'CVRM', 'CHC(Quarterly)'),
-- ('CVRM Market', 'CVRM', 'AIA(Monthly)'),
-- ('CVRM Market', 'CVRM', 'IQVIA-COUNTY(Quarterly)'),
-- ('CVRM Market', 'CVRM', 'XH Data(Quarterly)'),
-- ('CVRM Market', 'CVRM', 'IQVIA-CHPA(Monthly)'),
-- ('CVRM Market', 'CVRM', 'THC(Quarterly)')
-- ;
-- COMMAND ----------
-- DBTITLE 1,藏数逻辑标签
-- -- 建表
-- CREATE TABLE IF NOT EXISTS tmp.tmp_td_ext_market_hide_flag (
-- data_source VARCHAR(20),
-- market VARCHAR(100),
-- flag VARCHAR(1)
-- );
-- -- 插入数据（对应 case when 中返回 '1' 的规则）
-- INSERT overwrite tmp.tmp_td_ext_market_hide_flag (data_source, market, flag) VALUES
-- -- Retail(Quarterly) 部分
-- ('Retail(Quarterly)', 'Antacid anti GI swelling agent+PPI Oral', '1'),
-- ('Retail(Quarterly)', 'HTN Market', '1'),
-- ('Retail(Quarterly)', 'NIAD (Non-Insulin Anti-Diabetic) Market', '1'),
-- ('Retail(Quarterly)', 'Respules Market', '1'),
-- ('Retail(Quarterly)', 'Statin+XZK Market', '1'),
-- ('Retail(Quarterly)', 'BUDESONIDE Market', '1'),
-- ('Retail(Quarterly)', 'Rosuvastatin Market', '1'),
-- ('Retail(Quarterly)', 'Atorvastatin Market', '1'),
-- ('Retail(Quarterly)', 'Metoprolol Succinate Market', '1'),
-- ('Retail(Quarterly)', 'PPI Oral ESOMEPRAZOLE Market', '1'),
-- ('Retail(Quarterly)', 'PPI Oral OMEPRAZOLE Market', '1'),
-- ('Retail(Quarterly)', 'Inhaled Extended Market', '1'),
-- ('Retail(Quarterly)', 'Non-Omeprazole', '1'),
-- ('Retail(Quarterly)', 'Metoprolol Tartrate Market', '1'),
-- ('Retail(Quarterly)', 'Pediatric Cough Market', '1'),
-- ('Retail(Quarterly)', 'Anti-HER2 Market', '1'),
-- ('Retail(Quarterly)', 'Anti-HER2 mBC Market', '1'),
-- ('Retail(Quarterly)', 'Fasenra Market', '1'),
-- ('Retail(Quarterly)', 'Severe Asthma', '1'),
-- ('Retail(Quarterly)', 'DAPA Market', '1'),
-- -- DTP(Quarterly) 部分
-- ('DTP(Quarterly)', 'EGFR TKI Market', '1'),
-- ('DTP(Quarterly)', 'Anti-HER2 Market', '1'),
-- ('DTP(Quarterly)', 'Anti-HER2 mBC Market', '1'),
-- ('DTP(Quarterly)', 'Fasenra/Tezspire Related Market', '1'),
-- ('DTP(Quarterly)', 'Severe Asthma', '1'),
-- ('DTP(Quarterly)', 'Tezspire CRSwNP Market', '1'),
-- ('DTP(Quarterly)', 'CRSwNP Market', '1'),
-- ('DTP(Quarterly)', 'PAM Market', '1');
-- COMMAND ----------
-- ============================================================
-- 目标表：dm.dm_td_ext_unionall_market
-- 功能：整合各数据源的市场、TA、AZ关联、产品标志等维度信息
-- 调度：全量覆盖写入（INSERT OVERWRITE）
-- 作者/更新：chenwu
-- ============================================================
INSERT OVERWRITE dm.dm_td_ext_unionall_market (
MARKET, DATA_SOURCE, TA, AZ_RELATED, MARKET_DESC, TA_RN, PRD_FLAG, ETL_INSERT_DT, ETL_UPDATE_DT
)
WITH
-- ------------------------------------------------------------
-- market_info：从包映射表取市场-数据源组合，关联竞品表补充市场描述
-- 用 MAX(MARKET_DESC) 避免一对多 JOIN 导致重复行
-- ------------------------------------------------------------
market_info AS (
SELECT
t1.MARKET,
t1.DATASOURCE AS DATA_SOURCE,
MAX(t2.MARKETDESC) AS MARKET_DESC
FROM dm.dm_td_ext_unionall_market_pack_mapping t1
LEFT JOIN dm.dm_td_external_keycompatitor t2
ON t1.MARKET = t2.MARKET
GROUP BY 1, 2
),
-- ------------------------------------------------------------
-- TA_info：从市场属性表取市场-TA-排序，过滤占位行（MARKET_ORDER=999）
-- 末尾 UNION ALL 补充"ALL Market"汇总行
-- 改用 SELECT DISTINCT 替代无聚合的 GROUP BY
-- ------------------------------------------------------------
TA_info AS (
SELECT DISTINCT
MARKET,
TA,
TA_ORDER
FROM DM.DM_IMS_TD_MARKET_PROPERTY
WHERE MARKET_ORDER <> 999
UNION ALL
-- SELECT
-- 'CVRM Market' AS MARKET,
-- 'CVRM' AS TA,
-- 0 AS TA_ORDER
-- UNION ALL
SELECT
'ALL Market' AS MARKET,
'ALL' AS TA,
-1 AS TA_ORDER
),
-- ------------------------------------------------------------
-- AZ_RELATED_info：从临时映射表取市场-数据源-AZ关联关系（已去重）
-- ------------------------------------------------------------
AZ_RELATED_info AS (
SELECT DISTINCT
MARKET,
TA AS AZ_RELATED,
DATA_SOURCE
FROM tmp.tmp_td_ext_unionall_market_ta_mapping
)
-- ------------------------------------------------------------
-- 主查询：以 market_info 为驱动表，依次关联 TA、AZ关联、隐藏标志
-- hide_flag 为小维表，添加 BROADCAST hint 避免 shuffle
-- PRD_FLAG=1 表示市场在隐藏标志表中存在（需隐藏/特殊处理）
-- ------------------------------------------------------------
SELECT /*+ BROADCAST(t4) */
DISTINCT
t1.MARKET,
t1.DATA_SOURCE,
t2.TA,
t3.AZ_RELATED,
t1.MARKET_DESC,
t2.TA_ORDER AS TA_RN,
IF(t4.MARKET IS NULL, 0, 1) AS PRD_FLAG,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
FROM market_info t1
LEFT JOIN TA_info t2
ON t1.MARKET = t2.MARKET
LEFT JOIN AZ_RELATED_info t3
ON t1.MARKET = t3.MARKET
AND t1.DATA_SOURCE = t3.DATA_SOURCE
LEFT JOIN
tmp.tmp_td_ext_market_hide_flag t4
ON t1.MARKET = t4.MARKET
AND t1.DATA_SOURCE = t4.DATA_SOURCE
-- COMMAND ----------
insert overwrite dm.dm_td_ext_unionall_market_ta (
MARKET, DATA_SOURCE, TA, TA_RN, MARKET_DESC, ETL_INSERT_DT, ETL_UPDATE_DT
)
SELECT
MARKET,
DATA_SOURCE,
AZ_RELATED TA,
TA_RN,
MARKET_DESC,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
FROM
DM.dm_td_ext_unionall_market
-- WHERE MARKET NOT LIKE '%ALL Market%' and MARKET <>'Retail All Market'
-- UNION all
-- SELECT
-- MARKET,
-- DATA_SOURCE,
-- 'CVRM' TA,
-- 0 TA_RN,
-- MARKET_DESC,
-- FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
-- FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
-- FROM
-- DM.dm_td_ext_unionall_market
-- WHERE
-- MARKET = 'CVRM Market'
-- COMMAND ----------
-----------------------------------------------------------------------------------
--修改时间：20240822
--修改人：FanXujia
--修改内容：
--Retail、EC渠道，新增NIAD这个TA。数据与DM相同
-----------------------------------------------------------------------------------
insert into dm.dm_td_ext_unionall_market_ta (
MARKET, DATA_SOURCE, TA, TA_RN, MARKET_DESC, ETL_INSERT_DT, ETL_UPDATE_DT
)
select
market,
DATA_SOURCE,
'NIAD' as TA,
7 as TA_RN,
MARKET_DESC,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
from
dm.dm_td_ext_unionall_market_ta
where
DATA_SOURCE = 'Retail(Quarterly)'
and ta = 'DM'
union all
select
market,
DATA_SOURCE,
'NIAD' as TA,
7 as TA_RN,
MARKET_DESC,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
from
dm.dm_td_ext_unionall_market_ta
where
DATA_SOURCE = 'EC(Monthly)'
and ta = 'DM'
--20250701 chenwu 新增RARE = ONCO的Soliris Related Market
union all
select
market,
DATA_SOURCE,
'RARE' as TA,
8 as TA_RN,
MARKET_DESC,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
from
dm.dm_td_ext_unionall_market_ta
where
DATA_SOURCE = 'DTP(Quarterly)'
and ta = 'ONCO'
and MARKET = 'Soliris Related Market';
-- COMMAND ----------
-- log 20250226 auth:庄伟
-- 对market_ta表中市场进行补全，并依据手工文件维护的 market 与 ta 映射，匹配出完整市场对应的TA_NAME
insert overwrite table dm.dm_td_ext_unionall_market_ta (
MARKET, DATA_SOURCE, TA, TA_MAP, TA_RN, MARKET_DESC, ETL_INSERT_DT, ETL_UPDATE_DT
)
-----找出在market维度表中不存在的market列表
with temp_ma_ext_market as (select distinct
MARKET,
DATA_SOURCE
from
dm.dm_td_ext_unionall_market
where
MARKET NOT IN (
select distinct
MARKET
from
dm.dm_td_ext_unionall_market_ta
)
)
------补充完整的market列表，并通过market匹配手工文件维护的market对应的TA
select
distinct
dim_mkt_ta.MARKET,
DATA_SOURCE,
dim_mkt_ta.TA,
case
when dim_mkt_ta.MARKET = 'ALL Market' then 'ALL'
else nvl(mkt_property.ta, mkt_property_key.TA)
end as TA_MAP,
dim_mkt_ta.TA_RN,
dim_mkt_ta.MARKET_DESC,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
from
(
select
MARKET,
DATA_SOURCE,
TA,
TA_RN,
MARKET_DESC
from
dm.dm_td_ext_unionall_market_ta
) dim_mkt_ta
left join dm.dm_ims_td_market_property mkt_property
on dim_mkt_ta.market = mkt_property.market
left join dm.dm_ims_td_market_property mkt_property_key
on dim_mkt_ta.MARKET = mkt_property_key.MARKET_KEY
union all
select
ext_market.MARKET,
ext_market.DATA_SOURCE,
null TA,
mkt_property.TA TA_MAP,
'99' as TA_RN,
null MARKET_DESC,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
from
temp_ma_ext_market ext_market
left join dm.dm_ims_td_market_property mkt_property
on ext_market.market = mkt_property.MARKET
where
nvl(mkt_property.TA, '') <> '';

358
12 dm_td_external_org.sql Normal file
View File

@@ -0,0 +1,358 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXTERNAL_ORG (
-- REGION_CODE STRING,
-- region_name_en STRING,
-- RSM_Name STRING,
-- rc_code STRING,
-- rc_name_en STRING,
-- RC_Name STRING,
-- RegionCenter STRING,
-- AUDIT_COD STRING,
-- MARKET STRING,
-- bu_category STRING,
-- bu_name_en STRING,
-- sub_bu_name STRING,
-- sub_bu_code_auth STRING,
-- bu_code_auth STRING,
-- DATA_SOURCE STRING,
-- REGION_RATIO FLOAT,
-- TEAM STRING,
-- MR_TRTY_CODE STRING,
-- NSD_KCODE STRING,
-- NSD_NAME STRING,
-- RSD_KCODE STRING,
-- RSD_NAME STRING,
-- RSM_KCODE STRING,
-- DSM_KCODE STRING,
-- DSM_NAME STRING,
-- MR_KCODE STRING,
-- MR_NAME STRING)
-- USING delta
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_external_org';
-- COMMAND ----------
-- DBTITLE 1,UNIONALL_ORG
CREATE OR REPLACE TEMPORARY VIEW UNIONALL_ORG
AS
SELECT
REGION_CODE,
REGION_NAME_EN,
RSM_NAME,
RC_CODE,
RC_NAME_EN,
RC_NAME,
REGIONCENTER,
AUDIT_COD,
MARKET,
BU_CATEGORY,
BU_NAME_EN,
SUB_BU_NAME,
SUB_BU_CODE_AUTH,
BU_CODE_AUTH,
DATA_SOURCE,
REGION_RATIO,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
FROM
DM.DM_TD_EXT_AIA_ORG
UNION ALL
SELECT
REGION_CODE,
REGION_NAME_EN,
RSM_NAME,
RC_CODE,
RC_NAME_EN,
RC_NAME,
REGIONCENTER,
AUDIT_COD,
MARKET,
BU_CATEGORY,
BU_NAME_EN,
SUB_BU_NAME,
SUB_BU_CODE_AUTH,
BU_CODE_AUTH,
DATA_SOURCE,
REGION_RATIO,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
FROM
DM.DM_TD_EXT_CHC_ORG
UNION ALL
SELECT
REGION_CODE,
REGION_NAME_EN,
RSM_NAME,
RC_CODE,
RC_NAME_EN,
RC_NAME,
REGIONCENTER,
AUDIT_COD,
MARKET,
BU_CATEGORY,
BU_NAME_EN,
SUB_BU_NAME,
SUB_BU_CODE_AUTH,
BU_CODE_AUTH,
DATA_SOURCE,
REGION_RATIO,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
FROM
DM.DM_TD_EXT_CHPA_ORG
UNION ALL
SELECT
REGION_CODE,
REGION_NAME_EN,
RSM_NAME,
RC_CODE,
RC_NAME_EN,
RC_NAME,
REGIONCENTER,
AUDIT_COD,
MARKET,
BU_CATEGORY,
BU_NAME_EN,
SUB_BU_NAME,
SUB_BU_CODE_AUTH,
BU_CODE_AUTH,
DATA_SOURCE,
REGION_RATIO,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
FROM
DM.DM_TD_EXT_COUNTY_ORG
UNION ALL
SELECT
REGION_CODE,
REGION_NAME_EN,
RSM_NAME,
RC_CODE,
RC_NAME_EN,
RC_NAME,
REGIONCENTER,
AUDIT_COD,
MARKET,
BU_CATEGORY,
BU_NAME_EN,
SUB_BU_NAME,
SUB_BU_CODE_AUTH,
BU_CODE_AUTH,
DATA_SOURCE,
REGION_RATIO,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
FROM
DM.DM_TD_EXT_EC_ORG
UNION ALL
SELECT
REGION_CODE,
REGION_NAME_EN,
RSM_NAME,
RC_CODE,
RC_NAME_EN,
RC_NAME,
REGIONCENTER,
AUDIT_COD,
MARKET,
BU_CATEGORY,
BU_NAME_EN,
SUB_BU_NAME,
SUB_BU_CODE_AUTH,
BU_CODE_AUTH,
DATA_SOURCE,
REGION_RATIO,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
FROM
DM.DM_TD_EXT_RETAIL_ORG
UNION ALL
SELECT
REGION_CODE,
REGION_NAME_EN,
RSM_NAME,
RC_CODE,
RC_NAME_EN,
RC_NAME,
REGIONCENTER,
AUDIT_COD,
MARKET,
BU_CATEGORY,
BU_NAME_EN,
SUB_BU_NAME,
SUB_BU_CODE_AUTH,
BU_CODE_AUTH,
DATA_SOURCE,
REGION_RATIO,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
FROM
DM.DM_TD_EXT_THC_ORG
UNION ALL
SELECT
REGION_CODE,
REGION_NAME_EN,
RSM_NAME,
RC_CODE,
RC_NAME_EN,
RC_NAME,
REGIONCENTER,
AUDIT_COD,
MARKET,
BU_CATEGORY,
BU_NAME_EN,
SUB_BU_NAME,
SUB_BU_CODE_AUTH,
BU_CODE_AUTH,
DATA_SOURCE,
REGION_RATIO,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
FROM
DM.DM_TD_EXT_XIEHE_ORG
-- COMMAND ----------
-- DBTITLE 1,overwrite dm.dm_td_external_org
insert overwrite dm.dm_td_external_org
select
REGION_CODE,
region_name_en,
RSM_Name,
rc_code,
rc_name_en,
RC_Name,
RegionCenter,
AUDIT_COD,
MARKET,
bu_category,
bu_name_en,
sub_bu_name,
sub_bu_code_auth,
bu_code_auth,
DATA_SOURCE,
REGION_RATIO,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
from
UNIONALL_ORG
-- COMMAND ----------
-- /*
-- 修改时间：2025.09.18
-- 修改人：chenwu
-- 修改内容：新增一个org表给merged data使用，原org表在13 external auth 中会被删除掉'BBU_County','CHC','EC','Eagle','Retail'
-- 故要在被删除前将这部分数据保留，供pbi端打目标机构标签
-- */
INSERT OVERWRITE DM.DM_TD_EXTERNAL_ORG_MERGED_DATA
SELECT
REGION_CODE,
REGION_NAME_EN,
RSM_NAME,
RC_CODE,
RC_NAME_EN,
RC_NAME,
REGIONCENTER,
AUDIT_COD,
MARKET,
BU_CATEGORY,
BU_NAME_EN,
SUB_BU_NAME,
SUB_BU_CODE_AUTH,
BU_CODE_AUTH,
DATA_SOURCE,
REGION_RATIO,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
FROM DM.DM_TD_EXTERNAL_ORG

1230
13 external auth.sql Normal file
View File

File diff suppressed because it is too large Load Diff

22
14 dm_tf_external_retail_special_bkp.sql Normal file
View File

@@ -0,0 +1,22 @@
-- Databricks notebook source
insert overwrite table dm.dm_tf_external_retail_special
--dtp 部分的数据 EGFR TKI 需要直取
select
case when a.market = 'EGFR TKI' then 'EGFR TKI Market' else a.market end as market
,a.province_city
,a.audit_key
,a.audit_source
,a.date_key
,a.yyyymm
,a.sales_value
,a.sales_volume
,a.pdot_counting_unit
,c.sales_value sales_value_ly
,c.sales_volume sales_volume_ly
,c.pdot_counting_unit pdot_counting_unit_ly
,'dtp' `source`
from dws.dws_tf_external_retail_dtp_special a
left join dws.dws_tf_external_retail_dtp_special c --取去年同期数据
on a.province_city = c.province_city
and a.market = c.market
and (a.yyyymm-100) = c.yyyymm

355
AIA/01 dm_aia_pack_property.sql Normal file
View File

@@ -0,0 +1,355 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE dm.dm_aia_pack_property (
-- PACK_COD STRING,
-- pack_des STRING,
-- stgh_des STRING,
-- pack_lch STRING,
-- PROD_COD STRING,
-- prod_des STRING,
-- prod_des_c STRING,
-- Family_Code STRING,
-- Family_Name STRING,
-- cmps_cod STRING,
-- cmps_des STRING,
-- cmps_des_c STRING,
-- atc1_cod STRING,
-- atc1_des STRING,
-- atc1_des_c STRING,
-- atc2_cod STRING,
-- atc2_des STRING,
-- atc2_des_c STRING,
-- atc3_cod STRING,
-- atc3_des STRING,
-- atc3_des_c STRING,
-- atc4_cod STRING,
-- atc4_des STRING,
-- atc4_des_c STRING,
-- app1_cod STRING,
-- app1_des STRING,
-- app1_des_c STRING,
-- app2_cod STRING,
-- app2_des STRING,
-- app2_des_c STRING,
-- app3_cod STRING,
-- app3_des STRING,
-- app3_des_c STRING,
-- bio_desc STRING,
-- gene_orig_desc STRING,
-- eth_otc_desc STRING,
-- nrdl_desc STRING,
-- NRDL_Entry_Date STRING,
-- edl_desc STRING,
-- tcm_desc STRING,
-- paed_desc STRING,
-- gqce_desc STRING,
-- vbp_desc STRING,
-- manu_cod STRING,
-- manu_des STRING,
-- manu_des_c STRING,
-- mnfl_cod STRING,
-- mnfl_des STRING,
-- corp_cod STRING,
-- corp_des STRING,
-- corp_des_c STRING,
-- BrandType STRING,
-- IS_AZ STRING,
-- AZ_MAIN STRING,
-- AZ_Related STRING,
-- COUNTING_UNIT STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_aia_pack_property';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_aia_pack_property';
-- COMMAND ----------
--对用户上传的表进行去重处理
--对部分编码字段进行补0
create or replace temporary view aia_pack_property_distinct as
select distinct
iqvia_lineno,
case
when
length(trim(iqvia_pack_code)) < 12
and trim(iqvia_pack_code) REGEXP '^[0-9]'
then
right(concat('000000000000', trim(iqvia_pack_code)), 12)
else trim(iqvia_pack_code)
end as iqvia_pack_code,
case
when length(iqvia_prod_code) < 9 then right(concat('000000000', iqvia_prod_code), 9)
else iqvia_prod_code
end as iqvia_prod_code,
iqvia_notes,
IS_NEW_PRD,
IS_IN_CHPA,
ATC1,
ATC2,
ATC3,
ATC4,
org_prd_cd,
org_mole_nm_c,
org_manu_prd_nm_c,
org_prd_nm_c,
org_pk,
org_prd_str,
org_pth,
min_pk_unit,
org_pk_unit,
org_manu_nm,
app1_cod,
app1_des,
app1_des_c,
app2_cod,
app2_des,
app2_des_c,
app3_cod,
app3_des,
app3_des_c,
atc1_cod,
atc1_des,
atc1_des_c,
atc2_cod,
atc2_des,
atc2_des_c,
atc3_cod,
atc3_des,
atc3_des_c,
atc4_cod,
atc4_des,
atc4_des_c,
bio_desc,
case
when length(cmps_cod) < 6 then right(concat('000000', cmps_cod), 6)
else cmps_cod
end as cmps_cod,
cmps_des,
cmps_des_c,
corp_cod,
corp_des,
corp_des_c,
edl_desc,
eth_otc_desc,
gene_orig_desc,
gqce_desc,
manu_cod,
manu_des,
manu_des_c,
mnfl_cod,
mnfl_des,
nrdl_desc,
pack_des,
stgh_des,
pack_lch,
paed_desc,
prod_des,
prod_des_c,
tcm_desc,
vbp_desc,
unit,
case
when
counting_unit is null
and org_pk_unit is null
then
1
when counting_unit is null then cast(org_pk_unit as decimal(30, 10))
else cast(counting_unit / coalesce(unit, 1) as decimal(30, 10))
end as counting_unit,
dosage_unit
from
dwd.dwd_gnd_ext_aia_cpt_data;
-- COMMAND ----------
-- create or replace table dm.dm_aia_pack_property
insert overwrite table dm.dm_aia_pack_property
WITH dedup_l AS (
SELECT *,
ROW_NUMBER() OVER (PARTITION BY iqvia_pack_code ORDER BY iqvia_pack_code) AS rn
FROM aia_pack_property_distinct
QUALIFY rn = 1
),
dedup_r AS (
SELECT *,
ROW_NUMBER() OVER (PARTITION BY PACK_COD ORDER BY PACK_COD) AS rn
FROM tmp.tmp_ext_chpa_pack_propetry
QUALIFY rn = 1
)
SELECT
l.iqvia_pack_code AS PACK_COD,
-- Pack / Product 描述
COALESCE(r.PACK_DES, l.pack_des) AS pack_des,
COALESCE(r.STGH_DES, l.stgh_des) AS stgh_des,
COALESCE(r.PACK_LCH, l.pack_lch) AS pack_lch,
COALESCE(r.PROD_COD, l.iqvia_prod_code) AS PROD_COD,
COALESCE(r.PROD_DES, l.prod_des) AS prod_des,
COALESCE(r.PROD_DES_C, l.prod_des_c) AS prod_des_c,
-- 右表独有：Family
r.Family_Code,
r.Family_Name,
-- 成分
COALESCE(r.CMPS_COD, l.cmps_cod) AS cmps_cod,
COALESCE(r.CMPS_DES, l.cmps_des) AS cmps_des,
COALESCE(r.CMPS_DES_C, l.cmps_des_c) AS cmps_des_c,
-- ATC
COALESCE(r.ATC1_COD, l.atc1_cod) AS atc1_cod,
COALESCE(r.atc1_des, l.atc1_des) AS atc1_des,
COALESCE(r.atc1_des_c, l.atc1_des_c) AS atc1_des_c,
COALESCE(r.ATC2_COD, l.atc2_cod) AS atc2_cod,
COALESCE(r.atc2_des, l.atc2_des) AS atc2_des,
COALESCE(r.atc2_des_c, l.atc2_des_c) AS atc2_des_c,
COALESCE(r.ATC3_COD, l.atc3_cod) AS atc3_cod,
COALESCE(r.atc3_des, l.atc3_des) AS atc3_des,
COALESCE(r.atc3_des_c, l.atc3_des_c) AS atc3_des_c,
COALESCE(r.ATC4_COD, l.atc4_cod) AS atc4_cod,
COALESCE(r.atc4_des, l.atc4_des) AS atc4_des,
COALESCE(r.atc4_des_c, l.atc4_des_c) AS atc4_des_c,
-- APP
COALESCE(r.APP1_COD, l.app1_cod) AS app1_cod,
COALESCE(r.app1_des, l.app1_des) AS app1_des,
COALESCE(r.app1_des_c, l.app1_des_c) AS app1_des_c,
COALESCE(r.APP2_COD, l.app2_cod) AS app2_cod,
COALESCE(r.app2_des, l.app2_des) AS app2_des,
COALESCE(r.app2_des_c, l.app2_des_c) AS app2_des_c,
COALESCE(r.APP3_COD, l.app3_cod) AS app3_cod,
COALESCE(r.app3_des, l.app3_des) AS app3_des,
COALESCE(r.app3_des_c, l.app3_des_c) AS app3_des_c,
-- 标志位
COALESCE(r.BIO_DESC, l.bio_desc) AS bio_desc,
COALESCE(r.GENE_ORIG_DESC, l.gene_orig_desc) AS gene_orig_desc,
COALESCE(r.ETH_OTC_DESC, l.eth_otc_desc) AS eth_otc_desc,
COALESCE(r.NRDL_DESC, l.nrdl_desc) AS nrdl_desc,
r.NRDL_Entry_Date,
COALESCE(r.EDL_DESC, l.edl_desc) AS edl_desc,
COALESCE(r.TCM_DESC, l.tcm_desc) AS tcm_desc,
COALESCE(r.PAED_DESC, l.paed_desc) AS paed_desc,
COALESCE(r.GQCE_DESC, l.gqce_desc) AS gqce_desc,
COALESCE(r.VBP_DESC, l.vbp_desc) AS vbp_desc,
-- 厂商
COALESCE(r.MANU_COD, l.manu_cod) AS manu_cod,
COALESCE(r.MANU_DES, l.manu_des) AS manu_des,
COALESCE(r.MANU_DES_C, l.manu_des_c) AS manu_des_c,
COALESCE(r.MNFL_COD, l.mnfl_cod) AS mnfl_cod,
COALESCE(r.MNFL_DES, l.mnfl_des) AS mnfl_des,
-- 公司
COALESCE(r.CORP_COD, l.corp_cod) AS corp_cod,
COALESCE(r.CORP_DES, l.corp_des) AS corp_des,
COALESCE(r.CORP_DES_C, l.corp_des_c) AS corp_des_c,
-- 右表独有：品牌 / AZ 标志
r.BrandType,
r.IS_AZ,
r.AZ_MAIN,
r.AZ_Related,
-- 左表独有
l.COUNTING_UNIT,
from_utc_timestamp(current_timestamp(),'UTC+8') as ETL_INSERT_DT,
from_utc_timestamp(current_timestamp(),'UTC+8') as ETL_UPDATE_DT
FROM dedup_l l
LEFT JOIN dedup_r r
ON l.iqvia_pack_code = r.PACK_COD
-- COMMAND ----------
-- -----------------------------------------------------------
-- --修改时间：20241118
-- --修改人：Fanxujia
-- --修改内容：
-- --使用手工文件排除掉每个渠道的不需要显示的市场
-- --由于AIA涉及后面的Flag逻辑、NEW AIA Dashboard，因此不能直接在最后的DM表进行限制，需要在这里限制。
-- -----------------------------------------------------------
-- with hidden_market as (
-- select distinct
-- upper(hidden_market) as hidden_market
-- from dwd.dwd_gnd_ims_hidden_market
-- where upper(data_source) = 'AIA'
-- )
-- ,tmp as (
-- select
-- MARKET_PACK_KEY,
-- PACK_COD,
-- PACK_DES,
-- STGH_DES,
-- PACK_LCH,
-- Family_Code,
-- Family_Name,
-- PROD_COD,
-- PROD_DES,
-- PROD_DES_C,
-- CMPS_COD,
-- CMPS_DES,
-- CMPS_DES_C,
-- ATC1_COD,
-- ATC2_COD,
-- ATC3_COD,
-- ATC4_COD,
-- APP1_COD,
-- APP2_COD,
-- APP3_COD,
-- BIO_DESC,
-- GENE_ORIG_DESC,
-- ETH_OTC_DESC,
-- NRDL_DESC,
-- NRDL_Entry_Date,
-- EDL_DESC,
-- TCM_DESC,
-- PAED_DESC,
-- GQCE_DESC,
-- VBP_DESC,
-- MANU_COD,
-- MANU_DES,
-- MANU_DES_C,
-- MNFL_COD,
-- MNFL_DES,
-- CORP_COD,
-- CORP_DES,
-- CORP_DES_C,
-- BrandType,
-- t1.MARKET,
-- t1.MARKET_CHPA,
-- KEY_COMPETITOR,
-- IS_AZ,
-- AZ_MAIN,
-- AZ_Related,
-- atc1_des,
-- atc1_des_c,
-- atc2_des,
-- atc2_des_c,
-- atc3_des,
-- atc3_des_c,
-- atc4_des,
-- atc4_des_c,
-- app1_des,
-- app1_des_c,
-- app2_des,
-- app2_des_c,
-- app3_des,
-- app3_des_c,
-- Class,
-- ETL_INSERT_DT,
-- ETL_UPDATE_DT,
-- market_ratio,
-- counting_unit
-- from dm.dm_aia_pack_property t1
-- left join hidden_market t3
-- on upper(t1.MARKET) = t3.hidden_market
-- where t3.hidden_market is null
-- union
-- select *
-- from dm.dm_aia_pack_property
-- where MARKET = 'AIA ALL Market'
-- )
-- insert overwrite dm.dm_aia_pack_property
-- select * from tmp

710
AIA/02 dm_ext_aia_sales.sql Normal file
View File

@@ -0,0 +1,710 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TF_EXT_AIA_SALES (
-- YYYYMM STRING,
-- PACK_CODE STRING,
-- CORP_CODE STRING,
-- AUDIT_CODE STRING,
-- PLATFORM_TYPE STRING,
-- STORE_NAME STRING,
-- STORE_TYPE STRING,
-- REGION_TYPE STRING,
-- PACK_FLAG INT,
-- PROD_FLAG INT,
-- DTP_FLAG INT,
-- SALES_UNIT_CAL DECIMAL(38,10),
-- SALES_UNIT_CAL_LY DECIMAL(38,10),
-- SALES_VALUE_CAL DECIMAL(38,10),
-- SALES_VALUE_CAL_LY DECIMAL(38,10),
-- CONUTING_UNIT DECIMAL(38,10),
-- CONUTING_UNIT_LY DECIMAL(38,10),
-- DATA_SOURCE STRING,
-- INST_CODE STRING COMMENT '内部机构编码',
-- CMPS_FLAG STRING COMMENT '分子式标签',
-- DEPT_NAME STRING COMMENT '科室名称',
-- PRESCRIPTION DECIMAL(38,10) COMMENT '处方张数',
-- PRESCRIPTION_LY DECIMAL(38,10) COMMENT '去年同期处方张数',
-- NEW_CODE STRING COMMENT '主数据关联CODE',
-- AREA STRING COMMENT '城市',
-- H_LEVEL STRING COMMENT '医院类型',
-- REIMBURSE STRING COMMENT '报销情况',
-- REIMBURSE_TYPE STRING COMMENT '报销类型',
-- PRESCRIPTION_SOURCE STRING COMMENT '处方来源',
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP
-- )
-- USING delta
-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_aia_sales';
-- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_aia_sales';
-- COMMAND ----------
-- MAGIC %run ../../../Common/config
-- COMMAND ----------
-- MAGIC %python
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dim_product_wide`').createOrReplaceTempView('cdw_dwd_dim_product_wide')
-- COMMAND ----------
-- 原始数据
-- 字段值的初步处理
CREATE OR REPLACE TEMPORARY VIEW AIA_RAWDATA
AS
SELECT
YEAR,
QTR,
CAST(YEAR * 100 + YM AS INT) YM,
ORG_PROV,
ORG_CITY,
INS_LEVEL,
CASE
WHEN
INS_CD IS NOT NULL
AND
LENGTH (INS_CD) < 7
THEN RIGHT (CONCAT ('0000000', INS_CD), 7)
ELSE INS_CD
END AS ORG_INS_CD,
ATC1,
ATC2,
ATC3,
ATC4,
ATC,
MOLE_NM,
PROD_NM,
ORG_PRD_NM_C,
ORG_PK,
ORG_PRD_STR,
ORG_PTH,
MIN_UNIT,
VALUE,
UNIT,
CAST(PACK_DESC AS INT) AS PACK_DESC,
ORG_PK_UN,
MANU
FROM
DWD.DWD_GND_AIA_RAWDATA
-- COMMAND ----------
--关联出AUDIT_COD、INST_CODE
CREATE OR REPLACE TEMPORARY VIEW AIA_RAWDATA_WITH_AUDIT_INST
AS
SELECT
UPPER(NVL (T2.INST_CODE, T1.ORG_INS_CD)) AS AUDIT_COD,
T1.YEAR,
T1.QTR,
T1.YM,
T1.ORG_PROV,
T1.ORG_CITY,
T1.INS_LEVEL,
T1.ORG_INS_CD,
T1.ATC1,
T1.ATC2,
T1.ATC3,
T1.ATC4,
T1.ATC,
T1.MOLE_NM,
T1.PROD_NM,
T1.ORG_PRD_NM_C,
T1.ORG_PK,
T1.ORG_PRD_STR,
T1.ORG_PTH,
T1.MIN_UNIT,
T1.VALUE,
T1.UNIT,
T1.PACK_DESC,
T1.ORG_PK_UN,
T1.MANU,
NVL (T2.INST_CODE, T1.ORG_INS_CD) AS INST_CODE
FROM
AIA_RAWDATA AS T1
LEFT JOIN
(
SELECT
DISTINCT
CASE
WHEN LENGTH (CPA_HOSPITAL_CODE) < 7
THEN RIGHT (CONCAT ('0000000', CPA_HOSPITAL_CODE), 7)
ELSE CPA_HOSPITAL_CODE
END AS CPA_HOSPITAL_CODE,
INS_CD_NL AS INST_CODE
FROM
DWD.DWD_GND_HOSPITAL_NOT_PROVIDED
) AS T2
ON
T1.ORG_INS_CD = T2.CPA_HOSPITAL_CODE
-- COMMAND ----------
/*
修改人 CHENWU
修改时间 20250513
修改内容 打包分子的数据，不论 打通表 和 事实表，都没有系数，需要用 事实表里的 最小制剂单位数量 作为COUNTING_UNIT
*/
--关联出PACK_COD、CORP_COD、COUNTING_UNIT
CREATE OR REPLACE TEMPORARY VIEW AIA_RAWDATA_WIHT_COUNTING_UNIT
AS
SELECT
CASE
WHEN
LENGTH (TRIM(T2.IQVIA_PACK_CODE)) < 12
AND
TRIM(T2.IQVIA_PACK_CODE) REGEXP '^[0-9]'
THEN RIGHT (CONCAT ('000000000000', TRIM(T2.IQVIA_PACK_CODE)),12)
ELSE TRIM(T2.IQVIA_PACK_CODE)
END AS PACK_COD,
T3.CORP_COD,
CASE
WHEN LEFT (T2.IQVIA_PACK_CODE, 4) = 'AZP_'
THEN CAST(T1.MIN_UNIT / T1.ORG_PK_UN AS DECIMAL(38, 10))
ELSE T3.COUNTING_UNIT
END COUNTING_UNIT,
T1.AUDIT_COD,
T1.YEAR,
T1.QTR,
T1.YM,
T1.ORG_PROV,
T1.ORG_CITY,
T1.INS_LEVEL,
T1.ORG_INS_CD,
T1.ATC1,
T1.ATC2,
T1.ATC3,
T1.ATC4,
T1.ATC,
T1.MOLE_NM,
T1.PROD_NM,
T1.ORG_PRD_NM_C,
T1.ORG_PK,
T1.ORG_PRD_STR,
T1.ORG_PTH,
T1.MIN_UNIT,
T1.VALUE,
T1.UNIT,
T1.PACK_DESC,
T1.ORG_PK_UN,
T1.MANU,
T1.INST_CODE
FROM
AIA_RAWDATA_WITH_AUDIT_INST T1
LEFT JOIN
(
SELECT
DISTINCT
TRIM(IQVIA_PACK_CODE) AS IQVIA_PACK_CODE,
ATC1,
ATC2,
ATC3,
ATC4,
ORG_PRD_CD,
ORG_MOLE_NM_C,
ORG_MANU_PRD_NM_C,
ORG_PRD_NM_C,
ORG_PK,
ORG_PRD_STR,
ORG_PTH,
MIN_PK_UNIT,
ORG_PK_UNIT,
ORG_MANU_NM
FROM
DWD.DWD_GND_EXT_AIA_CPT_DATA
) AS T2
--ATC1编码+ATC2编码+ATC3编码+ATC4编码+药品编码(ATC)+药品通用名+药品产品名+药品商品名+规格+剂型+给药途径+最小销售包装单位+包装规格+生产企业
ON
NVL (T1.ATC1, '') = NVL (T2.ATC1, '')
AND NVL (T1.ATC2, '') = NVL (T2.ATC2, '')
AND NVL (T1.ATC3, '') = NVL (T2.ATC3, '')
AND NVL (T1.ATC4, '') = NVL (T2.ATC4, '')
AND NVL (T1.ATC, '') = NVL (T2.ORG_PRD_CD, '')
AND NVL (T1.MOLE_NM, '') = NVL (T2.ORG_MOLE_NM_C, '')
AND NVL (T1.PROD_NM, '') = NVL (T2.ORG_MANU_PRD_NM_C, '')
AND NVL (T1.ORG_PRD_NM_C, '') = NVL (T2.ORG_PRD_NM_C, '')
AND NVL (T1.ORG_PK, '') = NVL (T2.ORG_PK, '')
AND NVL (T1.ORG_PRD_STR, '') = NVL (T2.ORG_PRD_STR, '')
AND NVL (T1.ORG_PTH, '') = NVL (T2.ORG_PTH, '')
AND NVL (T1.UNIT, '') = NVL (T2.MIN_PK_UNIT, '')
AND NVL (T1.PACK_DESC, '') = NVL (T2.ORG_PK_UNIT, '')
AND NVL (T1.MANU, '') = NVL (T2.ORG_MANU_NM, '')
LEFT JOIN
(
SELECT
PACK_COD,
CORP_COD,
COUNTING_UNIT
FROM
DM.DM_AIA_PACK_PROPERTY
GROUP BY
PACK_COD,
CORP_COD,
COUNTING_UNIT
) T3 ON T2.IQVIA_PACK_CODE = T3.PACK_COD
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW FILTER_YM
AS
SELECT
MAX(YM) AS MAX_YYYYMM,
CONCAT(CAST(FLOOR(MAX(YM)/100-2) AS STRING),'01') AS MIN_YYYYMM
FROM
AIA_RAWDATA_WIHT_COUNTING_UNIT
-- COMMAND ----------
--计算出LY销量
CREATE OR REPLACE TEMPORARY VIEW AIA_RAWDATA_FINAL AS
SELECT
YM,
ORG_INS_CD,
INST_CODE,
AUDIT_COD,
PACK_COD,
CORP_COD,
SUM(VALUE) AS VALUE,
SUM(VOLUME) AS VOLUME,
SUM(COUNTING_UNIT) AS COUNTING_UNIT,
SUM(VALUE_LY) AS VALUE_LY,
SUM(VOLUME_LY) AS VOLUME_LY,
SUM(COUNTING_UNIT_LY) AS COUNTING_UNIT_LY
FROM
(
SELECT
YM,
ORG_INS_CD,
INST_CODE,
AUDIT_COD,
PACK_COD,
CORP_COD,
CAST(VALUE AS DECIMAL(38, 10)) AS VALUE,
CAST(ORG_PK_UN AS DECIMAL(38, 10)) AS VOLUME,
CAST(ORG_PK_UN AS DECIMAL(38, 10)) * COALESCE(CAST(COUNTING_UNIT AS DECIMAL(38, 10)), 1) AS COUNTING_UNIT,
0 AS VALUE_LY,
0 AS VOLUME_LY,
0 AS COUNTING_UNIT_LY
FROM
AIA_RAWDATA_WIHT_COUNTING_UNIT
UNION ALL
SELECT
CAST(CAST(YM AS INT) + 100 AS STRING) AS YM,
ORG_INS_CD,
INST_CODE,
AUDIT_COD,
PACK_COD,
CORP_COD,
0 AS VALUE,
0 AS VOLUME,
0 AS COUNTING_UNIT,
CAST(VALUE AS DECIMAL(38, 10)) AS VALUE_LY,
CAST(ORG_PK_UN AS DECIMAL(38, 10)) AS VOLUME_LY,
CAST(ORG_PK_UN AS DECIMAL(38, 10)) * COALESCE(CAST(COUNTING_UNIT AS DECIMAL(38, 10)), 1) AS COUNTING_UNIT_LY
FROM
AIA_RAWDATA_WIHT_COUNTING_UNIT T1
JOIN
FILTER_YM T2
ON
1 = 1
WHERE
CAST(CAST(YM AS INT) + 100 AS STRING) <= T2.MAX_YYYYMM
)
GROUP BY
YM,
ORG_INS_CD,
INST_CODE,
AUDIT_COD,
PACK_COD,
CORP_COD
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW FINAL_DM_TF_EXT_AIA_SALES
AS
SELECT
YM,
ORG_INS_CD,
INST_CODE,
AUDIT_COD,
PACK_COD,
CORP_COD,
SUM(VALUE) AS VALUE,
SUM(VOLUME) AS VOLUME,
SUM(COUNTING_UNIT) AS COUNTING_UNIT,
SUM(VALUE_LY) AS VALUE_LY,
SUM(VOLUME_LY) AS VOLUME_LY,
SUM(COUNTING_UNIT_LY) AS COUNTING_UNIT_LY
FROM
AIA_RAWDATA_FINAL
GROUP BY
YM,
ORG_INS_CD,
INST_CODE,
AUDIT_COD,
PACK_COD,
CORP_COD
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TF_EXT_AIA_SALES (
YYYYMM,
PACK_CODE,
AUDIT_CODE,
DATA_SOURCE,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
PRESCRIPTION,
PRESCRIPTION_LY,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
CMPS_FLAG,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
INST_CODE,
DEPT_NAME,
NEW_CODE,
AREA,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE,
CORP_CODE,
ETL_INSERT_DT,
ETL_UPDATE_DT
)
SELECT
A.ym,
A.PACK_COD as PACK_CODE,
A.AUDIT_COD AS AUDIT_CODE,
'AIA(Monthly)' DATA_SOURCE,
-------------------------------------
sum(volume) as SALES_UNIT_CAL,
sum(volume_ly) as SALES_UNIT_CAL_LY,
sum(value) as SALES_VALUE_CAL,
sum(value_ly) as SALES_VALUE_CAL_LY,
sum(counting_unit) as CONUTING_UNIT,
sum(counting_unit_ly) as CONUTING_UNIT_LY,
null as prescription,
null as prescription_ly,
-------------------------------------
--Retail藏数逻辑标签------------------
1 PACK_FLAG,
1 PROD_FLAG,
0 DTP_FLAG,
null as cmps_flag,
-------------------------------------
--EC数据标签--------------------------
'' PLATFORM_TYPE,
'' STORE_NAME,
'' STORE_TYPE,
-------------------------------------
--COUNTY数据标签----------------------
'' REGION_TYPE,
-------------------------------------
--AIA数据标签-------------------------
inst_code AS inst_code, --可以移除，报告没有使用
-------------------------------------
--XIE HE 数据标签---------------------
'' as dept_name,
'' as new_code,
'' as area,
'' as h_level,
'' as reimburse,
'' as reimburse_type,
'' as prescription_source,
-------------------------------------
'' as CORP_CODE, --可以移除，报告没有使用
-------------------------------------
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
FROM
FINAL_DM_TF_EXT_AIA_SALES A
WHERE
ym >= '202401'
GROUP BY
YM,
PACK_COD,
CORP_COD,
AUDIT_COD,
INST_CODE
-- COMMAND ----------
-- 20260109 dm.dm_ext_aia_sales 是老的表，现在已经用DM.DM_TF_EXT_AIA_SALES 替换，但是为了下游，老表暂时不下线，仍然写入
insert overwrite table dm.dm_ext_aia_sales (
ym,
org_ins_cd,
inst_code,
AUDIT_COD,
pack_cod,
corp_cod,
value,
volume,
counting_unit,
value_ly,
volume_ly,
counting_unit_ly
)
select
ym,
org_ins_cd,
inst_code,
AUDIT_COD,
pack_cod,
corp_cod,
sum(value) as value,
sum(volume) as volume,
sum(counting_unit) as counting_unit,
sum(value_ly) as value_ly,
sum(volume_ly) as volume_ly,
sum(counting_unit_ly) as counting_unit_ly
from
aia_rawdata_FINAL
CROSS JOIN
filter_ym
WHERE
YM BETWEEN filter_ym.min_yyyymm AND filter_ym.max_yyyymm
group by
ym,
org_ins_cd,
inst_code,
AUDIT_COD,
pack_cod,
corp_cod
-- COMMAND ----------
-- MAGIC %md
-- MAGIC 下面是重构验证，chenwu可删
-- COMMAND ----------
-- %sql
-- inst_code
-- ym
-- MARKET
-- COMMAND ----------
-- SELECT ym,AUDIT_COD,inst_code,SUM(value) value,SUM(volume) volume,SUM(counting_unit) counting_unit,SUM(value_ly) value_ly,SUM(volume_ly) volume_ly,SUM(counting_unit_ly) counting_unit_ly
-- FROM dm.dm_ext_aia_sales
-- GROUP BY ym,AUDIT_COD,inst_code
-- ORDER BY ym,AUDIT_COD,inst_code
-- COMMAND ----------
-- SELECT ym,AUDIT_COD,SUM(value) value,SUM(volume) volume,SUM(counting_unit) counting_unit,SUM(value_ly) value_ly,SUM(volume_ly) volume_ly,SUM(counting_unit_ly) counting_unit_ly
-- FROM dm.dm_ext_aia_sales
-- GROUP BY ym,AUDIT_COD
-- ORDER BY ym,AUDIT_COD
-- COMMAND ----------
-- SELECT
-- old.ym,
-- OLD.AUDIT_COD,
-- OLD.inst_code,
-- OLD.value,
-- NEW.value,
-- OLD.value-NEW.value as dif_value,
-- OLD.value_ly,
-- NEW.value_ly,
-- OLD.value_ly-NEW.value_ly as dif_value_ly,
-- OLD.volume,
-- NEW.volume,
-- OLD.volume-NEW.volume as dif_volume,
-- OLD.volume_ly,
-- NEW.volume_ly,
-- OLD.volume_ly-NEW.volume_ly as dif_volume_ly,
-- OLD.counting_unit,
-- NEW.counting_unit,
-- OLD.counting_unit-NEW.counting_unit as dif_counting_unit,
-- OLD.counting_unit_ly,
-- NEW.counting_unit_ly,
-- OLD.counting_unit_ly-NEW.counting_unit_ly as dif_counting_unit_ly
-- FROM
-- (
-- SELECT ym,AUDIT_COD,inst_code,SUM(value) value,SUM(volume) volume,SUM(counting_unit) counting_unit,SUM(value_ly) value_ly,SUM(volume_ly) volume_ly,SUM(counting_unit_ly) counting_unit_ly
-- FROM dm.dm_ext_aia_sales
-- GROUP BY ym,AUDIT_COD,inst_code
-- -- ORDER BY ym
-- ) AS OLD
-- LEFT JOIN
-- (
-- SELECT YYYYMM,AUDIT_CODE,inst_code,SUM(SALES_VALUE_CAL) value,SUM(SALES_UNIT_CAL) volume,SUM(CONUTING_UNIT) counting_unit,SUM(SALES_VALUE_CAL_LY) value_ly,SUM(SALES_UNIT_CAL_LY) volume_ly,SUM(CONUTING_UNIT_LY) counting_unit_ly
-- FROM DM.DM_TF_EXT_AIA_SALES
-- GROUP BY YYYYMM,AUDIT_CODE,inst_code
-- -- ORDER BY YYYYMM
-- ) AS NEW
-- ON OLD.YM =NEW.YYYYMM AND old.AUDIT_COD = NEW.AUDIT_CODE AND old.inst_code = new.inst_code
-- where
-- ABS(OLD.value - NEW.value) > 0.0000000001 OR
-- ABS(OLD.volume - NEW.volume) > 0.0000000001 OR
-- ABS(OLD.value_ly - NEW.value_ly) > 0.0000000001 OR
-- ABS(OLD.volume_ly - NEW.volume_ly) > 0.0000000001 OR
-- ABS(OLD.counting_unit - NEW.counting_unit) > 0.0000000001 OR
-- ABS(OLD.counting_unit_ly - NEW.counting_unit_ly) > 0.0000000001;
-- COMMAND ----------
-- SELECT
-- old.ym,
-- OLD.AUDIT_COD,
-- OLD.value,
-- NEW.value,
-- OLD.value-NEW.value as dif_value,
-- OLD.value_ly,
-- NEW.value_ly,
-- OLD.value_ly-NEW.value_ly as dif_value_ly,
-- OLD.volume,
-- NEW.volume,
-- OLD.volume-NEW.volume as dif_volume,
-- OLD.volume_ly,
-- NEW.volume_ly,
-- OLD.volume_ly-NEW.volume_ly as dif_volume_ly,
-- OLD.counting_unit,
-- NEW.counting_unit,
-- OLD.counting_unit-NEW.counting_unit as dif_counting_unit,
-- OLD.counting_unit_ly,
-- NEW.counting_unit_ly,
-- OLD.counting_unit_ly-NEW.counting_unit_ly as dif_counting_unit_ly
-- FROM
-- (
-- SELECT ym,AUDIT_COD,SUM(value) value,SUM(volume) volume,SUM(counting_unit) counting_unit,SUM(value_ly) value_ly,SUM(volume_ly) volume_ly,SUM(counting_unit_ly) counting_unit_ly
-- FROM dm.dm_ext_aia_sales
-- GROUP BY ym,AUDIT_COD
-- -- ORDER BY ym
-- ) AS OLD
-- LEFT JOIN
-- (
-- SELECT YYYYMM,AUDIT_CODE,SUM(SALES_VALUE_CAL) value,SUM(SALES_UNIT_CAL) volume,SUM(CONUTING_UNIT) counting_unit,SUM(SALES_VALUE_CAL_LY) value_ly,SUM(SALES_UNIT_CAL_LY) volume_ly,SUM(CONUTING_UNIT_LY) counting_unit_ly
-- FROM DM.DM_TF_EXT_AIA_SALES
-- GROUP BY YYYYMM,AUDIT_CODE
-- -- ORDER BY YYYYMM
-- ) AS NEW
-- ON OLD.YM =NEW.YYYYMM AND old.AUDIT_COD = NEW.AUDIT_CODE
-- where
-- ABS(OLD.value - NEW.value) > 0.0000000001 OR
-- ABS(OLD.volume - NEW.volume) > 0.0000000001 OR
-- ABS(OLD.value_ly - NEW.value_ly) > 0.0000000001 OR
-- ABS(OLD.volume_ly - NEW.volume_ly) > 0.0000000001 OR
-- ABS(OLD.counting_unit - NEW.counting_unit) > 0.0000000001 OR
-- ABS(OLD.counting_unit_ly - NEW.counting_unit_ly) > 0.0000000001;
-- COMMAND ----------
-- SELECT
-- old.ym,
-- OLD.value,
-- NEW.value,
-- OLD.value-NEW.value as dif_value,
-- OLD.value_ly,
-- NEW.value_ly,
-- OLD.value_ly-NEW.value_ly as dif_value_ly,
-- OLD.volume,
-- NEW.volume,
-- OLD.volume-NEW.volume as dif_volume,
-- OLD.volume_ly,
-- NEW.volume_ly,
-- OLD.volume_ly-NEW.volume_ly as dif_volume_ly,
-- OLD.counting_unit,
-- NEW.counting_unit,
-- OLD.counting_unit-NEW.counting_unit as dif_counting_unit,
-- OLD.counting_unit_ly,
-- NEW.counting_unit_ly,
-- OLD.counting_unit_ly-NEW.counting_unit_ly as dif_counting_unit_ly
-- FROM
-- (
-- SELECT ym,SUM(value) value,SUM(volume) volume,SUM(counting_unit) counting_unit,SUM(value_ly) value_ly,SUM(volume_ly) volume_ly,SUM(counting_unit_ly) counting_unit_ly
-- FROM dm.dm_ext_aia_sales
-- GROUP BY ym
-- ORDER BY ym
-- ) AS OLD
-- LEFT JOIN
-- (
-- SELECT YYYYMM,SUM(SALES_VALUE_CAL) value,SUM(SALES_UNIT_CAL) volume,SUM(CONUTING_UNIT) counting_unit,SUM(SALES_VALUE_CAL_LY) value_ly,SUM(SALES_UNIT_CAL_LY) volume_ly,SUM(CONUTING_UNIT_LY) counting_unit_ly
-- FROM DM.DM_TF_EXT_AIA_SALES
-- GROUP BY YYYYMM
-- ORDER BY YYYYMM
-- ) AS NEW
-- ON OLD.YM =NEW.YYYYMM
-- where
-- ABS(OLD.value - NEW.value) > 0.0000000001 OR
-- ABS(OLD.volume - NEW.volume) > 0.0000000001 OR
-- ABS(OLD.value_ly - NEW.value_ly) > 0.0000000001 OR
-- ABS(OLD.volume_ly - NEW.volume_ly) > 0.0000000001 OR
-- ABS(OLD.counting_unit - NEW.counting_unit) > 0.0000000001 OR
-- ABS(OLD.counting_unit_ly - NEW.counting_unit_ly) > 0.0000000001;
-- COMMAND ----------
-- -- 这个sql证明，两张表的inst_code，值也相同
-- SELECT
-- *
-- FROM
-- (
-- SELECT inst_code,COUNT(*) as c1 FROM DM.DM_TF_EXT_AIA_SALES
-- GROUP BY inst_code
-- ) AS NEW
-- FULL JOIN
-- (
-- SELECT inst_code,COUNT(*) as c1 FROM DM.dm_ext_aia_sales
-- GROUP BY inst_code
-- ) AS OLD
-- ON NEW.inst_code = old.inst_code
-- WHERE new.c1 != old.c1
-- COMMAND ----------
-- -- 这个sql证明，两张表的inst_code相同
-- SELECT
-- *
-- FROM
-- (
-- SELECT inst_code,COUNT(*) as c1 FROM DM.DM_TF_EXT_AIA_SALES
-- GROUP BY inst_code
-- ) AS NEW
-- FULL JOIN
-- (
-- SELECT inst_code,COUNT(*) as c1 FROM DM.DM_TF_EXT_AIA_SALES
-- GROUP BY inst_code
-- ) AS OLD
-- ON NEW.inst_code = old.inst_code
-- WHERE NEW.inst_code IS NULL OR old.inst_code IS NULL
-- COMMAND ----------
-- -- 这个sql证明，两张表的inst_code，值也相同
-- SELECT
-- *
-- FROM
-- (
-- SELECT inst_code,COUNT(*) as c1 FROM DM.DM_TF_EXT_AIA_SALES
-- GROUP BY inst_code
-- ) AS NEW
-- FULL JOIN
-- (
-- SELECT inst_code,COUNT(*) as c1 FROM DM.DM_TF_EXT_AIA_SALES
-- GROUP BY inst_code
-- ) AS OLD
-- ON NEW.inst_code = old.inst_code
-- WHERE new.c1 != old.c1
-- COMMAND ----------
-- SELECT COUNT(*) FROM DM.DM_TF_EXT_AIA_SALES
-- COMMAND ----------
-- SELECT COUNT(*) FROM DM.dm_ext_aia_sales

195
AIA/02 dm_ext_aia_sales_bakup_20230327.sql Normal file
View File

@@ -0,0 +1,195 @@
-- Databricks notebook source
--字段值的初步处理
create or replace temporary view aia_rawdata_1 as
select
year,
qtr,
cast(year * 100 + ym as int) ym,
org_prov,
org_city,
ins_level,
case when ins_cd is not null and length(ins_cd) < 7 then right(concat('0000000',ins_cd),7) else ins_cd end as org_ins_cd,
atc1,
atc2,
atc3,
atc4,
atc,
mole_nm,
prod_nm,
org_prd_nm_c,
org_pk,
org_prd_str,
org_pth,
min_unit,
value,
unit,
cast(pack_desc as int) as pack_desc,
org_pk_un,
manu
from dwd.dwd_gnd_aia_rawdata
-- COMMAND ----------
--关联出AUDIT_COD、inst_code
create or replace temporary view aia_rawdata_2 as
with hospital_mapping as (
select distinct
case when length(cpa_hospital_code) < 7 then right(concat('0000000',cpa_hospital_code),7) else cpa_hospital_code end as cpa_hospital_code,
ins_cd_nl as inst_code
from dwd.dwd_gnd_hospital_not_provided
)
select upper(nvl(t2.inst_code,t1.org_ins_cd)) AUDIT_COD,
t1.*,
nvl(t2.inst_code,t1.org_ins_cd) as inst_code
from aia_rawdata_1 t1
left join hospital_mapping t2
on t1.org_ins_cd = t2.cpa_hospital_code
-- left join dm.dm_td_institution t3
-- on t2.ins_code = t3.inst_code
-- left join dm.dm_ims_td_geo t4
-- on t3.city_name_en = t4.city
-- COMMAND ----------
/*
修改人 chenwu
修改时间 20250513
修改内容 打包分子的数据，不论 打通表 和 事实表，都没有系数，需要用 事实表里的 最小制剂单位数量 作为counting_unit
*/
--关联出pack_cod、corp_cod、counting_unit
create or replace temporary view aia_rawdata_3 as
select case when length(trim(t2.iqvia_pack_code)) < 12 and trim(t2.iqvia_pack_code) REGEXP '^[0-9]' then right(concat('000000000000',trim(t2.iqvia_pack_code)),12)
else trim(t2.iqvia_pack_code)
end as pack_cod,
t3.corp_cod,
case when left(t2.iqvia_pack_code,4) = 'AZP_' then cast(t1.min_unit/t1.org_pk_un as decimal(38,10)) else t3.counting_unit end counting_unit,
-- t3.counting_unit,20250513
t1.*
from aia_rawdata_2 t1
left join (select distinct trim(iqvia_pack_code) as iqvia_pack_code,
ATC1,ATC2,ATC3,ATC4,org_prd_cd,org_mole_nm_c,
org_manu_prd_nm_c,org_prd_nm_c,org_pk,org_prd_str,
org_pth,min_pk_unit,org_pk_unit,org_manu_nm from dwd.dwd_gnd_ext_aia_cpt_data) t2
--ATC1编码+ATC2编码+ATC3编码+ATC4编码+药品编码(ATC)+药品通用名+药品产品名+药品商品名+规格+剂型+给药途径+最小销售包装单位+包装规格+生产企业
on nvl(t1.ATC1,'') = nvl(t2.ATC1,'')
and nvl(t1.ATC2,'') = nvl(t2.ATC2,'')
and nvl(t1.ATC3,'') = nvl(t2.ATC3,'')
and nvl(t1.ATC4,'') = nvl(t2.ATC4,'')
and nvl(t1.ATC,'') = nvl(t2.org_prd_cd,'')
and nvl(t1.MOLE_NM,'') = nvl(t2.org_mole_nm_c,'')
and nvl(t1.PROD_NM,'') = nvl(t2.org_manu_prd_nm_c,'')
and nvl(t1.ORG_PRD_NM_C,'') = nvl(t2.org_prd_nm_c,'')
and nvl(t1.ORG_PK,'') = nvl(t2.org_pk,'')
and nvl(t1.ORG_PRD_STR,'') = nvl(t2.org_prd_str,'')
and nvl(t1.ORG_PTH,'') = nvl(t2.org_pth,'')
and nvl(t1.UNIT,'') = nvl(t2.min_pk_unit,'')
and nvl(t1.PACK_DESC,'') = nvl(t2.org_pk_unit,'')
and nvl(t1.MANU,'') = nvl(t2.org_manu_nm,'')
left join (select distinct PACK_COD,corp_cod,counting_unit from dm.dm_aia_pack_property) t3
on t2.iqvia_pack_code = t3.PACK_COD
-- COMMAND ----------
--计算出ly销量
create or replace temporary view aia_rawdata_4 as
with max_ym as (
select max(ym) as max_ym from aia_rawdata_3
)
select
YM,
org_ins_cd,
inst_code,
AUDIT_COD,
pack_cod,
corp_cod,
sum(value) as value,
sum(volume) as volume,
sum(counting_unit) as counting_unit,
sum(value_ly) as value_ly,
sum(volume_ly) as volume_ly,
sum(counting_unit_ly) as counting_unit_ly
from (
select
ym,
org_ins_cd,
inst_code,
AUDIT_COD,
pack_cod,
corp_cod,
cast(value as decimal(38,10)) as value,
cast(ORG_PK_UN as decimal(38,10)) as volume,
cast(ORG_PK_UN as decimal(38,10)) * coalesce(cast(counting_unit as decimal(38,10)),1) as counting_unit,
0 as value_ly,
0 as volume_ly,
0 as counting_unit_ly
from aia_rawdata_3
union all
select
cast(cast(ym as int) + 100 as string) as ym,
org_ins_cd,
inst_code,
AUDIT_COD,
pack_cod,
corp_cod,
0 as value,
0 as volume,
0 as counting_unit,
cast(value as decimal(38,10)) as value_ly,
cast(ORG_PK_UN as decimal(38,10)) as volume_ly,
cast(ORG_PK_UN as decimal(38,10)) * coalesce(cast(counting_unit as decimal(38,10)),1) as counting_unit_ly
from aia_rawdata_3 t1
join max_ym t2
on 1=1
where cast(cast(ym as int) + 100 as string) <= t2.max_ym
)
group by ym,
org_ins_cd,
inst_code,
AUDIT_COD,
pack_cod,
corp_cod
-- COMMAND ----------
insert overwrite table dm.dm_ext_aia_sales(
ym,
org_ins_cd,
inst_code,
AUDIT_COD,
pack_cod,
corp_cod,
value,
volume,
counting_unit,
value_ly,
volume_ly,
counting_unit_ly
)
select
ym,
org_ins_cd,
inst_code,
AUDIT_COD,
pack_cod,
corp_cod,
sum(value) as value,
sum(volume) as volume,
sum(counting_unit) as counting_unit,
sum(value_ly) as value_ly,
sum(volume_ly) as volume_ly,
sum(counting_unit_ly) as counting_unit_ly
from aia_rawdata_4
group by
ym,
org_ins_cd,
inst_code,
AUDIT_COD,
pack_cod,
corp_cod

84
AIA/03 dm_aia_flag.sql Normal file
View File

@@ -0,0 +1,84 @@
-- Databricks notebook source
-----------------------------------------------------------------------------------
--修改时间：20241021
--修改人：FanXujia
--修改内容：
--只取最新年-1 ~ 最新年范围内的，称为“2年共有”。增加
-----------------------------------------------------------------------------------
--共有机构
insert overwrite table dm.dm_aia_hp_flag
with max_year as (
select
max(cast(year as int)) as max_year,
max(cast(year as int) - 1) as max_year_ly
from dwd.dwd_gnd_hospital_not_provided
)
,hp_flag as (
select
distinct concat(t1.year, '-Y') as hp_his_flag,
case when length(t1.cpa_hospital_code) < 7 then right(concat('0000000',t1.cpa_hospital_code),7) else t1.cpa_hospital_code end as cpa_hospital_code
from
dwd.dwd_gnd_hospital_not_provided t1
cross join max_year t2
where cast(t1.year as int) >= t2.max_year_ly
)
,hp_flag2 as (
SELECT
cpa_hospital_code,
CONCAT_WS(' ∩ ', SORT_ARRAY(COLLECT_LIST(hp_his_flag))) AS FLAG
FROM
hp_flag
GROUP BY
cpa_hospital_code
)
,inst_mapping as (
-- select
-- distinct right(concat('0000000', org_cd), 7) org_cd,
-- ins_cd
-- from
-- dwd.dwd_gnd_hospitalmapping
select distinct
case when length(cpa_hospital_code) < 7 then right(concat('0000000',cpa_hospital_code),7) else cpa_hospital_code end as cpa_hospital_code,
ins_cd_nl as inst_code
from dwd.dwd_gnd_hospital_not_provided
)
,ALL_INS as (
select distinct inst_code
from dm.dm_ext_aia_sales
)
,flag as (
select distinct
coalesce(t2.inst_code,t1.cpa_hospital_code) as inst_code,
t1.flag as aia_hp_flag,
'Y' as FLAG,
'AIA(Monthly)' as DATA_SOURCE
from hp_flag2 t1
left join inst_mapping t2 on t1.cpa_hospital_code = t2.cpa_hospital_code
where
length(t1.flag) > 6
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'IQVIA-CHPA(Monthly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'XH Data(Quarterly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'EC(Monthly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'Retail(Quarterly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'CHC(Quarterly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'THC(Quarterly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'IQVIA-COUNTY(Quarterly)' as DATA_SOURCE
)
select t1.inst_code,
NVL(t2.aia_hp_flag,'') as aia_hp_flag,
NVL(t2.FLAG,'N') as FLAG,
'AIA(Monthly)' as DATA_SOURCE
from ALL_INS t1
left join flag t2
on t1.inst_code = t2.inst_code

112
AIA/04 dm_aia_provided_flag.sql Normal file
View File

@@ -0,0 +1,112 @@
-- Databricks notebook source
-- MAGIC %run ../../../Common/config
-- COMMAND ----------
-- MAGIC %python
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dim_product_wide`').createOrReplaceTempView('cdw_dwd_dim_product_wide')
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dwd_hospital_classification`').createOrReplaceTempView('cdw_dwd_dwd_hospital_classification')
-- COMMAND ----------
--连续取数机构
----------------------------------------------------------------------------------------------------------------------
--修改时间：20241029
--修改人：FanXujia
--修改内容：
--根据事实表最新月，往前推12个月（含最新月），按机构 + 年月粒度汇总，如果每个月销量都不等于0，这家机构称为：滚动一年有数
--根据事实表最新月，往前推24个月（含最新月），按机构 + 年月粒度汇总，如果每个月销量都不等于0，这家机构称为：滚动两年有数
--2022年没有销量为正常情况，判断时仅考虑2023年01月起销量是否等于0
----------------------------------------------------------------------------------------------------------------------
insert overwrite table dm.dm_aia_provided_flag
with max_ym as (
select max(ym) as max_ym
from dm.dm_ext_aia_sales
)
,ym_range as (
select max_ym,
case when max_ym <= '202412' then cast(right(max_ym,2) as int)
else cast(12 as int)
end ym_12m,
case when max_ym <= '202312' then '202401'
when right(max_ym,2) = '12' then concat(left(max_ym,4),'01')
else concat(cast(left(max_ym, 4) - 1 as int),right(concat('0', cast(right(max_ym, 2) + 1 as int)), 2))
end as ym_12m_before,
case when max_ym <= '202312' then cast(right(max_ym,2) as int)
when max_ym <= '202512' then cast(right(max_ym,2) as int) + 12
else cast(24 as int)
end as ym_24m,
case when max_ym <= '202512' then '202401'
when right(max_ym,2) = '12' then concat(cast(left(max_ym,4) - 1 as int),'01')
else concat(cast(left(max_ym, 4) -2 as int),right(concat('0', cast(right(max_ym, 2) + 1 as int)), 2))
end as ym_24m_before
from max_ym
)
,sales_1 as (
select
inst_code,
ym
from
dm.dm_ext_aia_sales
group by
ym,
inst_code
having
sum(value) <> 0
)
,sales_2 as (
select
t1.inst_code,
t1.ym,
t2.max_ym,
t2.ym_12m,
t2.ym_12m_before,
t2.ym_24m,
t2.ym_24m_before
from sales_1 t1
cross join ym_range t2
)
,flag_12m as (
select inst_code,
'滚动一年有数' as aia_provided_flag,
'AIA(Monthly)' as DATA_SOURCE
from sales_2
where ym <= max_ym
and ym >= ym_12m_before
group by inst_code
having count(1) = (select ym_12m from ym_range)
)
,flag_24m as (
select inst_code,
'滚动两年有数' as aia_provided_flag,
'AIA(Monthly)' as DATA_SOURCE
from sales_2
where ym <= max_ym
and ym >= ym_24m_before
group by inst_code
having count(1) = (select ym_24m from ym_range)
)
select inst_code,
aia_provided_flag,
DATA_SOURCE
from flag_12m
union all
select inst_code,
aia_provided_flag,
DATA_SOURCE
from flag_24m
union all
select '' as inst_code,'' as aia_provided_flag,'IQVIA-CHPA(Monthly)' as data_source
union all
select '' as inst_code,'' as aia_provided_flag,'XH Data(Quarterly)' as data_source
union all
select '' as inst_code,'' as aia_provided_flag,'EC(Monthly)' as data_source
union all
select '' as inst_code,'' as aia_provided_flag,'Retail(Quarterly)' as data_source
union all
select '' as inst_code,'' as aia_provided_flag,'CHC(Quarterly)' as data_source
union all
select '' as inst_code,'' as aia_provided_flag,'THC(Quarterly)' as data_source
union all
select '' as inst_code,'' as aia_provided_flag,'IQVIA-COUNTY(Quarterly)' as data_source

146
AIA/06 DM_TD_EXT_AIA_PACK_PROPERTY.sql Normal file
View File

@@ -0,0 +1,146 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_AIA_PACK_PROPERTY
-- (
-- -- MARKET_PACK_KEY STRING,
-- PACK_CODE STRING,
-- PACK_DESC STRING,
-- STGH_DESC STRING,
-- PACK_LCH STRING,
-- FAMILY_CODE STRING,
-- FAMILY_NAME STRING,
-- PROD_CODE STRING,
-- PROD_DESC STRING,
-- PROD_DESC_C STRING,
-- CMPS_CODE STRING,
-- CMPS_DESC STRING,
-- CMPS_DESC_C STRING,
-- ATC1_CODE STRING,
-- ATC2_CODE STRING,
-- ATC3_CODE STRING,
-- ATC4_CODE STRING,
-- APP1_CODE STRING,
-- APP2_CODE STRING,
-- APP3_CODE STRING,
-- BIO_DESC STRING,
-- GENE_ORIG_DESC STRING,
-- ETH_OTC_DESC STRING,
-- NRDL_DESC STRING,
-- NRDL_ENTRY_DATE STRING,
-- EDL_DESC STRING,
-- TCM_DESC STRING,
-- PAED_DESC STRING,
-- GQCE_DESC STRING,
-- VBP_DESC_V STRING,
-- VBP_DESC STRING,
-- MANU_CODE STRING,
-- MANU_DESC STRING,
-- MANU_DESC_C STRING,
-- MNFL_CODE STRING,
-- MNFL_DESC STRING,
-- CORP_CODE STRING,
-- CORP_DESC STRING,
-- CORP_DESC_C STRING,
-- BRANDTYPE STRING,
-- -- MARKET STRING,
-- -- KEY_COMPETITOR STRING,
-- IS_AZ STRING,
-- AZ_MAIN STRING,
-- -- AZ_RELATED STRING,
-- ATC1_DESC STRING,
-- ATC1_DESC_C STRING,
-- ATC2_DESC STRING,
-- ATC2_DESC_C STRING,
-- ATC3_DESC STRING,
-- ATC3_DESC_C STRING,
-- ATC4_DESC STRING,
-- ATC4_DESC_C STRING,
-- APP1_DESC STRING,
-- APP1_DESC_C STRING,
-- APP2_DESC STRING,
-- APP2_DESC_C STRING,
-- APP3_DESC STRING,
-- APP3_DESC_C STRING,
-- -- CLASS STRING,
-- -- MARKET_RATIO STRING,
-- COUNTINGUNIT STRING,
-- VBP_BRAND STRING,
-- REPLENISH_FALG STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP
-- )
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_aia_pack_property';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_aia_pack_property';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_AIA_PACK_PROPERTY
SELECT
-- MARKET_PACK_KEY,
PACK_COD AS PACK_CODE,
PACK_DES AS PACK_DESC,
STGH_DES AS STGH_DESC,
PACK_LCH,
FAMILY_CODE,
FAMILY_NAME,
PROD_COD AS PROD_CODE,
PROD_DES AS PROD_DESC,
PROD_DES_C AS PROD_DESC_C,
CMPS_COD AS CMPS_CODE,
CMPS_DES AS CMPS_DESC,
CMPS_DES_C AS CMPS_DESC_C,
ATC1_COD AS ATC1_CODE,
ATC2_COD AS ATC2_CODE,
ATC3_COD AS ATC3_CODE,
ATC4_COD AS ATC4_CODE,
APP1_COD AS APP1_CODE,
APP2_COD AS APP2_CODE,
APP3_COD AS APP3_CODE,
BIO_DESC,
GENE_ORIG_DESC,
ETH_OTC_DESC,
NRDL_DESC,
NRDL_ENTRY_DATE,
EDL_DESC,
TCM_DESC,
PAED_DESC,
GQCE_DESC,
null as VBP_DESC_V,
VBP_DESC,
MANU_COD AS MANU_CODE,
MANU_DES AS MANU_DESC,
MANU_DES_C AS MANU_DESC_C,
MNFL_COD AS MNFL_CODE,
MNFL_DES AS MNFL_DESC,
CORP_COD AS CORP_CODE,
CORP_DES AS CORP_DESC,
CORP_DES_C AS CORP_DESC_C,
BRANDTYPE,
-- MARKET,
-- KEY_COMPETITOR,
IS_AZ,
AZ_MAIN,
-- AZ_RELATED,
ATC1_DES AS ATC1_DESC,
ATC1_DES_C AS ATC1_DESC_C,
ATC2_DES AS ATC2_DESC,
ATC2_DES_C AS ATC2_DESC_C,
ATC3_DES AS ATC3_DESC,
ATC3_DES_C AS ATC3_DESC_C,
ATC4_DES AS ATC4_DESC,
ATC4_DES_C AS ATC4_DESC_C,
APP1_DES AS APP1_DESC,
APP1_DES_C AS APP1_DESC_C,
APP2_DES AS APP2_DESC,
APP2_DES_C AS APP2_DESC_C,
APP3_DES AS APP3_DESC,
APP3_DES_C AS APP3_DESC_C,
-- CLASS,
-- NULL AS MARKET_RATIO,
NULL AS COUNTINGUNIT,
NULL AS VBP_BRAND,
NULL AS REPLENISH_FALG,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_AIA_PACK_PROPERTY;

334
AIA/07 DM_TD_EXT_AIA_PACK2MARKET.sql Normal file
View File

@@ -0,0 +1,334 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_AIA_PACK2MARKET (
-- MARKET STRING,
-- PACK_CODE STRING,
-- PACK_DESC STRING,
-- STGH_DESC STRING,
-- PACK_LCH STRING,
-- PROD_CODE STRING,
-- CMPS_CODE STRING,
-- CMPS_DESC STRING,
-- ATC1_CODE STRING,
-- ATC2_CODE STRING,
-- ATC3_CODE STRING,
-- ATC4_CODE STRING,
-- APP1_CODE STRING,
-- APP2_CODE STRING,
-- APP3_CODE STRING,
-- BIO_DESC STRING,
-- GENE_ORIG_DESC STRING,
-- ETH_OTC_DESC STRING,
-- NRDL_DESC STRING,
-- NRDL_ENTRY_DATE STRING,
-- EDL_DESC STRING,
-- TCM_DESC STRING,
-- PAED_DESC STRING,
-- GQCE_DESC STRING,
-- VBP_DESC STRING,
-- MANU_CODE STRING,
-- MANU_DESC STRING,
-- MNFL_CODE STRING,
-- MNFL_DESC STRING,
-- CORP_CODE STRING,
-- CORP_DESC STRING,
-- BRANDTYPE STRING,
-- BU STRING,
-- STARTTIME STRING,
-- ENDTIME STRING,
-- MARKET_RATIO STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_aia_pack2market';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_aia_pack2market';
-- COMMAND ----------
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_AIA_PACK2MARKET_KEYCOMPETITOR (
-- KEY_COMPETITOR STRING,
-- MARKET STRING,
-- PACK_CODE STRING)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_aia_pack2market_keycompetitor';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_aia_pack2market_keycompetitor';
-- COMMAND ----------
-- CREATE or REPLACE TABLE DM.DM_TD_EXT_AIA_PACK2MARKET_CLASS (
-- MARKET STRING,
-- PACK_CODE STRING,
-- CLASS STRING)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_aia_pack2market_class';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_aia_pack2market_class';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_AIA_PACK2MARKET
-- 包含规则
SELECT DISTINCT
T2.MARKET,
T1.PACK_CODE,
T1.PACK_DESC,
T1.STGH_DESC,
T1.PACK_LCH,
T1.PROD_CODE,
T1.CMPS_CODE,
T1.CMPS_DESC,
T1.ATC1_CODE,
T1.ATC2_CODE,
T1.ATC3_CODE,
T1.ATC4_CODE,
T1.APP1_CODE,
T1.APP2_CODE,
T1.APP3_CODE,
T1.BIO_DESC,
T1.GENE_ORIG_DESC,
T1.ETH_OTC_DESC,
T1.NRDL_DESC,
T1.NRDL_ENTRY_DATE,
T1.EDL_DESC,
T1.TCM_DESC,
T1.PAED_DESC,
T1.GQCE_DESC,
T1.VBP_DESC,
T1.MANU_CODE,
T1.MANU_DESC,
T1.MNFL_CODE,
T1.MNFL_DESC,
T1.CORP_CODE,
T1.CORP_DESC,
T1.BRANDTYPE,
T2.BU,
T2.STARTTIME,
T2.ENDTIME,
COALESCE(T2.EXTEND_MARKET_RATIO, '1') AS MARKET_RATIO,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_AIA_PACK_PROPERTY AS T1
LEFT JOIN (
SELECT
MARKET,
BU,
STARTTIME,
ENDTIME,
EXTEND_MARKET_RATIO,
ATC1_CODE,
ATC2_CODE,
ATC3_CODE,
ATC4_CODE,
NFC1_CODE,
NFC2_CODE,
NFC3_CODE,
CORPORATION_CODE,
MANUFACTURER_CODE,
PRODUCT_CODE,
PACK_CODE,
STRENGTH,
MOLECULE_CODE
FROM DWD.DWD_GND_IMS_TBLMARKET_BYMONTH
WHERE EXTEND_MARKET IS NULL AND (NOT_IN_FLAG IS NULL OR NOT_IN_FLAG = '1')
) AS T2
ON (T2.ATC1_CODE IS NULL OR T1.ATC1_CODE = T2.ATC1_CODE)
AND (T2.ATC2_CODE IS NULL OR T1.ATC2_CODE = T2.ATC2_CODE)
AND (T2.ATC3_CODE IS NULL OR T1.ATC3_CODE = T2.ATC3_CODE)
AND (T2.ATC4_CODE IS NULL OR T1.ATC4_CODE = T2.ATC4_CODE)
AND (T2.NFC1_CODE IS NULL OR T1.APP1_CODE = T2.NFC1_CODE)
AND (T2.NFC2_CODE IS NULL OR T1.APP2_CODE = T2.NFC2_CODE)
AND (T2.NFC3_CODE IS NULL OR T1.APP3_CODE = T2.NFC3_CODE)
AND (T2.CORPORATION_CODE IS NULL OR T1.CORP_CODE = T2.CORPORATION_CODE)
AND (T2.MANUFACTURER_CODE IS NULL OR T1.MANU_CODE = T2.MANUFACTURER_CODE)
AND (T2.PRODUCT_CODE IS NULL OR T1.PROD_CODE = T2.PRODUCT_CODE)
AND (T2.PACK_CODE IS NULL OR T1.PACK_CODE = T2.PACK_CODE)
AND (T2.STRENGTH IS NULL OR T1.STGH_DESC = T2.STRENGTH)
AND (T2.MOLECULE_CODE IS NULL OR T1.CMPS_CODE = T2.MOLECULE_CODE)
WHERE T2.MARKET IS NOT NULL
EXCEPT
-- 排除规则
SELECT DISTINCT
T2.MARKET,
T1.PACK_CODE,
T1.PACK_DESC,
T1.STGH_DESC,
T1.PACK_LCH,
T1.PROD_CODE,
T1.CMPS_CODE,
T1.CMPS_DESC,
T1.ATC1_CODE,
T1.ATC2_CODE,
T1.ATC3_CODE,
T1.ATC4_CODE,
T1.APP1_CODE,
T1.APP2_CODE,
T1.APP3_CODE,
T1.BIO_DESC,
T1.GENE_ORIG_DESC,
T1.ETH_OTC_DESC,
T1.NRDL_DESC,
T1.NRDL_ENTRY_DATE,
T1.EDL_DESC,
T1.TCM_DESC,
T1.PAED_DESC,
T1.GQCE_DESC,
T1.VBP_DESC,
T1.MANU_CODE,
T1.MANU_DESC,
T1.MNFL_CODE,
T1.MNFL_DESC,
T1.CORP_CODE,
T1.CORP_DESC,
T1.BRANDTYPE,
T2.BU,
T2.STARTTIME,
T2.ENDTIME,
COALESCE(T2.EXTEND_MARKET_RATIO, '1') AS MARKET_RATIO,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_AIA_PACK_PROPERTY AS T1
LEFT JOIN (
SELECT
MARKET,
BU,
STARTTIME,
ENDTIME,
EXTEND_MARKET_RATIO,
ATC1_CODE,
ATC2_CODE,
ATC3_CODE,
ATC4_CODE,
NFC1_CODE,
NFC2_CODE,
NFC3_CODE,
CORPORATION_CODE,
MANUFACTURER_CODE,
PRODUCT_CODE,
PACK_CODE,
STRENGTH,
MOLECULE_CODE
FROM DWD.DWD_GND_IMS_TBLMARKET_BYMONTH
WHERE EXTEND_MARKET IS NULL AND NOT_IN_FLAG = '0'
) AS T2
ON (T2.ATC1_CODE IS NULL OR T1.ATC1_CODE = T2.ATC1_CODE)
AND (T2.ATC2_CODE IS NULL OR T1.ATC2_CODE = T2.ATC2_CODE)
AND (T2.ATC3_CODE IS NULL OR T1.ATC3_CODE = T2.ATC3_CODE)
AND (T2.ATC4_CODE IS NULL OR T1.ATC4_CODE = T2.ATC4_CODE)
AND (T2.NFC1_CODE IS NULL OR T1.APP1_CODE = T2.NFC1_CODE)
AND (T2.NFC2_CODE IS NULL OR T1.APP2_CODE = T2.NFC2_CODE)
AND (T2.NFC3_CODE IS NULL OR T1.APP3_CODE = T2.NFC3_CODE)
AND (T2.CORPORATION_CODE IS NULL OR T1.CORP_CODE = T2.CORPORATION_CODE)
AND (T2.MANUFACTURER_CODE IS NULL OR T1.MANU_CODE = T2.MANUFACTURER_CODE)
AND (T2.PRODUCT_CODE IS NULL OR T1.PROD_CODE = T2.PRODUCT_CODE)
AND (T2.PACK_CODE IS NULL OR T1.PACK_CODE = T2.PACK_CODE)
AND (T2.STRENGTH IS NULL OR T1.STGH_DESC = T2.STRENGTH)
AND (T2.MOLECULE_CODE IS NULL OR T1.CMPS_CODE = T2.MOLECULE_CODE)
WHERE T2.MARKET IS NOT NULL;
-- COMMAND ----------
--删除需要隐藏的市场
MERGE INTO DM.DM_TD_EXT_AIA_PACK2MARKET AS target
USING dwd.dwd_gnd_ims_hidden_market AS source
ON UPPER(target.market) = UPPER(source.hidden_market)
AND source.data_source = 'AIA'
WHEN MATCHED THEN DELETE;
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_AIA_PACK2MARKET_KEYCOMPETITOR
WITH kc AS (
SELECT
keycompetitor, no,
ROW_NUMBER() OVER (ORDER BY
CASE
WHEN ATC1_Code IS NOT NULL THEN 1
WHEN ATC2_Code IS NOT NULL THEN 2
WHEN ATC3_Code IS NOT NULL THEN 3
WHEN ATC4_Code IS NOT NULL THEN 4
WHEN Molecule_Code IS NOT NULL THEN 5
WHEN Product_Code IS NOT NULL THEN 6
WHEN Pack_Code IS NOT NULL THEN 7
ELSE 999
END,
CASE WHEN NFC1_CODE IS NULL THEN 0 ELSE 1 END,
CASE WHEN NFC2_CODE IS NULL THEN 0 ELSE 1 END,
CASE WHEN NFC3_CODE IS NULL THEN 0 ELSE 1 END
) AS no1,
ATC1_Code, ATC2_Code, ATC3_Code, ATC4_Code,
NFC1_Code, NFC2_Code, NFC3_Code,
corporation_code, Manufacturer_Code,
Product_Code, Pack_Code, Strength, Molecule_Code, market
FROM dwd.dwd_gnd_ims_tblkeycompetitor
),
joined AS (
SELECT
/*+ BROADCAST(kc) */
COALESCE(kc.keycompetitor, 'OTHERS') AS Key_Competitor, -- ✅ UPDATE 逻辑内联
t1.market, t1.PACK_CODE
/*bug 调试区
,
kc.no,
kc.no1,
t1.PROD_COD, t1.CMPS_COD, t1.CMPS_DES,
t1.ATC1_COD, t1.ATC2_COD, t1.ATC3_COD, t1.ATC4_COD,
t1.APP1_COD, t1.APP2_COD, t1.APP3_COD,
t1.BIO_DESC, t1.GENE_ORIG_DESC, t1.ETH_OTC_DESC,
t1.NRDL_DESC, t1.NRDL_Entry_Date, t1.EDL_DESC, t1.TCM_DESC,
t1.PAED_DESC, t1.GQCE_DESC, t1.VBP_DESC,
t1.MANU_COD, t1.MANU_DES, t1.MNFL_COD, t1.MNFL_DES,
t1.CORP_COD, t1.CORP_DES, t1.BrandType, t1.bu, t1.Market_Ratio
*/
,ROW_NUMBER() OVER (
PARTITION BY t1.market, t1.PACK_CODE, t1.PROD_CODE
ORDER BY kc.no1 DESC, kc.no DESC
) AS _rn
FROM DM.DM_TD_EXT_AIA_PACK2MARKET t1
LEFT JOIN kc
ON (kc.ATC1_Code IS NULL OR t1.ATC1_CODE = kc.ATC1_Code)
AND (kc.ATC2_Code IS NULL OR t1.ATC2_CODE = kc.ATC2_Code)
AND (kc.ATC3_Code IS NULL OR t1.ATC3_CODE = kc.ATC3_Code)
AND (kc.ATC4_Code IS NULL OR t1.ATC4_CODE = kc.ATC4_Code)
AND (kc.NFC1_Code IS NULL OR t1.APP1_CODE = kc.NFC1_Code)
AND (kc.NFC2_Code IS NULL OR t1.APP2_CODE = kc.NFC2_Code)
AND (kc.NFC3_Code IS NULL OR t1.APP3_CODE = kc.NFC3_Code)
AND (kc.corporation_code IS NULL OR t1.CORP_CODE = kc.corporation_code)
AND (kc.Manufacturer_Code IS NULL OR t1.MANU_CODE = kc.Manufacturer_Code)
AND (kc.Product_Code IS NULL OR t1.PROD_CODE = kc.Product_Code)
AND (kc.Pack_Code IS NULL OR t1.PACK_CODE = kc.Pack_Code)
AND (kc.Strength IS NULL OR t1.STGH_DESC = kc.Strength)
AND (kc.Molecule_Code IS NULL OR t1.CMPS_CODE = kc.Molecule_Code)
AND (kc.market IS NULL OR t1.market = kc.market)
)
SELECT distinct Key_Competitor,market,pack_code
FROM joined
WHERE _rn = 1;
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_AIA_PACK2MARKET_CLASS
SELECT
/*+ BROADCAST(T2) */
DISTINCT
T1.MARKET,
T1.PACK_CODE,
COALESCE( T2.CLASS, 'OTHERS') AS CLASS
FROM DM.DM_TD_EXT_AIA_PACK2MARKET T1
LEFT JOIN DWD.DWD_GND_TBLCLASS T2
ON (T2.MARKET IS NULL OR T1.MARKET = T2.MARKET)
AND (T2.PACK_CODE IS NULL OR T1.PACK_CODE = T2.PACK_CODE)
AND (T2.PRODUCT_CODE IS NULL OR T1.PROD_CODE = T2.PRODUCT_CODE)
AND (T2.MOLECULE_CODE IS NULL OR T1.CMPS_CODE = T2.MOLECULE_CODE)
AND (T2.CORPORATION_CODE IS NULL OR T1.CORP_CODE = T2.CORPORATION_CODE)
AND (T2.MANUFACTURER_CODE IS NULL OR T1.MANU_CODE = T2.MANUFACTURER_CODE)
AND (T2.ATC1_CODE IS NULL OR T1.ATC1_CODE = T2.ATC1_CODE)
AND (T2.ATC2_CODE IS NULL OR T1.ATC2_CODE = T2.ATC2_CODE)
AND (T2.ATC3_CODE IS NULL OR T1.ATC3_CODE = T2.ATC3_CODE)
AND (T2.ATC4_CODE IS NULL OR T1.ATC4_CODE = T2.ATC4_CODE)
AND (T2.NFC1_CODE IS NULL OR T1.APP1_CODE = T2.NFC1_CODE)
AND (T2.NFC2_CODE IS NULL OR T1.APP2_CODE = T2.NFC2_CODE)
AND (T2.NFC3_CODE IS NULL OR T1.APP3_CODE = T2.NFC3_CODE)
AND (T2.STRENGTH IS NULL OR T1.STGH_DESC = T2.STRENGTH)

30
AIA/08 DM_TD_EXT_AIA_MARKET_RATIO.sql Normal file
View File

@@ -0,0 +1,30 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_AIA_MARKET_RATIO (
-- MARKET STRING,
-- PACK_CODE STRING,
-- CMPS_CODE STRING,
-- STARTTIME STRING,
-- ENDTIME STRING,
-- MARKET_RATIO STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_aia_market_ratio';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_aia_market_ratio';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_AIA_MARKET_RATIO
SELECT
DISTINCT
MARKET,
PACK_CODE,
CMPS_CODE,
STARTTIME,
ENDTIME,
MARKET_RATIO,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_AIA_PACK2MARKET
WHERE UPPER(MARKET) NOT LIKE '%ALL%'

163
AIA/09 DM_TD_EXT_AIA_MARKET_BRAND_RATIO.sql Normal file
View File

@@ -0,0 +1,163 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_AIA_MARKET_BRAND_RATIO (
-- MARKET STRING,
-- PACK_CODE STRING,
-- VALUE_BRAND_RATIO FLOAT,
-- VALUE_BRAND_RATIO_START STRING,
-- VALUE_BRAND_RATIO_END STRING,
-- UNIT_BRAND_RATIO FLOAT,
-- UNIT_BRAND_START STRING,
-- UNIT_BRAND_END STRING,
-- COUNTINGUNIT_BRAND_RATIO FLOAT,
-- COUNTINGUNIT_BRAND_START STRING,
-- COUNTINGUNIT_BRAND_END STRING,
-- PDOT_BRAND_RATIO FLOAT,
-- PDOT_BRAND_START STRING,
-- PDOT_BRAND_END STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_aia_market_brand_ratio';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_aia_market_brand_ratio';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_AIA_MARKET_BRAND_RATIO
SELECT
DISTINCT
T1.MARKET,
T1.PACK_CODE,
CASE
WHEN T2.RATIO IS NOT NULL THEN T2.RATIO
ELSE NVL(T5.RATIO, 1) END AS VALUE_BRAND_RATIO,
CASE
WHEN T2.RATIO IS NOT NULL THEN T2.STARTTIME
ELSE NVL(T5.STARTTIME, 200001) END AS VALUE_BRAND_RATIO_START,
CASE
WHEN T2.RATIO IS NOT NULL THEN T2.ENDTIME
ELSE NVL(T5.ENDTIME, 299912) END AS VALUE_BRAND_RATIO_END,
CASE
WHEN T3.RATIO IS NOT NULL THEN T3.RATIO
ELSE NVL(T6.RATIO, 1) END AS UNIT_BRAND_RATIO,
CASE
WHEN T3.RATIO IS NOT NULL THEN T3.STARTTIME
ELSE NVL(T6.STARTTIME, 200001) END AS UNIT_BRAND_START,
CASE
WHEN T3.RATIO IS NOT NULL THEN T3.ENDTIME
ELSE NVL(T6.ENDTIME, 299912) END AS UNIT_BRAND_END,
1 AS COUNTINGUNIT_BRAND_RATIO,
200001 AS COUNTINGUNIT_BRAND_START,
299912 AS COUNTINGUNIT_BRAND_END,
CASE
WHEN T4.RATIO IS NOT NULL THEN T4.RATIO
ELSE NVL(T7.RATIO, 1) END AS PDOT_BRAND_RATIO,
CASE
WHEN T4.RATIO IS NOT NULL THEN T4.STARTTIME
ELSE NVL(T7.STARTTIME, 200001) END AS PDOT_BRAND_START,
CASE
WHEN T4.RATIO IS NOT NULL THEN T4.ENDTIME
ELSE NVL(T7.ENDTIME, 299912) END AS PDOT_BRAND_END,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_AIA_MARKET_RATIO T1
LEFT JOIN ( --VALUE--PACK
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'VALUE' AND UPPER(LEVEL) = 'PACK'
) T2 ON T1.MARKET = T2.MARKET AND T1.PACK_CODE = T2.PACK_COD
LEFT JOIN ( --UNIT--PACK
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'UNIT' AND UPPER(LEVEL) = 'PACK'
) T3 ON T1.MARKET = T3.MARKET AND T1.PACK_CODE = T3.PACK_COD
LEFT JOIN ( --COUNTINGUNIT--PACK
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'COUNTINGUNIT' AND UPPER(LEVEL) = 'PACK'
) T4 ON T1.MARKET = T4.MARKET AND T1.PACK_CODE = T4.PACK_COD
LEFT JOIN ( --VALUE--MOLECULE
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'VALUE' AND UPPER(LEVEL) = 'MOLECULE'
) T5 ON T1.MARKET = T5.MARKET AND T1.CMPS_CODE = T5.CMPS_COD
LEFT JOIN ( --UNIT--MOLECULE
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'UNIT' AND UPPER(LEVEL) = 'MOLECULE'
) T6 ON T1.MARKET = T6.MARKET AND T1.CMPS_CODE = T6.CMPS_COD
LEFT JOIN ( --COUNTINGUNIT--MOLECULE
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'COUNTINGUNIT' AND UPPER(LEVEL) = 'MOLECULE'
) T7 ON T1.MARKET = T7.MARKET
AND T1.CMPS_CODE = T7.CMPS_COD
-- COMMAND ----------

71
AIA/10 DM_TD_EXT_AIA_MARKET_PACK_MAPPING.sql Normal file
View File

@@ -0,0 +1,71 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_AIA_MARKET_PACK_MAPPING
-- (
-- PACK_CODE STRING,
-- MARKET STRING,
-- CLASS STRING,
-- KEY_COMPETITOR STRING,
-- MARKET_RATIO STRING,
-- STARTTIME STRING,
-- ENDTIME STRING,
-- VALUE_BRAND_RATIO STRING,
-- VALUE_BRAND_RATIO_START STRING,
-- VALUE_BRAND_RATIO_END STRING,
-- UNIT_BRAND_RATIO STRING,
-- UNIT_BRAND_START STRING,
-- UNIT_BRAND_END STRING,
-- COUNTINGUNIT_BRAND_RATIO STRING,
-- COUNTINGUNIT_BRAND_START STRING,
-- COUNTINGUNIT_BRAND_END STRING,
-- PDOT_BRAND_RATIO STRING,
-- PDOT_BRAND_START STRING,
-- PDOT_BRAND_END STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP
-- )
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_aia_market_pack_mapping';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_aia_market_pack_mapping';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_AIA_MARKET_PACK_MAPPING
SELECT DISTINCT
T1.PACK_CODE,
T1.MARKET,
T4.CLASS,
T5.KEY_COMPETITOR,
T2.MARKET_RATIO,
T2.STARTTIME,
T2.ENDTIME,
T3.VALUE_BRAND_RATIO,
T3.VALUE_BRAND_RATIO_START,
T3.VALUE_BRAND_RATIO_END,
T3.UNIT_BRAND_RATIO,
T3.UNIT_BRAND_START,
T3.UNIT_BRAND_END,
T3.COUNTINGUNIT_BRAND_RATIO,
T3.COUNTINGUNIT_BRAND_START,
T3.COUNTINGUNIT_BRAND_END,
T3.PDOT_BRAND_RATIO,
T3.PDOT_BRAND_START,
T3.PDOT_BRAND_END,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
FROM
DM.DM_TD_EXT_AIA_PACK2MARKET T1
LEFT JOIN DM.DM_TD_EXT_AIA_MARKET_RATIO T2
ON T1.MARKET = T2.MARKET
AND T1.PACK_CODE = T2.PACK_CODE
LEFT JOIN DM.DM_TD_EXT_AIA_MARKET_BRAND_RATIO T3
ON T1.MARKET = T3.MARKET
AND T1.PACK_CODE = T3.PACK_CODE
LEFT JOIN DM.DM_TD_EXT_AIA_PACK2MARKET_CLASS T4
ON T1.PACK_CODE = T4.PACK_CODE
AND T1.MARKET = T4.MARKET
LEFT JOIN DM.DM_TD_EXT_AIA_PACK2MARKET_KEYCOMPETITOR T5
ON T1.PACK_CODE = T5.PACK_CODE
AND T1.MARKET = T5.MARKET
WHERE
UPPER(T1.MARKET) NOT LIKE "%ALL%"

11
AIA/11 DM_TD_EXT_AIA_TARGET_INST.sql Normal file
View File

@@ -0,0 +1,11 @@
-- Databricks notebook source
-- MAGIC %run ../../../Common/config
-- COMMAND ----------
-- MAGIC %python
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dim_product_wide`').createOrReplaceTempView('cdw_dwd_dim_product_wide')
-- COMMAND ----------

366
AIA/11 dm_aia_targethp_flag.sql Normal file
View File

@@ -0,0 +1,366 @@
-- Databricks notebook source
-- MAGIC %run ../../../Common/config
-- COMMAND ----------
-- MAGIC %python
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dim_product_wide`').createOrReplaceTempView('cdw_dwd_dim_product_wide')
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dwd_hospital_classification`').createOrReplaceTempView('cdw_dwd_dwd_hospital_classification')
-- COMMAND ----------
---------------------------------------------------------------------------
--修改时间：20241108
--修改人：Fanxujia
--修改内容：
--for AIA Dashboard
--增加sub_channel,dragon_flag,ka_flag,target_ins_level字段
---------------------------------------------------------------------------
---------------------------------------------------------------------------
--修改时间：20241209
--修改人：Fanxujia
--修改内容：
--增加sub channel的排序
---------------------------------------------------------------------------
--目标医院
insert overwrite table dm.dm_aia_targethp_flag --获取全部的机构 + 市场
with sales as (
select distinct t1.inst_code,t2.MARKET
from dm.dm_ext_aia_sales t1
left join DM.DM_TD_EXT_AIA_MARKET_PACK_MAPPING t2
on t1.pack_cod = t2.PACK_CODE
)
--获取产品与市场的映射关系
,pack_market as (
select distinct
PACK_CODE as PACK_COD,
MARKET
from DM.DM_TD_EXT_AIA_MARKET_PACK_MAPPING
)
--辖区表，获取到KA_Flag、target_ins_level以及目标机构Flag(在辖区表里存在的就是目标机构)
,td_territory_inst as (
select distinct
inst_code,
trty_code,
family_code,
ka_flag,
TARGET_LEVEL as target_ins_level
from dm.dm_td_sd_territory_indication --eagle的非重点医院同样加为目标机构
where yyyymm in (
select
key_value
from
dm.dm_td_date_config
where
key_code = 'pfmc_cvh'
)
and bu not in ('CUBE','KA')
and key_hp<>3 --chenwu 20250904 排除双考影响
)
--汇报线表，获取到Channel，后续用于判断出sub_channel
,td_org as (
select distinct
t1.mr_trty_code,
case when t1.bu_code in ('PC14','PC16','PC18','PC19') then 'BU'
when t1.bu_code = 'SC01' and t1.sub_bu_code = 'NA29' then sub_bu_name
when t1.bu_code = 'SC01' then 'BU'
when t1.bu_code <> 'SC01' and t2.bm_name = 'BCBH' then 'BU'
when t1.bu_code <> 'SC01' then t1.bu_name
end as channel
from dm.dm_td_org t1
left join dws.dws_td_bu_active t2
on t1.bu_name = t2.bu_name
where t1.yyyymm in (
select
key_value
from
dm.dm_td_date_config
where
key_code = 'pfmc_cvh'
)
)
--Family粒度和Brand粒度的映射表
,product_wide as (
select distinct
prd_family_cd,
prd_brand_cd
from cdw_dwd_dim_product_wide
)
--内部Brand粒度与外部pack_cod的映射表
,imscode_azcode as (
select distinct
brandcode,
case when length(ims_pack_cod) < 12 and ims_pack_cod REGEXP '^[0-9]' then right(concat('00000000000',ims_pack_cod),12) else ims_pack_cod end as ims_pack_cod
from dwd.dwd_td_imscode_azcode
)
--综合上面的几个表，得到KA_flag、target_ins_level、channel、目标机构Flag
,target_hp_all as (
select
distinct
t1.inst_code,
t1.ka_flag,
t1.target_ins_level,
t2.channel,
t5.MARKET
from
td_territory_inst t1
inner join td_org t2 on t1.trty_code = t2.mr_trty_code
left join product_wide t3 on t3.prd_family_cd = t1.family_code
left join imscode_azcode t4 on t3.prd_brand_cd = t4.brandcode
left join pack_market t5 on t4.ims_pack_cod = t5.pack_cod
where t5.MARKET is not null
)
--target_hp_all里一个机构一个市场对应了多个属性，这里按优先级依次取唯一值
--ka_flag
,ka_flag1 as (
select distinct
inst_code,
MARKET,
ka_flag
from target_hp_all
)
,ka_flag2 as (
select
inst_code,
MARKET,
ka_flag,
row_number() over(partition by inst_code,market
order by case when ka_flag = 'KA' then 1
ELSE 2 end asc) as RN
from ka_flag1
)
,ka_flag_final as (
select
inst_code,
MARKET,
ka_flag
from ka_flag2
where RN = 1
)
--target_ins_level不取唯一值
--加个hardcoding，ONC Market和ALL Market设置成NA
,target_ins_level1 as (
select distinct
inst_code,
MARKET,
case when MARKET in ('ALL Market','ONC Market') then 'NA'
else target_ins_level
end as target_ins_level
from target_hp_all
)
,target_ins_level_final as (
select distinct
inst_code,
MARKET,
target_ins_level
from target_ins_level1
)
--target_hp_all里一个机构一个市场对应了多个属性，这里按优先级依次取唯一值
--Channel
,Channel1 as (
select distinct
inst_code,
MARKET,
channel
from target_hp_all
)
,channel2 as (
select
inst_code,
MARKET,
channel,
row_number() over(partition by inst_code,market
order by case when channel = 'BU' then 1
when channel = 'BBU_County' then 2
when channel = 'CHC' then 3
when channel = 'Eagle' then 4
ELSE 5 end asc) as RN
from channel1
)
,channel_final as (
select
inst_code,
MARKET,
channel
from channel2
where RN = 1
)
--确定唯一值后重新放一起
,target_hp1 as (
select distinct
inst_code,
MARKET
from target_hp_all
)
,target_hp as (
select t1.inst_code,
t1.market,
t2.ka_flag,
t3.target_ins_level,
t4.channel
from target_hp1 t1
left join ka_flag_final t2
on t1.inst_code = t2.inst_code
and t1.market = t2.market
left join target_ins_level_final t3
on t1.inst_code = t3.inst_code
and t1.market = t3.market
left join channel_final t4
on t1.inst_code = t4.inst_code
and t1.market = t4.market
)
--获取Dragon_type
--判断Dragon_type是否为County
,county as (
select distinct inst_code,county_tier_desc
from dm.dm_td_institution
)
--判断另外2种
-- 20250825修改
,dragon_type as (
select src_hcc_code as inst_code,
src_prod_code as family_code,
max(is_core) is_core
from cdw_dwd_dwd_hospital_classification
where active_status = 1
and src_sales_cycle = (select key_value from dm.dm_td_date_config where key_code = 'pfmc_cvh')
group by src_hcc_code,
src_prod_code
)
--将Family_code转为pack_cod
,dragon_type_market1 as (
select distinct
t1.inst_code,
t1.is_core,
t4.market
from dragon_type t1
left join cdw_dwd_dim_product_wide t2
on t1.family_code = t2.prd_family_cd
left join dwd.dwd_td_imscode_azcode t3
on t2.prd_brand_cd = t3.brandcode
left join pack_market t4
on t3.ims_pack_cod = t4.pack_cod
where t4.market is not null
)
--按优先级取唯一值
,dragon_type_market2 as (
select distinct
inst_code,
is_core,
market,
row_number() over(partition by inst_code,market
order by case when is_core = 'CORE' then 1
when is_core = 'EMERGING' then 2
else 3 end asc) as RN
from dragon_type_market1
)
,dragon_type_market as (
select distinct
inst_code,
is_core,
market
from dragon_type_market2
where RN = 1
)
--综合前面几个表，得出dragon_flag
,dragon_flag1 as (
select distinct
t0.inst_code,
t0.market,
case when t1.county_tier_desc in ('County','County level city') then 'County'
when t2.is_core is null or length(t2.is_core) = 0 then 'Others'
else t2.is_core
end as dragon_flag
from target_hp t0
left join county t1
on t0.inst_code = t1.inst_code
left join dragon_type_market t2
on t0.inst_code = t2.inst_code
and t0.MARKET = t2.MARKET
)
--按优先级取唯一值
,dragon_flag2 as (
select inst_code,
market,
dragon_flag,
row_number() over(partition by inst_code,market
order by case when dragon_flag = 'CORE' then 1
when dragon_flag = 'EMERGING' then 2
when dragon_flag = 'County' then 3
else 4 end asc) as RN
from dragon_flag1
)
,dragon_flag as (
select inst_code,
market,
dragon_flag
from dragon_flag2
where RN = 1
)
--下面是为了把其他渠道并进来
,other_data_source as (
select explode(array('IQVIA-CHPA(Monthly)','XH Data(Quarterly)','EC(Monthly)','Retail(Quarterly)','CHC(Quarterly)','THC(Quarterly)','IQVIA-COUNTY(Quarterly)'))
)
,col as (
select '' as inst_code,
'' as MARKET,
'' as is_target_hp,
'' as sub_channel,
'' as dragon_flag,
'' as ka_flag,
'' as target_ins_level
)
,temp_1 as (
select distinct
case when t1.inst_code is null or t1.inst_code = '' then 'ROC'
else t1.inst_code
end as inst_code,
t1.MARKET,
case when t2.inst_code is null then 'N' else 'Y' end as is_target_hp,
case when t2.channel = 'BU' and geo.county_tier_desc in ('County','County level city') then 'BU_County'
when t2.channel = 'BU' and geo.county_tier_desc in ('City-3b','City-4a','City-4b') then 'BU_Emerging'
when t2.channel = 'BU' and t3.dragon_flag = 'CORE' then 'BU_Core'
when t2.channel = 'BU' then 'BU_Emerging'
when t2.channel in ('Eagle','CHC','BBU_County') then t2.channel
when t2.inst_code is null then 'NON_TARGET'
else 'NA'
end as sub_channel,
case when t3.dragon_flag is null then 'NON_TARGET'
else t3.dragon_flag
end as dragon_flag,
case when t2.ka_flag is null then 'NON_TARGET'
else t2.ka_flag
end as ka_flag,
case when t2.target_ins_level is null then 'NON_TARGET'
else t2.target_ins_level
end as target_ins_level,
'AIA(Monthly)' as DATA_SOURCE
from sales t1
left join target_hp t2
on t1.inst_code = t2.inst_code
and t1.MARKET = t2.MARKET
left join dragon_flag t3
on t1.inst_code = t3.inst_code
and t1.MARKET = t3.MARKET
left join dm.dm_td_institution inst
on t1.inst_code =inst.inst_code
left join dm.dm_td_geography geo
on inst.county_code = geo.geo_key
union all
select *
from col
cross join other_data_source
)
,sub_channel_rank as (
select distinct SUB_CHANNELSTD,SUB_CHANNEL_ORDER
from dm.dm_td_sd_channel
)
select t1.*,NVL(t2.SUB_CHANNEL_ORDER,100) as SUB_CHANNEL_ORDER
from temp_1 t1
left join sub_channel_rank t2
on t1.sub_channel = t2.SUB_CHANNELSTD

643
AIA/z_03 dm_aia_flag_备份用.sql Normal file
View File

@@ -0,0 +1,643 @@
-- Databricks notebook source
-- MAGIC %run ../../../Common/config
-- COMMAND ----------
-- MAGIC %python
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dim_product_wide`').createOrReplaceTempView('cdw_dwd_dim_product_wide')
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dwd_hospital_classification`').createOrReplaceTempView('cdw_dwd_dwd_hospital_classification')
-- COMMAND ----------
-----------------------------------------------------------------------------------
--修改时间：20241021
--修改人：FanXujia
--修改内容：
--只取最新年-1 ~ 最新年范围内的，称为“2年共有”。增加
-----------------------------------------------------------------------------------
--共有机构
insert overwrite table dm.dm_aia_hp_flag
with max_year as (
select
max(cast(year as int)) as max_year,
max(cast(year as int) - 1) as max_year_ly
from dwd.dwd_gnd_hospital_not_provided
)
,hp_flag as (
select
distinct concat(t1.year, '-Y') as hp_his_flag,
case when length(t1.cpa_hospital_code) < 7 then right(concat('0000000',t1.cpa_hospital_code),7) else t1.cpa_hospital_code end as cpa_hospital_code
from
dwd.dwd_gnd_hospital_not_provided t1
cross join max_year t2
where cast(t1.year as int) >= t2.max_year_ly
)
,hp_flag2 as (
SELECT
cpa_hospital_code,
CONCAT_WS(' ∩ ', SORT_ARRAY(COLLECT_LIST(hp_his_flag))) AS FLAG
FROM
hp_flag
GROUP BY
cpa_hospital_code
)
,inst_mapping as (
-- select
-- distinct right(concat('0000000', org_cd), 7) org_cd,
-- ins_cd
-- from
-- dwd.dwd_gnd_hospitalmapping
select distinct
case when length(cpa_hospital_code) < 7 then right(concat('0000000',cpa_hospital_code),7) else cpa_hospital_code end as cpa_hospital_code,
ins_cd_nl as inst_code
from dwd.dwd_gnd_hospital_not_provided
)
,ALL_INS as (
select distinct inst_code
from dm.dm_ext_aia_sales
)
,flag as (
select distinct
coalesce(t2.inst_code,t1.cpa_hospital_code) as inst_code,
t1.flag as aia_hp_flag,
'Y' as FLAG,
'AIA(Monthly)' as DATA_SOURCE
from hp_flag2 t1
left join inst_mapping t2 on t1.cpa_hospital_code = t2.cpa_hospital_code
where
length(t1.flag) > 6
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'IQVIA-CHPA(Monthly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'XH Data(Quarterly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'EC(Monthly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'Retail(Quarterly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'CHC(Quarterly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'THC(Quarterly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'IQVIA-COUNTY(Quarterly)' as DATA_SOURCE
)
select t1.inst_code,
NVL(t2.aia_hp_flag,'') as aia_hp_flag,
NVL(t2.FLAG,'N') as FLAG,
'AIA(Monthly)' as DATA_SOURCE
from ALL_INS t1
left join flag t2
on t1.inst_code = t2.inst_code
-- COMMAND ----------
-----------------------------------------------------------------------------------
--修改时间：20241021
--修改人：FanXujia
--修改内容：
--只取最新年-1 ~ 最新年范围内的，称为“2年共有”。增加
-----------------------------------------------------------------------------------
--共有机构
insert overwrite table dm.dm_aia_hp_flag
with max_year as (
select
max(cast(year as int)) as max_year,
max(cast(year as int) - 1) as max_year_ly
from dwd.dwd_gnd_hospital_not_provided
)
,hp_flag as (
select
distinct concat(t1.year, '-Y') as hp_his_flag,
case when length(t1.cpa_hospital_code) < 7 then right(concat('0000000',t1.cpa_hospital_code),7) else t1.cpa_hospital_code end as cpa_hospital_code
from
dwd.dwd_gnd_hospital_not_provided t1
cross join max_year t2
where cast(t1.year as int) >= t2.max_year_ly
)
,hp_flag2 as (
SELECT
cpa_hospital_code,
CONCAT_WS(' ∩ ', SORT_ARRAY(COLLECT_LIST(hp_his_flag))) AS FLAG
FROM
hp_flag
GROUP BY
cpa_hospital_code
)
,inst_mapping as (
-- select
-- distinct right(concat('0000000', org_cd), 7) org_cd,
-- ins_cd
-- from
-- dwd.dwd_gnd_hospitalmapping
select distinct
case when length(cpa_hospital_code) < 7 then right(concat('0000000',cpa_hospital_code),7) else cpa_hospital_code end as cpa_hospital_code,
ins_cd_nl as inst_code
from dwd.dwd_gnd_hospital_not_provided
)
,ALL_INS as (
select distinct inst_code
from dm.dm_ext_aia_sales
)
,flag as (
select distinct
coalesce(t2.inst_code,t1.cpa_hospital_code) as inst_code,
t1.flag as aia_hp_flag,
'Y' as FLAG,
'AIA(Monthly)' as DATA_SOURCE
from hp_flag2 t1
left join inst_mapping t2 on t1.cpa_hospital_code = t2.cpa_hospital_code
where
length(t1.flag) > 6
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'IQVIA-CHPA(Monthly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'XH Data(Quarterly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'EC(Monthly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'Retail(Quarterly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'CHC(Quarterly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'THC(Quarterly)' as DATA_SOURCE
union all
select '' as inst_code,'' as aia_hp_flag,'' as FLAG,'IQVIA-COUNTY(Quarterly)' as DATA_SOURCE
)
select t1.inst_code,
NVL(t2.aia_hp_flag,'') as aia_hp_flag,
NVL(t2.FLAG,'N') as FLAG,
'AIA(Monthly)' as DATA_SOURCE
from ALL_INS t1
left join flag t2
on t1.inst_code = t2.inst_code
-- COMMAND ----------
--连续取数机构
----------------------------------------------------------------------------------------------------------------------
--修改时间：20241029
--修改人：FanXujia
--修改内容：
--根据事实表最新月，往前推12个月（含最新月），按机构 + 年月粒度汇总，如果每个月销量都不等于0，这家机构称为：滚动一年有数
--根据事实表最新月，往前推24个月（含最新月），按机构 + 年月粒度汇总，如果每个月销量都不等于0，这家机构称为：滚动两年有数
--2022年没有销量为正常情况，判断时仅考虑2023年01月起销量是否等于0
----------------------------------------------------------------------------------------------------------------------
insert overwrite table dm.dm_aia_provided_flag
with max_ym as (
select max(ym) as max_ym
from dm.dm_ext_aia_sales
)
,ym_range as (
select max_ym,
case when max_ym <= '202412' then cast(right(max_ym,2) as int)
else cast(12 as int)
end ym_12m,
case when max_ym <= '202312' then '202401'
when right(max_ym,2) = '12' then concat(left(max_ym,4),'01')
else concat(cast(left(max_ym, 4) - 1 as int),right(concat('0', cast(right(max_ym, 2) + 1 as int)), 2))
end as ym_12m_before,
case when max_ym <= '202312' then cast(right(max_ym,2) as int)
when max_ym <= '202512' then cast(right(max_ym,2) as int) + 12
else cast(24 as int)
end as ym_24m,
case when max_ym <= '202512' then '202401'
when right(max_ym,2) = '12' then concat(cast(left(max_ym,4) - 1 as int),'01')
else concat(cast(left(max_ym, 4) -2 as int),right(concat('0', cast(right(max_ym, 2) + 1 as int)), 2))
end as ym_24m_before
from max_ym
)
,sales_1 as (
select
inst_code,
ym
from
dm.dm_ext_aia_sales
group by
ym,
inst_code
having
sum(value) <> 0
)
,sales_2 as (
select
t1.inst_code,
t1.ym,
t2.max_ym,
t2.ym_12m,
t2.ym_12m_before,
t2.ym_24m,
t2.ym_24m_before
from sales_1 t1
cross join ym_range t2
)
,flag_12m as (
select inst_code,
'滚动一年有数' as aia_provided_flag,
'AIA(Monthly)' as DATA_SOURCE
from sales_2
where ym <= max_ym
and ym >= ym_12m_before
group by inst_code
having count(1) = (select ym_12m from ym_range)
)
,flag_24m as (
select inst_code,
'滚动两年有数' as aia_provided_flag,
'AIA(Monthly)' as DATA_SOURCE
from sales_2
where ym <= max_ym
and ym >= ym_24m_before
group by inst_code
having count(1) = (select ym_24m from ym_range)
)
select inst_code,
aia_provided_flag,
DATA_SOURCE
from flag_12m
union all
select inst_code,
aia_provided_flag,
DATA_SOURCE
from flag_24m
union all
select '' as inst_code,'' as aia_provided_flag,'IQVIA-CHPA(Monthly)' as data_source
union all
select '' as inst_code,'' as aia_provided_flag,'XH Data(Quarterly)' as data_source
union all
select '' as inst_code,'' as aia_provided_flag,'EC(Monthly)' as data_source
union all
select '' as inst_code,'' as aia_provided_flag,'Retail(Quarterly)' as data_source
union all
select '' as inst_code,'' as aia_provided_flag,'CHC(Quarterly)' as data_source
union all
select '' as inst_code,'' as aia_provided_flag,'THC(Quarterly)' as data_source
union all
select '' as inst_code,'' as aia_provided_flag,'IQVIA-COUNTY(Quarterly)' as data_source
-- COMMAND ----------
---------------------------------------------------------------------------
--修改时间：20241108
--修改人：Fanxujia
--修改内容：
--for AIA Dashboard
--增加sub_channel,dragon_flag,ka_flag,target_ins_level字段
---------------------------------------------------------------------------
---------------------------------------------------------------------------
--修改时间：20241209
--修改人：Fanxujia
--修改内容：
--增加sub channel的排序
---------------------------------------------------------------------------
--目标医院
insert overwrite table dm.dm_aia_targethp_flag --获取全部的机构 + 市场
with sales as (
select distinct t1.inst_code,t2.MARKET
from dm.dm_ext_aia_sales t1
left join dm.dm_aia_pack_property t2
on t1.pack_cod = t2.PACK_COD
)
--获取产品与市场的映射关系
,pack_market as (
select distinct
PACK_COD,
MARKET
from dm.dm_aia_pack_property
)
--辖区表，获取到KA_Flag、target_ins_level以及目标机构Flag(在辖区表里存在的就是目标机构)
,td_territory_inst as (
select distinct
inst_code,
trty_code,
family_code,
ka_flag,
TARGET_LEVEL as target_ins_level
from dm.dm_td_sd_territory_indication --eagle的非重点医院同样加为目标机构
where yyyymm in (
select
key_value
from
dm.dm_td_date_config
where
key_code = 'pfmc_cvh'
)
and bu not in ('CUBE','KA')
and key_hp<>3 --chenwu 20250904 排除双考影响
)
--汇报线表，获取到Channel，后续用于判断出sub_channel
,td_org as (
select distinct
t1.mr_trty_code,
case when t1.bu_code in ('PC14','PC16','PC18','PC19') then 'BU'
when t1.bu_code = 'SC01' and t1.sub_bu_code = 'NA29' then sub_bu_name
when t1.bu_code = 'SC01' then 'BU'
when t1.bu_code <> 'SC01' and t2.bm_name = 'BCBH' then 'BU'
when t1.bu_code <> 'SC01' then t1.bu_name
end as channel
from dm.dm_td_org t1
left join dws.dws_td_bu_active t2
on t1.bu_name = t2.bu_name
where t1.yyyymm in (
select
key_value
from
dm.dm_td_date_config
where
key_code = 'pfmc_cvh'
)
)
--Family粒度和Brand粒度的映射表
,product_wide as (
select distinct
prd_family_cd,
prd_brand_cd
from cdw_dwd_dim_product_wide
)
--内部Brand粒度与外部pack_cod的映射表
,imscode_azcode as (
select distinct
brandcode,
case when length(ims_pack_cod) < 12 and ims_pack_cod REGEXP '^[0-9]' then right(concat('00000000000',ims_pack_cod),12) else ims_pack_cod end as ims_pack_cod
from dwd.dwd_td_imscode_azcode
)
--综合上面的几个表，得到KA_flag、target_ins_level、channel、目标机构Flag
,target_hp_all as (
select
distinct
t1.inst_code,
t1.ka_flag,
t1.target_ins_level,
t2.channel,
t5.MARKET
from
td_territory_inst t1
inner join td_org t2 on t1.trty_code = t2.mr_trty_code
left join product_wide t3 on t3.prd_family_cd = t1.family_code
left join imscode_azcode t4 on t3.prd_brand_cd = t4.brandcode
left join pack_market t5 on t4.ims_pack_cod = t5.pack_cod
where t5.MARKET is not null
)
--target_hp_all里一个机构一个市场对应了多个属性，这里按优先级依次取唯一值
--ka_flag
,ka_flag1 as (
select distinct
inst_code,
MARKET,
ka_flag
from target_hp_all
)
,ka_flag2 as (
select
inst_code,
MARKET,
ka_flag,
row_number() over(partition by inst_code,market
order by case when ka_flag = 'KA' then 1
ELSE 2 end asc) as RN
from ka_flag1
)
,ka_flag_final as (
select
inst_code,
MARKET,
ka_flag
from ka_flag2
where RN = 1
)
--target_ins_level不取唯一值
--加个hardcoding，ONC Market和ALL Market设置成NA
,target_ins_level1 as (
select distinct
inst_code,
MARKET,
case when MARKET in ('AIA ALL Market','ONC Market') then 'NA'
else target_ins_level
end as target_ins_level
from target_hp_all
)
,target_ins_level_final as (
select distinct
inst_code,
MARKET,
target_ins_level
from target_ins_level1
)
--target_hp_all里一个机构一个市场对应了多个属性，这里按优先级依次取唯一值
--Channel
,Channel1 as (
select distinct
inst_code,
MARKET,
channel
from target_hp_all
)
,channel2 as (
select
inst_code,
MARKET,
channel,
row_number() over(partition by inst_code,market
order by case when channel = 'BU' then 1
when channel = 'BBU_County' then 2
when channel = 'CHC' then 3
when channel = 'Eagle' then 4
ELSE 5 end asc) as RN
from channel1
)
,channel_final as (
select
inst_code,
MARKET,
channel
from channel2
where RN = 1
)
--确定唯一值后重新放一起
,target_hp1 as (
select distinct
inst_code,
MARKET
from target_hp_all
)
,target_hp as (
select t1.inst_code,
t1.market,
t2.ka_flag,
t3.target_ins_level,
t4.channel
from target_hp1 t1
left join ka_flag_final t2
on t1.inst_code = t2.inst_code
and t1.market = t2.market
left join target_ins_level_final t3
on t1.inst_code = t3.inst_code
and t1.market = t3.market
left join channel_final t4
on t1.inst_code = t4.inst_code
and t1.market = t4.market
)
--获取Dragon_type
--判断Dragon_type是否为County
,county as (
select distinct inst_code,county_tier_desc
from dm.dm_td_institution
)
--判断另外2种
-- 20250825修改
,dragon_type as (
select src_hcc_code as inst_code,
src_prod_code as family_code,
max(is_core) is_core
from cdw_dwd_dwd_hospital_classification
where active_status = 1
and src_sales_cycle = (select key_value from dm.dm_td_date_config where key_code = 'pfmc_cvh')
group by src_hcc_code,
src_prod_code
)
--将Family_code转为pack_cod
,dragon_type_market1 as (
select distinct
t1.inst_code,
t1.is_core,
t4.market
from dragon_type t1
left join cdw_dwd_dim_product_wide t2
on t1.family_code = t2.prd_family_cd
left join dwd.dwd_td_imscode_azcode t3
on t2.prd_brand_cd = t3.brandcode
left join pack_market t4
on t3.ims_pack_cod = t4.pack_cod
where t4.market is not null
)
--按优先级取唯一值
,dragon_type_market2 as (
select distinct
inst_code,
is_core,
market,
row_number() over(partition by inst_code,market
order by case when is_core = 'CORE' then 1
when is_core = 'EMERGING' then 2
else 3 end asc) as RN
from dragon_type_market1
)
,dragon_type_market as (
select distinct
inst_code,
is_core,
market
from dragon_type_market2
where RN = 1
)
--综合前面几个表，得出dragon_flag
,dragon_flag1 as (
select distinct
t0.inst_code,
t0.market,
case when t1.county_tier_desc in ('County','County level city') then 'County'
when t2.is_core is null or length(t2.is_core) = 0 then 'Others'
else t2.is_core
end as dragon_flag
from target_hp t0
left join county t1
on t0.inst_code = t1.inst_code
left join dragon_type_market t2
on t0.inst_code = t2.inst_code
and t0.MARKET = t2.MARKET
)
--按优先级取唯一值
,dragon_flag2 as (
select inst_code,
market,
dragon_flag,
row_number() over(partition by inst_code,market
order by case when dragon_flag = 'CORE' then 1
when dragon_flag = 'EMERGING' then 2
when dragon_flag = 'County' then 3
else 4 end asc) as RN
from dragon_flag1
)
,dragon_flag as (
select inst_code,
market,
dragon_flag
from dragon_flag2
where RN = 1
)
--下面是为了把其他渠道并进来
,other_data_source as (
select explode(array('IQVIA-CHPA(Monthly)','XH Data(Quarterly)','EC(Monthly)','Retail(Quarterly)','CHC(Quarterly)','THC(Quarterly)','IQVIA-COUNTY(Quarterly)'))
)
,col as (
select '' as inst_code,
'' as MARKET,
'' as is_target_hp,
'' as sub_channel,
'' as dragon_flag,
'' as ka_flag,
'' as target_ins_level
)
,temp_1 as (
select distinct
case when t1.inst_code is null or t1.inst_code = '' then 'ROC'
else t1.inst_code
end as inst_code,
t1.MARKET,
case when t2.inst_code is null then 'N' else 'Y' end as is_target_hp,
case when t2.channel = 'BU' and geo.county_tier_desc in ('County','County level city') then 'BU_County'
when t2.channel = 'BU' and geo.county_tier_desc in ('City-3b','City-4a','City-4b') then 'BU_Emerging'
when t2.channel = 'BU' and t3.dragon_flag = 'CORE' then 'BU_Core'
when t2.channel = 'BU' then 'BU_Emerging'
when t2.channel in ('Eagle','CHC','BBU_County') then t2.channel
when t2.inst_code is null then 'NON_TARGET'
else 'NA'
end as sub_channel,
case when t3.dragon_flag is null then 'NON_TARGET'
else t3.dragon_flag
end as dragon_flag,
case when t2.ka_flag is null then 'NON_TARGET'
else t2.ka_flag
end as ka_flag,
case when t2.target_ins_level is null then 'NON_TARGET'
else t2.target_ins_level
end as target_ins_level,
'AIA(Monthly)' as DATA_SOURCE
from sales t1
left join target_hp t2
on t1.inst_code = t2.inst_code
and t1.MARKET = t2.MARKET
left join dragon_flag t3
on t1.inst_code = t3.inst_code
and t1.MARKET = t3.MARKET
left join dm.dm_td_institution inst
on t1.inst_code =inst.inst_code
left join dm.dm_td_geography geo
on inst.county_code = geo.geo_key
union all
select *
from col
cross join other_data_source
)
,sub_channel_rank as (
select distinct SUB_CHANNELSTD,SUB_CHANNEL_ORDER
from dm.dm_td_sd_channel
)
select t1.*,NVL(t2.SUB_CHANNEL_ORDER,100) as SUB_CHANNEL_ORDER
from temp_1 t1
left join sub_channel_rank t2
on t1.sub_channel = t2.SUB_CHANNELSTD

231
CHC/01 dm_chc_pack_property.sql Normal file
View File

@@ -0,0 +1,231 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE dm.dm_chc_pack_property (
-- PACK_COD STRING,
-- pack_des STRING,
-- stgh_des STRING,
-- pack_lch STRING,
-- PROD_COD STRING,
-- prod_des STRING,
-- prod_des_c STRING,
-- Family_Code STRING,
-- Family_Name STRING,
-- cmps_cod STRING,
-- cmps_des STRING,
-- cmps_des_c STRING,
-- atc1_cod STRING,
-- atc1_des STRING,
-- atc1_des_c STRING,
-- atc2_cod STRING,
-- atc2_des STRING,
-- atc2_des_c STRING,
-- atc3_cod STRING,
-- atc3_des STRING,
-- atc3_des_c STRING,
-- atc4_cod STRING,
-- atc4_des STRING,
-- atc4_des_c STRING,
-- app1_cod STRING,
-- app1_des STRING,
-- app1_des_c STRING,
-- app2_cod STRING,
-- app2_des STRING,
-- app2_des_c STRING,
-- app3_cod STRING,
-- app3_des STRING,
-- app3_des_c STRING,
-- bio_desc STRING,
-- gene_orig_desc STRING,
-- eth_otc_desc STRING,
-- nrdl_desc STRING,
-- NRDL_Entry_Date STRING,
-- edl_desc STRING,
-- tcm_desc STRING,
-- paed_desc STRING,
-- gqce_desc STRING,
-- vbp_desc STRING,
-- manu_cod STRING,
-- manu_des STRING,
-- manu_des_c STRING,
-- mnfl_cod STRING,
-- mnfl_des STRING,
-- corp_cod STRING,
-- corp_des STRING,
-- corp_des_c STRING,
-- BrandType STRING,
-- IS_AZ STRING,
-- AZ_MAIN STRING,
-- AZ_Related STRING,
-- countingunit STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_aia_pack_property';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_aia_pack_property';
-- COMMAND ----------
-- update dwd.dwd_gnd_ext_chc_pack_property
-- set iqvia_pack_code = if( iqvia_pack_code REGEXP '^[0-9]',right(concat('000000000000',iqvia_pack_code),12),iqvia_pack_code)
-- ,iqvia_prod_code = right(concat('000000000',iqvia_prod_code),9)
-- COMMAND ----------
create or replace temporary view dwd_pack_property
as
select distinct
iqvia_pack_code ,
iqvia_prod_code ,
pack_cod ,
app1_cod ,
app1_des ,
app1_des_c ,
app2_cod ,
app2_des ,
app2_des_c ,
app3_cod ,
app3_des ,
app3_des_c ,
atc1_cod ,
atc1_des ,
atc1_des_c ,
atc2_cod ,
atc2_des ,
atc2_des_c ,
atc3_cod ,
atc3_des ,
atc3_des_c ,
atc4_cod ,
atc4_des ,
atc4_des_c ,
bio_desc ,
cmps_cod ,
cmps_des ,
cmps_des_c ,
corp_cod ,
corp_des ,
corp_des_c ,
edl_desc ,
eth_otc_desc ,
gene_orig_desc ,
gqce_desc ,
manu_cod ,
manu_des ,
manu_des_c ,
mnfl_cod ,
mnfl_des ,
nrdl_desc ,
pack_des ,
stgh_des ,
pack_lch ,
paed_desc ,
prod_des ,
prod_des_c ,
tcm_desc ,
vbp_desc ,
-- builtinarchivedate ,
source_file_path ,
source_file_name ,
etl_insert_dt ,
countingunit
from dwd.dwd_gnd_dept_pack_property
-- COMMAND ----------
-- create or replace table dm.dm_chc_pack_property
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_chc_pack_property'
-- as
insert overwrite table dm.dm_chc_pack_property
WITH dedup_l AS (
SELECT *,
ROW_NUMBER() OVER (PARTITION BY iqvia_pack_code ORDER BY iqvia_pack_code) AS rn
FROM dwd_pack_property
QUALIFY rn = 1
),
dedup_r AS (
SELECT *,
ROW_NUMBER() OVER (PARTITION BY PACK_COD ORDER BY PACK_COD) AS rn
FROM tmp.tmp_ext_chpa_pack_propetry
QUALIFY rn = 1
)
SELECT
l.iqvia_pack_code AS PACK_COD,
-- Pack / Product 描述
COALESCE(r.PACK_DES, l.pack_des) AS pack_des,
COALESCE(r.STGH_DES, l.stgh_des) AS stgh_des,
COALESCE(r.PACK_LCH, l.pack_lch) AS pack_lch,
COALESCE(r.PROD_COD, l.iqvia_prod_code) AS PROD_COD,
COALESCE(r.PROD_DES, l.prod_des) AS prod_des,
COALESCE(r.PROD_DES_C, l.prod_des_c) AS prod_des_c,
-- 右表独有：Family
r.Family_Code,
r.Family_Name,
-- 成分
COALESCE(r.CMPS_COD, l.cmps_cod) AS cmps_cod,
COALESCE(r.CMPS_DES, l.cmps_des) AS cmps_des,
COALESCE(r.CMPS_DES_C, l.cmps_des_c) AS cmps_des_c,
-- ATC
COALESCE(r.ATC1_COD, l.atc1_cod) AS atc1_cod,
COALESCE(r.atc1_des, l.atc1_des) AS atc1_des,
COALESCE(r.atc1_des_c, l.atc1_des_c) AS atc1_des_c,
COALESCE(r.ATC2_COD, l.atc2_cod) AS atc2_cod,
COALESCE(r.atc2_des, l.atc2_des) AS atc2_des,
COALESCE(r.atc2_des_c, l.atc2_des_c) AS atc2_des_c,
COALESCE(r.ATC3_COD, l.atc3_cod) AS atc3_cod,
COALESCE(r.atc3_des, l.atc3_des) AS atc3_des,
COALESCE(r.atc3_des_c, l.atc3_des_c) AS atc3_des_c,
COALESCE(r.ATC4_COD, l.atc4_cod) AS atc4_cod,
COALESCE(r.atc4_des, l.atc4_des) AS atc4_des,
COALESCE(r.atc4_des_c, l.atc4_des_c) AS atc4_des_c,
-- APP
COALESCE(r.APP1_COD, l.app1_cod) AS app1_cod,
COALESCE(r.app1_des, l.app1_des) AS app1_des,
COALESCE(r.app1_des_c, l.app1_des_c) AS app1_des_c,
COALESCE(r.APP2_COD, l.app2_cod) AS app2_cod,
COALESCE(r.app2_des, l.app2_des) AS app2_des,
COALESCE(r.app2_des_c, l.app2_des_c) AS app2_des_c,
COALESCE(r.APP3_COD, l.app3_cod) AS app3_cod,
COALESCE(r.app3_des, l.app3_des) AS app3_des,
COALESCE(r.app3_des_c, l.app3_des_c) AS app3_des_c,
-- 标志位
COALESCE(r.BIO_DESC, l.bio_desc) AS bio_desc,
COALESCE(r.GENE_ORIG_DESC, l.gene_orig_desc) AS gene_orig_desc,
COALESCE(r.ETH_OTC_DESC, l.eth_otc_desc) AS eth_otc_desc,
COALESCE(r.NRDL_DESC, l.nrdl_desc) AS nrdl_desc,
r.NRDL_Entry_Date,
COALESCE(r.EDL_DESC, l.edl_desc) AS edl_desc,
COALESCE(r.TCM_DESC, l.tcm_desc) AS tcm_desc,
COALESCE(r.PAED_DESC, l.paed_desc) AS paed_desc,
COALESCE(r.GQCE_DESC, l.gqce_desc) AS gqce_desc,
COALESCE(r.VBP_DESC, l.vbp_desc) AS vbp_desc,
-- 厂商
COALESCE(r.MANU_COD, l.manu_cod) AS manu_cod,
COALESCE(r.MANU_DES, l.manu_des) AS manu_des,
COALESCE(r.MANU_DES_C, l.manu_des_c) AS manu_des_c,
COALESCE(r.MNFL_COD, l.mnfl_cod) AS mnfl_cod,
COALESCE(r.MNFL_DES, l.mnfl_des) AS mnfl_des,
-- 公司
COALESCE(r.CORP_COD, l.corp_cod) AS corp_cod,
COALESCE(r.CORP_DES, l.corp_des) AS corp_des,
COALESCE(r.CORP_DES_C, l.corp_des_c) AS corp_des_c,
-- 右表独有：品牌 / AZ 标志
r.BrandType,
r.IS_AZ,
r.AZ_MAIN,
r.AZ_Related,
-- 左表独有
l.countingunit,
from_utc_timestamp(current_timestamp(),'UTC+8') as ETL_INSERT_DT,
from_utc_timestamp(current_timestamp(),'UTC+8') as ETL_UPDATE_DT
FROM dedup_l l
LEFT JOIN dedup_r r
ON l.iqvia_pack_code = r.PACK_COD

418
CHC/02 DM_TF_EXT_CHC_SALES.sql Normal file
View File

@@ -0,0 +1,418 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TF_EXT_CHC_SALES (
-- YYYYMM STRING,
-- PACK_CODE STRING,
-- CORP_CODE STRING,
-- AUDIT_CODE STRING,
-- PLATFORM_TYPE STRING,
-- STORE_NAME STRING,
-- STORE_TYPE STRING,
-- REGION_TYPE STRING,
-- PACK_FLAG INT,
-- PROD_FLAG INT,
-- DTP_FLAG INT,
-- SALES_UNIT_CAL DECIMAL(38,10),
-- SALES_UNIT_CAL_LY DECIMAL(38,10),
-- SALES_VALUE_CAL DECIMAL(38,10),
-- SALES_VALUE_CAL_LY DECIMAL(38,10),
-- CONUTING_UNIT DECIMAL(38,10),
-- CONUTING_UNIT_LY DECIMAL(38,10),
-- DATA_SOURCE STRING,
-- INST_CODE STRING COMMENT '内部机构编码',
-- CMPS_FLAG STRING COMMENT '分子式标签',
-- DEPT_NAME STRING COMMENT '科室名称',
-- PRESCRIPTION DECIMAL(38,10) COMMENT '处方张数',
-- PRESCRIPTION_LY DECIMAL(38,10) COMMENT '去年同期处方张数',
-- NEW_CODE STRING COMMENT '主数据关联CODE',
-- AREA STRING COMMENT '城市',
-- H_LEVEL STRING COMMENT '医院类型',
-- REIMBURSE STRING COMMENT '报销情况',
-- REIMBURSE_TYPE STRING COMMENT '报销类型',
-- PRESCRIPTION_SOURCE STRING COMMENT '处方来源',
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP
-- )
-- USING delta
-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_chc_sales';
-- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_chc_sales';
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW RAW_DATA
AS
SELECT
CASE
WHEN SUBSTRING(YQ, 5, 2) = 'Q1'
THEN CONCAT (SUBSTRING(YQ, 1, 4), '03')
WHEN SUBSTRING(YQ, 5, 2) = 'Q2'
THEN CONCAT (SUBSTRING(YQ, 1, 4), '06')
WHEN SUBSTRING(YQ, 5, 2) = 'Q3'
THEN CONCAT (SUBSTRING(YQ, 1, 4), '09')
WHEN SUBSTRING(YQ, 5, 2) = 'Q4'
THEN CONCAT (SUBSTRING(YQ, 1, 4), '12')
ELSE YQ
END AS YQ,
CITY_C,
PACK_COD,
PROD_COD,
VALUE,
TOTALUNIT,
COUNTINGUNIT
FROM
DWD.DWD_GND_CHC_RAWDATA
WHERE
YEAR >= '2021'
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW FILTER_YM
AS
SELECT
MAX(YQ) AS MAX_YYYYMM,
CONCAT(CAST(FLOOR(MAX(YQ)/100-1) AS STRING),'01') AS MIN_YYYYMM
FROM
RAW_DATA
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW CORP_DATA
AS
SELECT
PACK_COD,
CORP_COD,
CORP_DES
FROM
DM.DM_IMS_TD_PACK_PROPERTY
GROUP BY
PACK_COD,
CORP_COD,
CORP_DES
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW PACK_DATA
AS
SELECT
DISTINCT
IQVIA_PACK_CODE,
IQVIA_PROD_CODE,
PACK_COD,
APP1_COD,
APP1_DES,
APP1_DES_C,
APP2_COD,
APP2_DES,
APP2_DES_C,
APP3_COD,
APP3_DES,
APP3_DES_C,
ATC1_COD,
ATC1_DES,
ATC1_DES_C,
ATC2_COD,
ATC2_DES,
ATC2_DES_C,
ATC3_COD,
ATC3_DES,
ATC3_DES_C,
ATC4_COD,
ATC4_DES,
ATC4_DES_C,
BIO_DESC,
CMPS_COD,
CMPS_DES,
CMPS_DES_C,
CORP_COD,
CORP_DES,
CORP_DES_C,
EDL_DESC,
ETH_OTC_DESC,
GENE_ORIG_DESC,
GQCE_DESC,
MANU_COD,
MANU_DES,
MANU_DES_C,
MNFL_COD,
MNFL_DES,
NRDL_DESC,
PACK_DES,
STGH_DES,
PACK_LCH,
PAED_DESC,
PROD_DES,
PROD_DES_C,
TCM_DESC,
VBP_DESC,
-- BUILTINARCHIVEDATE,
SOURCE_FILE_PATH,
SOURCE_FILE_NAME,
ETL_INSERT_DT,
COUNTINGUNIT
FROM
DWD.dwd_gnd_dept_pack_property
-- COMMAND ----------
--CHC本身唯独表 存在CORP_COD以及 CORP_DES 与手工数据不一致的问题
CREATE OR REPLACE TEMPORARY VIEW PACK_WIHT_CORP_DATA
AS
SELECT
T1.PACK_COD,
T1.IQVIA_PACK_CODE,
T1.COUNTINGUNIT,
COALESCE(T2.CORP_COD, T1.CORP_COD) AS CORP_COD,
COALESCE(T2.CORP_DES, T1.CORP_DES) AS CORP_DES
FROM
PACK_DATA T1
LEFT JOIN
CORP_DATA T2
ON
T1.IQVIA_PACK_CODE = T2.PACK_COD
-- COMMAND ----------
--GEO相关信息 在CHPA中没有 苏州、无锡、福州等数据,根据业务反馈，按照 CITY_C当作AUDIT_COD
CREATE OR REPLACE TEMPORARY VIEW DIM_EXCHC_GEO
AS
SELECT
T1.CITY_E,
CONCAT (T1.CITY_C, '市') AS CITY_C,
UPPER(NVL (T2.AUDIT_COD, T1.CITY_E)) AS AUDIT_COD
FROM
DWD.DWD_GND_EXT_CHC_GEO T1
LEFT JOIN
DM.DM_IMS_TD_GEO T2
ON
T1.CITY_E = T2.CITY
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW RAW_DATA_WIHT_PACK_CORP_GEO_DATA
AS
SELECT
T1.YQ,
T1.CITY_C,
T3.IQVIA_PACK_CODE AS PACK_COD,
T3.CORP_COD AS CORP_COD,
C.AUDIT_COD AS AUDIT_COD,
T1.PROD_COD,
T1.VALUE,
T1.TOTALUNIT,
T1.TOTALUNIT * NVL (T3.COUNTINGUNIT, 1) AS COUNTINGUNIT
FROM
RAW_DATA T1
LEFT JOIN
PACK_WIHT_CORP_DATA T3
ON
T1.PACK_COD = T3.PACK_COD
LEFT JOIN
DIM_EXCHC_GEO C
ON
T1.CITY_C = C.CITY_C
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW RAW_DATA_WIHT_LY
AS
SELECT
A.YQ,
A.AUDIT_COD,
A.PACK_COD,
A.CORP_COD,
SUM(A.VALUE) VALUE,
SUM(A.TOTALUNIT) TOTALUNIT,
SUM(A.COUNTINGUNIT) COUNTINGUNIT,
SUM(VALUE_LY) VALUE_LY,
SUM(TOTALUNIT_LY) TOTALUNIT_LY,
SUM(COUNTINGUNIT_LY) COUNTINGUNIT_LY
FROM
(
SELECT
T1.YQ,
T1.AUDIT_COD,
T1.PACK_COD,
T1.CORP_COD,
T1.VALUE,
T1.TOTALUNIT,
T1.COUNTINGUNIT,
0 VALUE_LY,
0 TOTALUNIT_LY,
0 COUNTINGUNIT_LY
FROM
RAW_DATA_WIHT_PACK_CORP_GEO_DATA T1
UNION ALL
SELECT
CAST(T1.YQ + 100 AS INT),
T1.AUDIT_COD,
T1.PACK_COD,
T1.CORP_COD,
0 VALUE,
0 TOTALUNIT,
0 COUNTINGUNIT,
T1.VALUE VALUE_LY,
T1.TOTALUNIT TOTALUNIT_LY,
T1.COUNTINGUNIT COUNTINGUNIT_LY
FROM
RAW_DATA_WIHT_PACK_CORP_GEO_DATA AS T1
CROSS JOIN
FILTER_YM AS T2
WHERE
T1.YQ + 100 <= T2.MAX_YYYYMM
) A
GROUP BY
A.YQ,
A.AUDIT_COD,
A.PACK_COD,
A.CORP_COD
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW RAW_DATA_FINAL
AS
SELECT
T1.YQ AS YYYYMM,
T1.AUDIT_COD,
T1.PACK_COD,
T1.CORP_COD,
SUM(TOTALUNIT) AS SALES_UNIT_CAL, -- TOTALUNIT,
SUM(TOTALUNIT_LY) AS SALES_UNIT_CAL_LY, --TOTALUNIT_LY,
SUM(VALUE) AS SALES_VALUE_CAL, --VALUE,
SUM(VALUE_LY) AS SALES_VALUE_CAL_LY,
SUM(COUNTINGUNIT) AS CONUTING_UNIT, --COUNTINGUNIT,
SUM(COUNTINGUNIT_LY) AS CONUTING_UNIT_LY,
'CHC(Quarterly)' AS DATA_SOURCE
FROM
RAW_DATA_WIHT_LY T1
GROUP BY
T1.YQ,
T1.AUDIT_COD,
T1.PACK_COD,
T1.CORP_COD
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW DM_TF_EXT_CHC_SALES AS
SELECT
A.YYYYMM,
A.PACK_COD AS PACK_CODE,
A.AUDIT_COD AS AUDIT_CODE,
'CHC(Quarterly)' DATA_SOURCE,
-------------------------------------
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
null as prescription,
null as prescription_ly,
-------------------------------------
--Retail藏数逻辑标签------------------
1 PACK_FLAG,
1 PROD_FLAG,
0 DTP_FLAG,
null as cmps_flag,
-------------------------------------
--EC数据标签--------------------------
'' PLATFORM_TYPE,
'' STORE_NAME,
'' STORE_TYPE,
-------------------------------------
--COUNTY数据标签----------------------
'' REGION_TYPE,
-------------------------------------
--AIA数据标签-------------------------
null inst_code, --可以移除，报告没有使用
-------------------------------------
--XIE HE 数据标签---------------------
'' as dept_name,
'' as new_code,
'' as area,
'' as h_level,
'' as reimburse,
'' as reimburse_type,
'' as prescription_source,
-------------------------------------
'' as CORP_CODE --可以移除，报告没有使用E
-------------------------------------
FROM
RAW_DATA_FINAL A
WHERE
YYYYMM
>= (
SELECT
CAST((LEFT(MAX(YYYYMM), 4) - 3) * 100 + 1 AS INT)
FROM
RAW_DATA_FINAL
)
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TF_EXT_CHC_SALES (
YYYYMM,
PACK_CODE,
AUDIT_CODE,
DATA_SOURCE,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
PRESCRIPTION,
PRESCRIPTION_LY,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
CMPS_FLAG,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
INST_CODE,
DEPT_NAME,
NEW_CODE,
AREA,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE,
CORP_CODE,
ETL_INSERT_DT,
ETL_UPDATE_DT
)
SELECT
YYYYMM,
NVL(NULLIF(PACK_CODE, ''), CONCAT('PACK_CODE_', DATA_SOURCE)) AS PACK_CODE,
NVL(NULLIF(AUDIT_CODE, ''), 'ROC') AS AUDIT_CODE,
DATA_SOURCE,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
PRESCRIPTION,
PRESCRIPTION_LY,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
CMPS_FLAG,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
INST_CODE,
DEPT_NAME,
NEW_CODE,
AREA,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE,
CORP_CODE,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
FROM
DM_TF_EXT_CHC_SALES;

147
CHC/03 DM_TD_EXT_CHC_PACK_PROPERTY.sql Normal file
View File

@@ -0,0 +1,147 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_CHC_PACK_PROPERTY
-- (
-- -- MARKET_PACK_KEY STRING,
-- PACK_CODE STRING,
-- PACK_DESC STRING,
-- STGH_DESC STRING,
-- PACK_LCH STRING,
-- FAMILY_CODE STRING,
-- FAMILY_NAME STRING,
-- PROD_CODE STRING,
-- PROD_DESC STRING,
-- PROD_DESC_C STRING,
-- CMPS_CODE STRING,
-- CMPS_DESC STRING,
-- CMPS_DESC_C STRING,
-- ATC1_CODE STRING,
-- ATC2_CODE STRING,
-- ATC3_CODE STRING,
-- ATC4_CODE STRING,
-- APP1_CODE STRING,
-- APP2_CODE STRING,
-- APP3_CODE STRING,
-- BIO_DESC STRING,
-- GENE_ORIG_DESC STRING,
-- ETH_OTC_DESC STRING,
-- NRDL_DESC STRING,
-- NRDL_ENTRY_DATE STRING,
-- EDL_DESC STRING,
-- TCM_DESC STRING,
-- PAED_DESC STRING,
-- GQCE_DESC STRING,
-- VBP_DESC_V STRING,
-- VBP_DESC STRING,
-- MANU_CODE STRING,
-- MANU_DESC STRING,
-- MANU_DESC_C STRING,
-- MNFL_CODE STRING,
-- MNFL_DESC STRING,
-- CORP_CODE STRING,
-- CORP_DESC STRING,
-- CORP_DESC_C STRING,
-- BRANDTYPE STRING,
-- -- MARKET STRING,
-- -- KEY_COMPETITOR STRING,
-- IS_AZ STRING,
-- AZ_MAIN STRING,
-- -- AZ_RELATED STRING,
-- ATC1_DESC STRING,
-- ATC1_DESC_C STRING,
-- ATC2_DESC STRING,
-- ATC2_DESC_C STRING,
-- ATC3_DESC STRING,
-- ATC3_DESC_C STRING,
-- ATC4_DESC STRING,
-- ATC4_DESC_C STRING,
-- APP1_DESC STRING,
-- APP1_DESC_C STRING,
-- APP2_DESC STRING,
-- APP2_DESC_C STRING,
-- APP3_DESC STRING,
-- APP3_DESC_C STRING,
-- -- CLASS STRING,
-- -- MARKET_RATIO STRING,
-- COUNTINGUNIT STRING,
-- VBP_BRAND STRING,
-- REPLENISH_FALG STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP
-- )
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack_property';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack_property';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_CHC_PACK_PROPERTY
SELECT
-- MARKET_PACK_KEY,
PACK_COD AS PACK_CODE,
PACK_DES AS PACK_DESC,
STGH_DES AS STGH_DESC,
PACK_LCH,
FAMILY_CODE,
FAMILY_NAME,
PROD_COD AS PROD_CODE,
PROD_DES AS PROD_DESC,
PROD_DES_C AS PROD_DESC_C,
CMPS_COD AS CMPS_CODE,
CMPS_DES AS CMPS_DESC,
CMPS_DES_C AS CMPS_DESC_C,
ATC1_COD AS ATC1_CODE,
ATC2_COD AS ATC2_CODE,
ATC3_COD AS ATC3_CODE,
ATC4_COD AS ATC4_CODE,
APP1_COD AS APP1_CODE,
APP2_COD AS APP2_CODE,
APP3_COD AS APP3_CODE,
BIO_DESC,
GENE_ORIG_DESC,
ETH_OTC_DESC,
NRDL_DESC,
NRDL_ENTRY_DATE,
EDL_DESC,
TCM_DESC,
PAED_DESC,
GQCE_DESC,
null as VBP_DESC_V,
VBP_DESC,
MANU_COD AS MANU_CODE,
MANU_DES AS MANU_DESC,
MANU_DES_C AS MANU_DESC_C,
MNFL_COD AS MNFL_CODE,
MNFL_DES AS MNFL_DESC,
CORP_COD AS CORP_CODE,
CORP_DES AS CORP_DESC,
CORP_DES_C AS CORP_DESC_C,
BRANDTYPE,
-- MARKET,
-- KEY_COMPETITOR,
IS_AZ,
AZ_MAIN,
-- AZ_RELATED,
ATC1_DES AS ATC1_DESC,
ATC1_DES_C AS ATC1_DESC_C,
ATC2_DES AS ATC2_DESC,
ATC2_DES_C AS ATC2_DESC_C,
ATC3_DES AS ATC3_DESC,
ATC3_DES_C AS ATC3_DESC_C,
ATC4_DES AS ATC4_DESC,
ATC4_DES_C AS ATC4_DESC_C,
APP1_DES AS APP1_DESC,
APP1_DES_C AS APP1_DESC_C,
APP2_DES AS APP2_DESC,
APP2_DES_C AS APP2_DESC_C,
APP3_DES AS APP3_DESC,
APP3_DES_C AS APP3_DESC_C,
-- CLASS,
-- NULL AS MARKET_RATIO,
COUNTINGUNIT,
NULL AS VBP_BRAND,
NULL AS REPLENISH_FALG,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_CHC_PACK_PROPERTY;

327
CHC/04 DM_TD_EXT_CHC_PACK2MARKET.sql Normal file
View File

@@ -0,0 +1,327 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_CHC_PACK2MARKET (
-- MARKET STRING,
-- PACK_CODE STRING,
-- PACK_DESC STRING,
-- STGH_DESC STRING,
-- PACK_LCH STRING,
-- PROD_CODE STRING,
-- CMPS_CODE STRING,
-- CMPS_DESC STRING,
-- ATC1_CODE STRING,
-- ATC2_CODE STRING,
-- ATC3_CODE STRING,
-- ATC4_CODE STRING,
-- APP1_CODE STRING,
-- APP2_CODE STRING,
-- APP3_CODE STRING,
-- BIO_DESC STRING,
-- GENE_ORIG_DESC STRING,
-- ETH_OTC_DESC STRING,
-- NRDL_DESC STRING,
-- NRDL_ENTRY_DATE STRING,
-- EDL_DESC STRING,
-- TCM_DESC STRING,
-- PAED_DESC STRING,
-- GQCE_DESC STRING,
-- VBP_DESC STRING,
-- MANU_CODE STRING,
-- MANU_DESC STRING,
-- MNFL_CODE STRING,
-- MNFL_DESC STRING,
-- CORP_CODE STRING,
-- CORP_DESC STRING,
-- BRANDTYPE STRING,
-- BU STRING,
-- STARTTIME STRING,
-- ENDTIME STRING,
-- MARKET_RATIO STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP
-- )
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market';
-- COMMAND ----------
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_chc_PACK2MARKET_KEYCOMPETITOR (
-- KEY_COMPETITOR STRING,
-- MARKET STRING,
-- PACK_CODE STRING)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market_keycompetitor';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market_keycompetitor';
-- COMMAND ----------
-- CREATE or REPLACE TABLE DM.DM_TD_EXT_chc_PACK2MARKET_CLASS (
-- MARKET STRING,
-- PACK_CODE STRING,
-- CLASS STRING)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market_class';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_pack2market_class';
-- COMMAND ----------
-- 使用 EXCEPT（集合差操作）简化逻辑
INSERT OVERWRITE TABLE DM.DM_TD_EXT_CHC_PACK2MARKET
-- 包含规则
SELECT DISTINCT
T2.MARKET,
T1.PACK_CODE,
T1.PACK_DESC,
T1.STGH_DESC,
T1.PACK_LCH,
T1.PROD_CODE,
T1.CMPS_CODE,
T1.CMPS_DESC,
T1.ATC1_CODE,
T1.ATC2_CODE,
T1.ATC3_CODE,
T1.ATC4_CODE,
T1.APP1_CODE,
T1.APP2_CODE,
T1.APP3_CODE,
T1.BIO_DESC,
T1.GENE_ORIG_DESC AS GENE_ORIG,
T1.ETH_OTC_DESC AS ETH_OTC_D,
T1.NRDL_DESC,
T1.NRDL_ENTRY_DATE AS NRDL_ENTR,
T1.EDL_DESC,
T1.TCM_DESC,
T1.PAED_DESC,
T1.GQCE_DESC,
T1.VBP_DESC,
T1.MANU_CODE,
T1.MANU_DESC,
T1.MNFL_CODE,
T1.MNFL_DESC,
T1.CORP_CODE,
T1.CORP_DESC,
T1.BRANDTYPE,
T2.BU,
T2.STARTTIME,
T2.ENDTIME,
COALESCE(T2.EXTEND_MARKET_RATIO, '1') AS MARKET_RATIO,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_CHC_PACK_PROPERTY AS T1
LEFT JOIN (
SELECT
MARKET,
BU,
STARTTIME,
ENDTIME,
EXTEND_MARKET_RATIO,
ATC1_CODE,
ATC2_CODE,
ATC3_CODE,
ATC4_CODE,
NFC1_CODE,
NFC2_CODE,
NFC3_CODE,
CORPORATION_CODE,
MANUFACTURER_CODE,
PRODUCT_CODE,
PACK_CODE,
STRENGTH,
MOLECULE_CODE
FROM DWD.DWD_GND_IMS_TBLMARKET_BYMONTH
WHERE EXTEND_MARKET IS NULL AND (NOT_IN_FLAG IS NULL OR NOT_IN_FLAG = '1')
) AS T2
ON (T2.ATC1_CODE IS NULL OR T1.ATC1_CODE = T2.ATC1_CODE)
AND (T2.ATC2_CODE IS NULL OR T1.ATC2_CODE = T2.ATC2_CODE)
AND (T2.ATC3_CODE IS NULL OR T1.ATC3_CODE = T2.ATC3_CODE)
AND (T2.ATC4_CODE IS NULL OR T1.ATC4_CODE = T2.ATC4_CODE)
AND (T2.NFC1_CODE IS NULL OR T1.APP1_CODE = T2.NFC1_CODE)
AND (T2.NFC2_CODE IS NULL OR T1.APP2_CODE = T2.NFC2_CODE)
AND (T2.NFC3_CODE IS NULL OR T1.APP3_CODE = T2.NFC3_CODE)
AND (T2.CORPORATION_CODE IS NULL OR T1.CORP_CODE = T2.CORPORATION_CODE)
AND (T2.MANUFACTURER_CODE IS NULL OR T1.MANU_CODE = T2.MANUFACTURER_CODE)
AND (T2.PRODUCT_CODE IS NULL OR T1.PROD_CODE = T2.PRODUCT_CODE)
AND (T2.PACK_CODE IS NULL OR T1.PACK_CODE = T2.PACK_CODE)
AND (T2.STRENGTH IS NULL OR T1.STGH_DESC = T2.STRENGTH)
AND (T2.MOLECULE_CODE IS NULL OR T1.CMPS_CODE = T2.MOLECULE_CODE)
WHERE T2.MARKET IS NOT NULL
EXCEPT
-- 排除规则
SELECT DISTINCT
T2.MARKET,
T1.PACK_CODE,
T1.PACK_DESC,
T1.STGH_DESC,
T1.PACK_LCH,
T1.PROD_CODE,
T1.CMPS_CODE,
T1.CMPS_DESC,
T1.ATC1_CODE,
T1.ATC2_CODE,
T1.ATC3_CODE,
T1.ATC4_CODE,
T1.APP1_CODE,
T1.APP2_CODE,
T1.APP3_CODE,
T1.BIO_DESC,
T1.GENE_ORIG_DESC AS GENE_ORIG,
T1.ETH_OTC_DESC AS ETH_OTC_D,
T1.NRDL_DESC,
T1.NRDL_ENTRY_DATE AS NRDL_ENTR,
T1.EDL_DESC,
T1.TCM_DESC,
T1.PAED_DESC,
T1.GQCE_DESC,
T1.VBP_DESC,
T1.MANU_CODE,
T1.MANU_DESC,
T1.MNFL_CODE,
T1.MNFL_DESC,
T1.CORP_CODE,
T1.CORP_DESC,
T1.BRANDTYPE,
T2.BU,
T2.STARTTIME,
T2.ENDTIME,
COALESCE(T2.EXTEND_MARKET_RATIO, '1') AS MARKET_RATIO,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_CHC_PACK_PROPERTY AS T1
LEFT JOIN (
SELECT
MARKET,
BU,
STARTTIME,
ENDTIME,
EXTEND_MARKET_RATIO,
ATC1_CODE,
ATC2_CODE,
ATC3_CODE,
ATC4_CODE,
NFC1_CODE,
NFC2_CODE,
NFC3_CODE,
CORPORATION_CODE,
MANUFACTURER_CODE,
PRODUCT_CODE,
PACK_CODE,
STRENGTH,
MOLECULE_CODE
FROM DWD.DWD_GND_IMS_TBLMARKET_BYMONTH
WHERE EXTEND_MARKET IS NULL AND NOT_IN_FLAG = '0'
) AS T2
ON (T2.ATC1_CODE IS NULL OR T1.ATC1_CODE = T2.ATC1_CODE)
AND (T2.ATC2_CODE IS NULL OR T1.ATC2_CODE = T2.ATC2_CODE)
AND (T2.ATC3_CODE IS NULL OR T1.ATC3_CODE = T2.ATC3_CODE)
AND (T2.ATC4_CODE IS NULL OR T1.ATC4_CODE = T2.ATC4_CODE)
AND (T2.NFC1_CODE IS NULL OR T1.APP1_CODE = T2.NFC1_CODE)
AND (T2.NFC2_CODE IS NULL OR T1.APP2_CODE = T2.NFC2_CODE)
AND (T2.NFC3_CODE IS NULL OR T1.APP3_CODE = T2.NFC3_CODE)
AND (T2.CORPORATION_CODE IS NULL OR T1.CORP_CODE = T2.CORPORATION_CODE)
AND (T2.MANUFACTURER_CODE IS NULL OR T1.MANU_CODE = T2.MANUFACTURER_CODE)
AND (T2.PRODUCT_CODE IS NULL OR T1.PROD_CODE = T2.PRODUCT_CODE)
AND (T2.PACK_CODE IS NULL OR T1.PACK_CODE = T2.PACK_CODE)
AND (T2.STRENGTH IS NULL OR T1.STGH_DESC = T2.STRENGTH)
AND (T2.MOLECULE_CODE IS NULL OR T1.CMPS_CODE = T2.MOLECULE_CODE)
WHERE T2.MARKET IS NOT NULL;
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_CHC_PACK2MARKET_KEYCOMPETITOR
WITH kc AS (
SELECT
keycompetitor, no,
ROW_NUMBER() OVER (ORDER BY
CASE
WHEN ATC1_Code IS NOT NULL THEN 1
WHEN ATC2_Code IS NOT NULL THEN 2
WHEN ATC3_Code IS NOT NULL THEN 3
WHEN ATC4_Code IS NOT NULL THEN 4
WHEN Molecule_Code IS NOT NULL THEN 5
WHEN Product_Code IS NOT NULL THEN 6
WHEN Pack_Code IS NOT NULL THEN 7
ELSE 999
END,
CASE WHEN NFC1_CODE IS NULL THEN 0 ELSE 1 END,
CASE WHEN NFC2_CODE IS NULL THEN 0 ELSE 1 END,
CASE WHEN NFC3_CODE IS NULL THEN 0 ELSE 1 END
) AS no1,
ATC1_Code, ATC2_Code, ATC3_Code, ATC4_Code,
NFC1_Code, NFC2_Code, NFC3_Code,
corporation_code, Manufacturer_Code,
Product_Code, Pack_Code, Strength, Molecule_Code, market
FROM dwd.dwd_gnd_ims_tblkeycompetitor
),
joined AS (
SELECT
/*+ BROADCAST(kc) */
COALESCE(kc.keycompetitor, 'OTHERS') AS Key_Competitor, -- ✅ UPDATE 逻辑内联
t1.market, t1.PACK_CODE
/*bug 调试区
,
kc.no,
kc.no1,
t1.PROD_COD, t1.CMPS_COD, t1.CMPS_DES,
t1.ATC1_COD, t1.ATC2_COD, t1.ATC3_COD, t1.ATC4_COD,
t1.APP1_COD, t1.APP2_COD, t1.APP3_COD,
t1.BIO_DESC, t1.GENE_ORIG_DESC, t1.ETH_OTC_DESC,
t1.NRDL_DESC, t1.NRDL_Entry_Date, t1.EDL_DESC, t1.TCM_DESC,
t1.PAED_DESC, t1.GQCE_DESC, t1.VBP_DESC,
t1.MANU_COD, t1.MANU_DES, t1.MNFL_COD, t1.MNFL_DES,
t1.CORP_COD, t1.CORP_DES, t1.BrandType, t1.bu, t1.Market_Ratio
*/
,ROW_NUMBER() OVER (
PARTITION BY t1.market, t1.PACK_CODE, t1.PROD_CODE
ORDER BY kc.no1 DESC, kc.no DESC
) AS _rn
FROM DM.DM_TD_EXT_CHC_PACK2MARKET t1
LEFT JOIN kc
ON (kc.ATC1_Code IS NULL OR t1.ATC1_CODE = kc.ATC1_Code)
AND (kc.ATC2_Code IS NULL OR t1.ATC2_CODE = kc.ATC2_Code)
AND (kc.ATC3_Code IS NULL OR t1.ATC3_CODE = kc.ATC3_Code)
AND (kc.ATC4_Code IS NULL OR t1.ATC4_CODE = kc.ATC4_Code)
AND (kc.NFC1_Code IS NULL OR t1.APP1_CODE = kc.NFC1_Code)
AND (kc.NFC2_Code IS NULL OR t1.APP2_CODE = kc.NFC2_Code)
AND (kc.NFC3_Code IS NULL OR t1.APP3_CODE = kc.NFC3_Code)
AND (kc.corporation_code IS NULL OR t1.CORP_CODE = kc.corporation_code)
AND (kc.Manufacturer_Code IS NULL OR t1.MANU_CODE = kc.Manufacturer_Code)
AND (kc.Product_Code IS NULL OR t1.PROD_CODE = kc.Product_Code)
AND (kc.Pack_Code IS NULL OR t1.PACK_CODE = kc.Pack_Code)
AND (kc.Strength IS NULL OR t1.STGH_DESC = kc.Strength)
AND (kc.Molecule_Code IS NULL OR t1.CMPS_CODE = kc.Molecule_Code)
AND (kc.market IS NULL OR t1.market = kc.market)
)
SELECT distinct Key_Competitor,market,pack_code
FROM joined
WHERE _rn = 1;
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_CHC_PACK2MARKET_CLASS
SELECT
/*+ BROADCAST(T2) */
DISTINCT
T1.MARKET,
T1.PACK_CODE,
COALESCE( T2.CLASS, 'OTHERS') AS CLASS
FROM DM.DM_TD_EXT_CHC_PACK2MARKET T1
LEFT JOIN DWD.DWD_GND_TBLCLASS T2
ON (T2.MARKET IS NULL OR T1.MARKET = T2.MARKET)
AND (T2.PACK_CODE IS NULL OR T1.PACK_CODE = T2.PACK_CODE)
AND (T2.PRODUCT_CODE IS NULL OR T1.PROD_CODE = T2.PRODUCT_CODE)
AND (T2.MOLECULE_CODE IS NULL OR T1.CMPS_CODE = T2.MOLECULE_CODE)
AND (T2.CORPORATION_CODE IS NULL OR T1.CORP_CODE = T2.CORPORATION_CODE)
AND (T2.MANUFACTURER_CODE IS NULL OR T1.MANU_CODE = T2.MANUFACTURER_CODE)
AND (T2.ATC1_CODE IS NULL OR T1.ATC1_CODE = T2.ATC1_CODE)
AND (T2.ATC2_CODE IS NULL OR T1.ATC2_CODE = T2.ATC2_CODE)
AND (T2.ATC3_CODE IS NULL OR T1.ATC3_CODE = T2.ATC3_CODE)
AND (T2.ATC4_CODE IS NULL OR T1.ATC4_CODE = T2.ATC4_CODE)
AND (T2.NFC1_CODE IS NULL OR T1.APP1_CODE = T2.NFC1_CODE)
AND (T2.NFC2_CODE IS NULL OR T1.APP2_CODE = T2.NFC2_CODE)
AND (T2.NFC3_CODE IS NULL OR T1.APP3_CODE = T2.NFC3_CODE)
AND (T2.STRENGTH IS NULL OR T1.STGH_DESC = T2.STRENGTH)

30
CHC/05 DM_TD_EXT_CHC_MARKET_RATIO.sql Normal file
View File

@@ -0,0 +1,30 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_CHC_MARKET_RATIO (
-- MARKET STRING,
-- PACK_CODE STRING,
-- CMPS_CODE STRING,
-- STARTTIME STRING,
-- ENDTIME STRING,
-- MARKET_RATIO STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_market_ratio';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_market_ratio';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_CHC_MARKET_RATIO
SELECT
DISTINCT
MARKET,
PACK_CODE,
CMPS_CODE,
STARTTIME,
ENDTIME,
MARKET_RATIO,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_CHC_PACK2MARKET
WHERE UPPER(MARKET) NOT LIKE '%ALL%'

163
CHC/06 DM_TD_EXT_CHC_MARKET_BRAND_RATIO.sql Normal file
View File

@@ -0,0 +1,163 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_CHC_MARKET_BRAND_RATIO (
-- MARKET STRING,
-- PACK_CODE STRING,
-- VALUE_BRAND_RATIO FLOAT,
-- VALUE_BRAND_RATIO_START STRING,
-- VALUE_BRAND_RATIO_END STRING,
-- UNIT_BRAND_RATIO FLOAT,
-- UNIT_BRAND_START STRING,
-- UNIT_BRAND_END STRING,
-- COUNTINGUNIT_BRAND_RATIO FLOAT,
-- COUNTINGUNIT_BRAND_START STRING,
-- COUNTINGUNIT_BRAND_END STRING,
-- PDOT_BRAND_RATIO FLOAT,
-- PDOT_BRAND_START STRING,
-- PDOT_BRAND_END STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_market_brand_ratio';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_market_brand_ratio';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_CHC_MARKET_BRAND_RATIO
SELECT
DISTINCT
T1.MARKET,
T1.PACK_CODE,
CASE
WHEN T2.RATIO IS NOT NULL THEN T2.RATIO
ELSE NVL(T5.RATIO, 1) END AS VALUE_BRAND_RATIO,
CASE
WHEN T2.RATIO IS NOT NULL THEN T2.STARTTIME
ELSE NVL(T5.STARTTIME, 200001) END AS VALUE_BRAND_RATIO_START,
CASE
WHEN T2.RATIO IS NOT NULL THEN T2.ENDTIME
ELSE NVL(T5.ENDTIME, 299912) END AS VALUE_BRAND_RATIO_END,
CASE
WHEN T3.RATIO IS NOT NULL THEN T3.RATIO
ELSE NVL(T6.RATIO, 1) END AS UNIT_BRAND_RATIO,
CASE
WHEN T3.RATIO IS NOT NULL THEN T3.STARTTIME
ELSE NVL(T6.STARTTIME, 200001) END AS UNIT_BRAND_START,
CASE
WHEN T3.RATIO IS NOT NULL THEN T3.ENDTIME
ELSE NVL(T6.ENDTIME, 299912) END AS UNIT_BRAND_END,
1 AS COUNTINGUNIT_BRAND_RATIO,
200001 AS COUNTINGUNIT_BRAND_START,
299912 AS COUNTINGUNIT_BRAND_END,
CASE
WHEN T4.RATIO IS NOT NULL THEN T4.RATIO
ELSE NVL(T7.RATIO, 1) END AS PDOT_BRAND_RATIO,
CASE
WHEN T4.RATIO IS NOT NULL THEN T4.STARTTIME
ELSE NVL(T7.STARTTIME, 200001) END AS PDOT_BRAND_START,
CASE
WHEN T4.RATIO IS NOT NULL THEN T4.ENDTIME
ELSE NVL(T7.ENDTIME, 299912) END AS PDOT_BRAND_END,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_CHC_MARKET_RATIO T1
LEFT JOIN ( --VALUE--PACK
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'VALUE' AND UPPER(LEVEL) = 'PACK'
) T2 ON T1.MARKET = T2.MARKET AND T1.PACK_CODE = T2.PACK_COD
LEFT JOIN ( --UNIT--PACK
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'UNIT' AND UPPER(LEVEL) = 'PACK'
) T3 ON T1.MARKET = T3.MARKET AND T1.PACK_CODE = T3.PACK_COD
LEFT JOIN ( --COUNTINGUNIT--PACK
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'COUNTINGUNIT' AND UPPER(LEVEL) = 'PACK'
) T4 ON T1.MARKET = T4.MARKET AND T1.PACK_CODE = T4.PACK_COD
LEFT JOIN ( --VALUE--MOLECULE
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'VALUE' AND UPPER(LEVEL) = 'MOLECULE'
) T5 ON T1.MARKET = T5.MARKET AND T1.CMPS_CODE = T5.CMPS_COD
LEFT JOIN ( --UNIT--MOLECULE
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'UNIT' AND UPPER(LEVEL) = 'MOLECULE'
) T6 ON T1.MARKET = T6.MARKET AND T1.CMPS_CODE = T6.CMPS_COD
LEFT JOIN ( --COUNTINGUNIT--MOLECULE
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'COUNTINGUNIT' AND UPPER(LEVEL) = 'MOLECULE'
) T7 ON T1.MARKET = T7.MARKET
AND T1.CMPS_CODE = T7.CMPS_COD
-- COMMAND ----------

71
CHC/07 DM_TD_EXT_CHC_MARKET_PACK_MAPPING.sql Normal file
View File

@@ -0,0 +1,71 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_CHC_MARKET_PACK_MAPPING
-- (
-- PACK_CODE STRING,
-- MARKET STRING,
-- CLASS STRING,
-- KEY_COMPETITOR STRING,
-- MARKET_RATIO STRING,
-- STARTTIME STRING,
-- ENDTIME STRING,
-- VALUE_BRAND_RATIO STRING,
-- VALUE_BRAND_RATIO_START STRING,
-- VALUE_BRAND_RATIO_END STRING,
-- UNIT_BRAND_RATIO STRING,
-- UNIT_BRAND_START STRING,
-- UNIT_BRAND_END STRING,
-- COUNTINGUNIT_BRAND_RATIO STRING,
-- COUNTINGUNIT_BRAND_START STRING,
-- COUNTINGUNIT_BRAND_END STRING,
-- PDOT_BRAND_RATIO STRING,
-- PDOT_BRAND_START STRING,
-- PDOT_BRAND_END STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP
-- )
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_market_pack_mapping';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_market_pack_mapping';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_CHC_MARKET_PACK_MAPPING
SELECT DISTINCT
T1.PACK_CODE,
T1.MARKET,
T4.CLASS,
T5.KEY_COMPETITOR,
T2.MARKET_RATIO,
T2.STARTTIME,
T2.ENDTIME,
T3.VALUE_BRAND_RATIO,
T3.VALUE_BRAND_RATIO_START,
T3.VALUE_BRAND_RATIO_END,
T3.UNIT_BRAND_RATIO,
T3.UNIT_BRAND_START,
T3.UNIT_BRAND_END,
T3.COUNTINGUNIT_BRAND_RATIO,
T3.COUNTINGUNIT_BRAND_START,
T3.COUNTINGUNIT_BRAND_END,
T3.PDOT_BRAND_RATIO,
T3.PDOT_BRAND_START,
T3.PDOT_BRAND_END,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
FROM
DM.DM_TD_EXT_CHC_PACK2MARKET T1
LEFT JOIN DM.DM_TD_EXT_CHC_MARKET_RATIO T2
ON T1.MARKET = T2.MARKET
AND T1.PACK_CODE = T2.PACK_CODE
LEFT JOIN DM.DM_TD_EXT_CHC_MARKET_BRAND_RATIO T3
ON T1.MARKET = T3.MARKET
AND T1.PACK_CODE = T3.PACK_CODE
LEFT JOIN DM.DM_TD_EXT_CHC_PACK2MARKET_CLASS T4
ON T1.PACK_CODE = T4.PACK_CODE
AND T1.MARKET = T4.MARKET
LEFT JOIN DM.DM_TD_EXT_CHC_PACK2MARKET_KEYCOMPETITOR T5
ON T1.PACK_CODE = T5.PACK_CODE
AND T1.MARKET = T5.MARKET
WHERE
UPPER(T1.MARKET) NOT LIKE "%ALL%"

42
CHPA/01 dwd_ims_atc_hierarchy.sql Normal file
View File

@@ -0,0 +1,42 @@
-- Databricks notebook source
--dwd to dws
--清洗数据的操作
---处理宽维度表 202108
--PRINT(GETDATE())
---ATC 处理成宽表
insert overwrite table dwd.dwd_ims_atc_hierarchy
SELECT * --into DIM_ATC_Hierarchy
FROM (SELECT Therapeutic_ID AS ATC1_ID,
Therapeutic_Code AS ATC1_CODE,
Therapeutic_Name AS ATC1_DES
FROM dwd.dwd_ims_td_therapeutic_class ATC
WHERE ATC. Therapeutic_Level = '1') ATC1
LEFT JOIN (SELECT Therapeutic_ID AS ATC2_ID,
Therapeutic_Code AS ATC2_CODE,
Therapeutic_Name AS ATC2_DES
FROM dwd.dwd_ims_td_therapeutic_class ATC
WHERE ATC. Therapeutic_Level = '2') ATC2
ON ATC1. ATC1_CODE =
LEFT (ATC2. ATC2_CODE, 1)
LEFT JOIN (SELECT Therapeutic_ID AS ATC3_ID,
Therapeutic_Code AS ATC3_CODE,
Therapeutic_Name AS ATC3_DES
FROM dwd.dwd_ims_td_therapeutic_class ATC
WHERE ATC. Therapeutic_Level = '3') ATC3
ON ATC2. ATC2_CODE =
LEFT (ATC3. ATC3_CODE, 3)
LEFT JOIN (SELECT Therapeutic_ID AS ATC4_ID,
Therapeutic_Code AS ATC4_CODE,
Therapeutic_Name AS ATC4_DES
FROM dwd.dwd_ims_td_therapeutic_class ATC
WHERE ATC. Therapeutic_Level = '4') ATC4
ON ATC3. ATC3_CODE =
LEFT (ATC4. ATC4_CODE, 4)
;

26
CHPA/01 dwd_ims_nfc_hierarchy.sql Normal file
View File

@@ -0,0 +1,26 @@
-- Databricks notebook source
--NFC 处理成宽表
insert overwrite table dwd.dwd_ims_nfc_hierarchy
SELECT *
FROM (SELECT NewFormClass_ID as NFC1_ID,
NewFormClass_Code as NFC1_CODE,
NewFormClass_Name as NFC1_DES
FROM dwd.dwd_ims_td_new_form_class NFC
WHERE NewFormClass_Level = '1') NFC1
LEFT JOIN (SELECT NewFormClass_ID as NFC2_ID,
NewFormClass_Code as NFC2_CODE,
NewFormClass_Name as NFC2_DES
FROM dwd.dwd_ims_td_new_form_class NFC
WHERE NewFormClass_Level = '2') NFC2
ON NFC1.NFC1_CODE =
LEFT (NFC2.NFC2_CODE, 1)
LEFT JOIN (SELECT NewFormClass_ID as NFC3_ID,
NewFormClass_Code as NFC3_CODE,
NewFormClass_Name as NFC3_DES
FROM dwd.dwd_ims_td_new_form_class NFC
WHERE NewFormClass_Level = '3') NFC3
ON NFC2.NFC2_CODE =
LEFT (NFC3.NFC3_CODE, 2)
;

24
CHPA/01 dwd_ims_td_manufacturer_corp.sql Normal file
View File

@@ -0,0 +1,24 @@
-- Databricks notebook source
-----------------------------------------------------------------------------------------------------------------
--修改时间：20240904
--修改人：FanXujia
--修改内容：
--id为93的数，manufacturer_abbr不是SH/了，所以不能使用这个语句更新了。
------------------------------------------------------------------------
--MANU维度表，ManufacturerType_ID中存在为0的数据，关联不出对应的ManufacturerType
--经过检查，ManufacturerType_ID = 0 的数据，其下属子集或者本身为MNC，所以将其修改成2，让报告上能正常识别成MNC
UPDATE dwd.dwd_ims_td_manufacturer SET ManufacturerType_ID = 2 WHERE ManufacturerType_ID = 0;
-- SH/的一行数据 中 / 为转义符，后面的数据不能有效识别，需要手动更新
-- update dwd.dwd_ims_td_manufacturer set manufacturer_abbr ='SH/' where manufacturer_id = '93';
-- update dwd.dwd_ims_td_manufacturer set corporation_id ='93' where manufacturer_id = '93';
-- update dwd.dwd_ims_td_manufacturer set corporation_code ='00221' where manufacturer_id = '93';
-- update dwd.dwd_ims_td_manufacturer set manufacturertype_id ='1' where manufacturer_id = '93';
-----------------------------------------------------------------------------------------------------------------
--处理manu跟corp的mapping关系
insert overwrite table dwd.dwd_ims_td_manufacturer_corp
SELECT T1.*,T2.Corporation_Code AS CORP_MASTER_CODE,T2.Manufacturer_Abbr AS CORP_ABBR,T2.Manufacturer_Name AS CORP_DES
FROM dwd.dwd_ims_td_manufacturer T1
LEFT JOIN dwd.dwd_ims_td_manufacturer T2
ON T1.Corporation_Code = T2.Manufacturer_CODE
;

89
CHPA/01 dwd_ims_td_pack_property.sql Normal file
View File

@@ -0,0 +1,89 @@
-- Databricks notebook source
-------使用新的DIM表，生成PackProfile，保证后续的逻辑
-- 20260119 依赖前面 dwd.dwd_ims_nfc_hierarchy、dwd.dwd_ims_atc_hierarchy、dwd.dwd_ims_td_manufacturer_corp
-- 20260119 依赖上面补0 dwd.dwd_gnd_ims_tblbrandtype
insert overwrite table dwd.dwd_ims_td_pack_property
select DISTINCT
if( PACK.Pack_Code REGEXP '^[0-9]',right(concat('000000000000',PACK.Pack_Code),12),PACK.Pack_Code) as PACK_COD, --RIGHT(concat('0000000',PACK.Pack_Code),7)
PACK.Pack_Description AS PACK_DES,
PACK.STRENGTH AS STGH_DES,
concat('Y',LEFT(PACK.LAUNCHTIME,4),'M',RIGHT(PACK.LAUNCHTIME,2)) AS PACK_LCH,
RIGHT(concat('000000000',PROD.Product_Code),9) AS PROD_COD, --RIGHT(concat('00000',PROD.Product_Code),5)
RIGHT(concat('000000',MOLE.MoleCompCode),6) AS CMPS_COD,
MOLE.MoleCompDesc AS CMPS_DES,
ATCH.ATC1_CODE AS ATC1_COD,
ATCH.ATC2_CODE AS ATC2_COD,
ATCH.ATC3_CODE AS ATC3_COD,
ATCH.ATC4_CODE AS ATC4_COD,
NFCH.NFC1_CODE AS APP1_COD,
NFCH.NFC2_CODE AS APP2_COD,
NFCH.NFC3_CODE AS APP3_COD,
MOLE.BIO AS BIO_DESC,
MOLE.Gene_Orig AS GENE_ORIG_DESC,
MOLE.Rx_Flag AS ETH_OTC_DESC,
MOLE.NRDL AS NRDL_DESC,
MOLE.NRDL_Entry_Date,
MOLE.EDL AS EDL_DESC,
MOLE.TCMEX AS TCM_DESC,
MOLE.PAED AS PAED_DESC,
MOLE.GQCE AS GQCE_DESC,
MOLE.VBP AS VBP_DESC,
MANU.Manufacturer_Abbr AS MANU_COD,
MANU.Manufacturer_Name AS MANU_DES,
MANUT.ManufacturerType_CODE AS MNFL_COD,
MANUT.ManufacturerType_Name AS MNFL_DES,
MANU.CORP_ABBR AS CORP_COD,
MANU.CORP_DES AS CORP_DES,
BRANDTYPE.Brand_Type AS BrandType
from dwd.dwd_ims_td_pack PACK
LEFT JOIN dwd.dwd_ims_td_product PROD ON PACK.Product_ID = PROD.Product_ID
LEFT JOIN dwd.dwd_ims_td_new_form_class NFC ON PACK.NewFormClass_ID = NFC.NewFormClass_ID
LEFT JOIN dwd.dwd_ims_nfc_hierarchy NFCH ON NFC.NewFormClass_Code = NFCH.NFC3_CODE
LEFT JOIN dwd.dwd_ims_td_therapeutic_class ATC ON PACK.Therapeutic_ID = ATC.Therapeutic_ID
LEFT JOIN dwd.dwd_ims_atc_hierarchy ATCH ON ATC.Therapeutic_Code = ATCH.ATC4_CODE
LEFT JOIN dwd.dwd_ims_td_pack_additional_attribute MOLE ON PACK.Pack_ID = MOLE.Pack_ID
LEFT JOIN dwd.dwd_ims_td_manufacturer_corp MANU ON PROD.Manufacturer_ID = MANU.Manufacturer_ID
LEFT JOIN dwd.dwd_ims_td_manufacturertype MANUT ON MANU.ManufacturerType_ID = MANUT.ManufacturerType_ID
LEFT JOIN dwd.dwd_gnd_ims_tblbrandtype BRANDTYPE ON PACK.Pack_Code = BRANDTYPE.PACK_COD
;
update dwd.dwd_ims_td_pack_property set CORP_COD ='' where CORP_COD is null ;
update dwd.dwd_ims_td_pack_property set STGH_DES ='' where STGH_DES is null ;
-- COMMAND ----------
-- 20260119 修改 dwd.dwd_ims_td_pack_property 我感觉可以拉到前面去
-- 要调研下，为什么前面没有写，而在后面写。
--Update AZ BRANDS SCOPE---MANU的页面显示
---by Product&Pack add in 'Fluimucil, Broncho-vaxom, Ai Rui Zhuo', by AC 202103
update dwd.dwd_ims_td_pack_property
set CORP_COD = 'A5Z', CORP_DES = 'ASTRAZENECA GROUP'
where PROD_COD in(
select
right(concat('0000000000',PROD_COD),9) as PROD_COD --right(concat('000000',PROD_COD),5)
from dwd.dwd_gnd_tbl_corp_change where corp_cod='A5Z')
or PACK_COD in(
select
if( PACK_COD REGEXP '^[0-9]',right(concat('000000000000',PACK_COD),12),PACK_COD) as PACK_COD --right(concat('000000',PACK_COD),7)
from dwd.dwd_gnd_tbl_corp_change where corp_cod='A5Z')
-- where PROD_COD in ('97336','73148') OR PACK_COD = '0795918'
;
---by Product&Pack remove out tblAZScope.xlsx', by AC 202103
update dwd.dwd_ims_td_pack_property
set CORP_COD = 'A5ZD', CORP_DES = 'AZDealed',MANU_COD = 'A5ZD' ,MANU_DES = 'AZDealed'
where PROD_COD in(
select
right(concat('0000000000',PROD_COD),9) as PROD_COD
from dwd.dwd_gnd_tbl_corp_change where corp_cod='A5ZD')
or PACK_COD in(
select
if( PACK_COD REGEXP '^[0-9]',right(concat('000000000000',PACK_COD),12),PACK_COD) as PACK_COD
from dwd.dwd_gnd_tbl_corp_change where corp_cod='A5ZD')
-- where PROD_COD in ('02267','14691','58423','12754','06482','97266','14164')
;

32
CHPA/01 dwd_update.sql Normal file
View File

@@ -0,0 +1,32 @@
-- Databricks notebook source
--对折算规则文件中的Pack，Mole code 补0
--时间条件为空的，赋默认值
--地理维度表增加KCT和ROC
UPDATE dwd.dwd_gnd_ims_tblBrandRatio
SET
PACK_COD = if( CAST(PACK_COD AS string) REGEXP '^[0-9]',right(concat('000000000000',CAST(PACK_COD AS string)),12),CAST(PACK_COD AS string)),
CMPS_COD = RIGHT(concat('000000',CAST(CMPS_COD AS string)),6),
StartTime = CASE WHEN StartTime IS NULL OR UPPER(StartTime)= 'ALL' THEN '190001' ELSE StartTime END,
EndTime = CASE WHEN EndTime IS NULL OR UPPER(EndTime)= 'ALL' THEN '209901' ELSE EndTime END
;
-- COMMAND ----------
-- 20260119 这个补0操作，可以放最前面，或者专门找个地方放。
----确保Market和KeyCompetitor配置表中，Pack_Code，Product_Code，Molecule_Code 三个字段都已经补0
UPDATE dwd.dwd_gnd_ims_tblmarket
--20250428 chenwu iqivacode扩到12位,brandcode 扩到9位
SET Pack_Code = if( Pack_Code REGEXP '^[0-9]',right(concat('000000000000',Pack_Code),12),Pack_Code)
,Product_Code = RIGHT(concat('000000000',Product_Code),9) --RIGHT(concat('00000',Product_Code),5)
,Molecule_Code = RIGHT(concat('000000',Molecule_Code),6)
;
UPDATE dwd.dwd_gnd_ims_tblkeycompetitor
SET Pack_Code = if( Pack_Code REGEXP '^[0-9]',right(concat('000000000000',Pack_Code),12),Pack_Code)
,Product_Code = RIGHT(concat('000000000',Product_Code),9) --RIGHT(concat('00000',Product_Code),5)
,Molecule_Code = RIGHT(concat('000000',Molecule_Code),6)
;
UPDATE dwd.dwd_gnd_ims_tblbrandtype
SET PACK_COD = if( PACK_COD REGEXP '^[0-9]',right(concat('000000000000',PACK_COD),12),PACK_COD) --RIGHT(concat('0000000',PACK_COD),7)
;

210
CHPA/01_FB_BLOB_TO_DWD.sql Normal file
View File

@@ -0,0 +1,210 @@
-- Databricks notebook source
-- MAGIC %md
-- MAGIC ### 从blob读取csv文件作为chap法伯的事实表
-- COMMAND ----------
-- 源表：文件
-- 目标表：DWD.DWD_GND_PHARBERS_PROV_FACT
-- COMMAND ----------
-- MAGIC %run ../../../Common/config
-- COMMAND ----------
-- MAGIC %python
-- MAGIC from datetime import datetime, timedelta
-- MAGIC import pandas as pd
-- COMMAND ----------
-- MAGIC %python
-- MAGIC if ENVIRONMENT == PRD_ENVIRONMENT_VALUE:
-- MAGIC factsales_file_path_template = "abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/ODS/GND/UserUpload/"
-- MAGIC elif ENVIRONMENT == TEST_ENVIRONMENT_VALUE:
-- MAGIC factsales_file_path_template = "abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/ODS/GND/UserUpload/"
-- COMMAND ----------
-- MAGIC %python
-- MAGIC # 计算时间得到当天的路径
-- MAGIC current_date = datetime.utcnow() + timedelta(hours=8)
-- MAGIC date_path = current_date.strftime("%Y/%m/%d/")
-- MAGIC base_path = factsales_file_path_template + date_path
-- COMMAND ----------
-- MAGIC %python
-- MAGIC # 路径是否存在
-- MAGIC def path_exists(path):
-- MAGIC try:
-- MAGIC dbutils.fs.ls(path)
-- MAGIC return True
-- MAGIC except Exception as e:
-- MAGIC if "java.io.FileNotFoundException" in str(e):
-- MAGIC return False
-- MAGIC else:
-- MAGIC print(f"检查路径 {path} 时出错: {e}")
-- MAGIC raise
-- COMMAND ----------
-- MAGIC %python
-- MAGIC # 列出blob上的文件列表
-- MAGIC def list_file_name(path):
-- MAGIC first_path_list = [i.path for i in dbutils.fs.ls(path)]
-- MAGIC second_path_list = [dbutils.fs.ls(i)[0] for i in first_path_list ]
-- MAGIC return second_path_list
-- COMMAND ----------
-- MAGIC %python
-- MAGIC # 从blob下载文件到local
-- MAGIC def download_file(file_path, local_path):
-- MAGIC # dbutils.fs.cp(file_path, local_path.replace("/dbfs", ""))
-- MAGIC dbutils.fs.cp(file_path, local_path)
-- MAGIC print(f"已下载 {file_path} 到 {local_path}")
-- MAGIC return local_path
-- COMMAND ----------
-- MAGIC %md
-- MAGIC ### 获取路径下的文件名称
-- MAGIC - 并挑出符合条件的文件路径
-- COMMAND ----------
-- MAGIC %python
-- MAGIC try:
-- MAGIC if path_exists(base_path):
-- MAGIC all_file_list = list_file_name(base_path)
-- MAGIC # 生成df来筛选内容
-- MAGIC files_df = pd.DataFrame([{
-- MAGIC 'path':f.path,
-- MAGIC 'modificationtime': f.modificationTime,
-- MAGIC 'name': f.name
-- MAGIC } for f in all_file_list])
-- MAGIC print(f"{base_path} 路径存在")
-- MAGIC else:
-- MAGIC print(f"{base_path} 路径不存在")
-- MAGIC except Exception as e:
-- MAGIC print(e)
-- COMMAND ----------
-- MAGIC %python
-- MAGIC try:
-- MAGIC files_df = files_df.sort_values('modificationtime', ascending=False).drop_duplicates('name').sort_index()
-- MAGIC files_df = files_df[files_df['name'].str.match(r'^Pharbers_PROV_Fact.*\.csv$')]
-- MAGIC files_df
-- MAGIC except Exception as e:
-- MAGIC print(e)
-- COMMAND ----------
-- MAGIC %md
-- MAGIC ### 读取文件内容
-- COMMAND ----------
-- MAGIC %python
-- MAGIC import os
-- COMMAND ----------
-- MAGIC %python
-- MAGIC # 下载数据到local
-- MAGIC try:
-- MAGIC if files_df['path'].tolist():
-- MAGIC # 如果列表不为空
-- MAGIC df_all = []
-- MAGIC for file in files_df['path'].tolist():
-- MAGIC local_path = download_file(file, f"/Volumes/{NGBI_CATALOG}/tmp/volume_tmp/tmp/{os.path.basename(file)}")
-- MAGIC file_df = (spark.read.option("header", "true").option("quote", '"').option("escape", '"').option("multiLine", "true").option("mode", "PERMISSIVE").csv(local_path))
-- MAGIC file_df = file_df.drop("TA",'Market')
-- MAGIC print(f'已读取{local_path}')
-- MAGIC df_all.append(file_df)
-- MAGIC # df_combine = pd.concat(df_all)
-- MAGIC df_ifexists = True
-- MAGIC else:
-- MAGIC print('没有符合条件的文件')
-- MAGIC df_ifexists= False
-- MAGIC except Exception as e:
-- MAGIC print(e)
-- COMMAND ----------
-- MAGIC %python
-- MAGIC try:
-- MAGIC if df_ifexists: # 如果不为空
-- MAGIC spark.sql(f"TRUNCATE table tmp.tmp_chpa_raw_data")
-- MAGIC num = 1
-- MAGIC for i in df_all:
-- MAGIC i.withColumnRenamed('IMS.药品ID','IMS_DRUG_ID') \
-- MAGIC .withColumnRenamed('是否法伯编码','IS_HOSP_CODE') \
-- MAGIC .withColumnRenamed('规格','SPEC') \
-- MAGIC .withColumnRenamed('转换比','CONVERSION_RATIO') \
-- MAGIC .withColumnRenamed('剂型','DOSAGE_FORM') \
-- MAGIC .withColumnRenamed('价格','PRICE')
-- MAGIC # i.columns = ['area','city','yyyyqq','yyyymm','reimburse','prescription_source','prescription_dept_lv1','prescription_dept_lv2','prescription_dept_lv3','ATC','new_code','common_name','product_name','pack_des','PackSize','PackageType','nfc','manu_des','drug_delivery_route','prescription','sales_vol','sales_value'] #'h_level',
-- MAGIC # sdf = spark.createDataFrame(i)
-- MAGIC i.createOrReplaceTempView('fact_sales')
-- MAGIC spark.sql(f"INSERT into tmp.tmp_chpa_raw_data SELECT * FROM fact_sales")
-- MAGIC print(f'第{num}个')
-- MAGIC num +=1
-- MAGIC except Exception as e:
-- MAGIC print(e)
-- COMMAND ----------
-- MAGIC %md
-- MAGIC ### 将读取到的dataframe写入表中
-- COMMAND ----------
-- 全量覆盖
insert overwrite dwd.dwd_gnd_pharbers_prov_fact
select
year,
ym,
province_c,
ims_drug_id,
is_hosp_code,
prod_corp,
prod_cod,
pack_cod,
phcd,
prod_des,
cmps_des,
corp_des,
mnfl_cod,
prod_des_c,
cmps_c,
corp_des_c,
pack_des,
spec,
conversion_ratio,
dosage_form,
atc4_cod,
app1_cod,
app1_des,
app1_des_c,
app2_cod,
app2_des,
app2_des_c,
app3_cod,
app3_des,
app3_des_c,
vbp_batch,
vbp,
value,
totalunit,
countingunit,
price,
manu_des,
manu_des_c,
NULL AS source_file_path,
NULL AS source_file_name,
from_utc_timestamp(current_timestamp(),'UTC+8') AS etl_insert_dt
from tmp.tmp_chpa_raw_data

19
CHPA/02 DWS_IMS_TD_GEO.sql Normal file
View File

@@ -0,0 +1,19 @@
-- Databricks notebook source
-- 20260119 地理数据 我感觉可以拉到前面去
--地理维度信息 Audit
-- 20260122 因为城市数据不再提供,改为硬编码省份数据. By Yusuo/Chenwu
INSERT OVERWRITE TABLE DWS.DWS_IMS_TD_GEO
SELECT
AUDIT_COD,
AUDIT_DES,
AUDIT_DES_C,
AUDIT_TYPE,
CITY_TIER,
AZ_CITY_TIER,
PROVINCE,
PROVINCE_C,
REGIONCENTER,
ETL_INSERT_DT,
ETL_UPDATE_DT
FROM
tmp.tmp_province_rawdata

33
CHPA/02 dws_ims_td_atc_cn.sql Normal file
View File

@@ -0,0 +1,33 @@
-- Databricks notebook source
-- 源表：DWD.DWD_GND_IMS_TBLATC,DWD.DWD_IMS_ATC_HIERARCHY
-- 目标表：DWS.DWS_IMS_TD_ATC_CN
-- COMMAND ----------
--ATC挂中文名
insert overwrite table dws.dws_ims_td_atc_cn
SELECT DISTINCT
T1.ATC1_CODE AS ATC1_COD,
T1.ATC1_DES,
case when ATC1.ATC1_DES_C is null then T1.ATC1_DES else ATC1.ATC1_DES_C end AS ATC1_DES_C,
T1.ATC2_CODe AS ATC2_COD,
T1.ATC2_DES,
case when ATC2.ATC2_DES_C is null then T1.ATC2_DES else ATC2.ATC2_DES_C end AS ATC2_DES_C,
T1.ATC3_CODe AS ATC3_COD,
T1.ATC3_DES,
case when ATC3.ATC3_DES_C is null then T1.ATC3_DES else ATC3.ATC3_DES_C end AS ATC3_DES_C,
T1.ATC4_CODe AS ATC4_COD,
T1.ATC4_DES,
case when ATC4.ATC4_DES_C is null then T1.ATC4_DES else ATC4.ATC4_DES_C end AS ATC4_DES_C,
from_utc_timestamp(current_timestamp(),'UTC+8'),
from_utc_timestamp(current_timestamp(),'UTC+8')
FROM dwd.dwd_ims_ATC_Hierarchy T1
LEFT JOIN (SELECT DISTINCT ATC4_COD,ATC4_DES_C FROM dwd.dwd_gnd_ims_tblATC) ATC4
ON T1.ATC4_CODE = ATC4.ATC4_COD
LEFT JOIN (SELECT DISTINCT ATC3_COD,ATC3_DES_C FROM dwd.dwd_gnd_ims_tblATC) ATC3
ON T1.ATC3_CODE = ATC3.ATC3_COD
LEFT JOIN (SELECT DISTINCT ATC2_COD,ATC2_DES_C FROM dwd.dwd_gnd_ims_tblATC) ATC2
ON T1.ATC2_CODE = ATC2.ATC2_COD
LEFT JOIN (SELECT DISTINCT ATC1_COD,ATC1_DES_C FROM dwd.dwd_gnd_ims_tblATC) ATC1
ON T1.ATC1_CODE = ATC1.ATC1_COD
;

19
CHPA/02 dws_ims_td_corp_cn.sql Normal file
View File

@@ -0,0 +1,19 @@
-- Databricks notebook source
-- 20260119 CORP 公司名中文信息 我感觉可以拉到前面去
--CORP 公司名中文信息
insert overwrite table dws.dws_ims_td_corp_cn
SELECT DISTINCT T1.CORP_ABBR AS CORP_COD,T1.CORP_DES,T2.nameC AS CORP_DES_C,
from_utc_timestamp(current_timestamp(),'UTC+8'),
from_utc_timestamp(current_timestamp(),'UTC+8')
FROM dwd.dwd_ims_td_manufacturer_corp T1
LEFT JOIN dwd.dwd_gnd_ims_tblmanucn T2
ON T1.CORP_ABBR = T2.abbrev
WHERE CORP_ABBR IS NOT NULL
;
-- COMMAND ----------
--新增AZDealed部分的维度
INSERT INTO dws.dws_ims_td_corp_cn SELECT 'A5ZD','AZDealed','AZDealed',
from_utc_timestamp(current_timestamp(),'UTC+8'),
from_utc_timestamp(current_timestamp(),'UTC+8');

28
CHPA/02 dws_ims_td_date.sql Normal file
View File

@@ -0,0 +1,28 @@
-- Databricks notebook source
-- 依赖事实表 tmp.tmp_ims_tf_fact_sales； 所以一定要放后面。
--基于Fact数据动态计算时间维度 最近五年
insert overwrite table dws.dws_ims_td_date
select distinct
YM,
left(cast(YM as string),4) as YEAR,
right(cast(YM as string),2) as MONTH,
concat(
'Q',quarter(
date(
concat(
left(cast(YM as string),4),'-',right(cast(YM as string),2),'-01'
)))) as QUARTER,
concat(
left(cast(YM as string),4),'Q',quarter(
date(
concat(
left(cast(YM as string),4),'-',right(cast(YM as string),2),'-01'
)))) as YQ,
case when YM=(select max(YM) from tmp.tmp_ims_tf_fact_sales) then 'R' else right(cast(YM as string),2) end as DATE_FLAG,
case when YM%100>6 then concat(left(cast(YM as string),4),'H2') else concat(left(cast(YM as string),4),'H1') end as HALF_YEAR,
from_utc_timestamp(current_timestamp(),'UTC+8'),
from_utc_timestamp(current_timestamp(),'UTC+8')
from tmp.tmp_ims_tf_fact_sales
where YM > (select max(YM)-900 from tmp.tmp_ims_tf_fact_sales)
order by YM desc
;

18
CHPA/02 dws_ims_td_manu_cn.sql Normal file
View File

@@ -0,0 +1,18 @@
-- Databricks notebook source
-- 20260119 MANU 生产商名中文信息 我感觉可以拉到前面去
---MANU 生产商名中文信息
insert overwrite table dws.dws_ims_td_manu_cn
SELECT DISTINCT T1.Manufacturer_Abbr AS MANU_COD,T1.Manufacturer_Name AS MANU_DES,T2.nameC AS MANU_DES_C,
from_utc_timestamp(current_timestamp(),'UTC+8'),
from_utc_timestamp(current_timestamp(),'UTC+8')
FROM dwd.dwd_ims_td_manufacturer_corp T1
LEFT JOIN dwd.dwd_gnd_ims_tblmanucn T2
ON T1.Manufacturer_Abbr = T2.abbrev
WHERE T1.Manufacturer_Abbr IS NOT NULL
;
-- COMMAND ----------
INSERT INTO dws.dws_ims_td_manu_cn SELECT 'A5ZD','AZDealed','AZDealed',
from_utc_timestamp(current_timestamp(),'UTC+8'),
from_utc_timestamp(current_timestamp(),'UTC+8');

238
CHPA/02 dws_ims_td_market.sql Normal file
View File

@@ -0,0 +1,238 @@
-- Databricks notebook source
-- 20260119 依赖上面 dwd.dwd_ims_td_pack_property
-- 20260119 依赖上面补0 dwd.dwd_ims_td_pack_property dwd.dwd_gnd_ims_tblmarket
-----MARKET
--第一步：
--先处理 Extend_Market 和 NOT_IN_FLAG 都为空的数据，代表有明确定义得市场
insert overwrite table tmp.tmp_ims_td_temp_market
select distinct t2.market,t1.*,
t2.bu,CASE WHEN t2.extend_market_ratio IS NULL THEN '1' ELSE t2.extend_market_ratio END AS Market_Ratio
from dwd.dwd_ims_td_pack_property t1
left join (select * from dwd.dwd_gnd_ims_tblmarket WHERE Extend_Market IS NULL AND ( NOT_IN_FLAG IS NULL or NOT_IN_FLAG = '1' )) t2
on t1.ATC1_COD = case when t2.ATC1_Code is null then t1.ATC1_COD else t2.ATC1_Code end
and t1.ATC2_COD = case when t2.ATC2_Code is null then t1.ATC2_COD else t2.ATC2_Code end
and t1.ATC3_COD = case when t2.ATC3_Code is null then t1.ATC3_COD else t2.ATC3_Code end
and t1.ATC4_COD = case when t2.ATC4_Code is null then t1.ATC4_COD else t2.ATC4_Code end
and t1.APP1_COD = case when t2.NFC1_Code is null then t1.APP1_COD else t2.NFC1_Code end
and t1.APP2_COD = case when t2.NFC2_Code is null then t1.APP2_COD else t2.NFC2_Code end
and t1.APP3_COD = case when t2.NFC3_Code is null then t1.APP3_COD else t2.NFC3_Code end
and t1.CORP_COD = case when t2.corporation_code is null then t1.CORP_COD else t2.corporation_code end
and t1.MANU_COD = case when t2.Manufacturer_Code is null then t1.MANU_COD else t2.Manufacturer_Code end
and t1.PROD_COD = case when t2.Product_Code is null then t1.PROD_COD else t2.Product_Code end
and t1.PACK_COD = case when t2.Pack_Code is null then t1.PACK_COD else t2.Pack_Code end
and t1.STGH_DES = case when t2.Strength is null then t1.STGH_DES else t2.Strength end
and t1.CMPS_COD = case when t2.Molecule_Code is null then t1.CMPS_COD else t2.Molecule_Code end
where t2.market is not null
;
-- COMMAND ----------
--第二步：
--处理 Extend_Market 为空，并且 NOT_IN_FLAG 有值的数据，代表反选定义市场的规则
insert overwrite table tmp.tmp_ims_td_temp_market_del
select distinct t2.market,t1.*,t2.bu,CASE WHEN t2.extend_market_ratio IS NULL THEN '1' ELSE t2.extend_market_ratio END AS Market_Ratio
from dwd.dwd_ims_td_pack_property t1
left join (select * from dwd.dwd_gnd_ims_tblmarket WHERE Extend_Market IS NULL AND NOT_IN_FLAG = '0' ) t2
on t1.ATC1_COD = case when t2.ATC1_Code is null then t1.ATC1_COD else t2.ATC1_Code end
and t1.ATC2_COD = case when t2.ATC2_Code is null then t1.ATC2_COD else t2.ATC2_Code end
and t1.ATC3_COD = case when t2.ATC3_Code is null then t1.ATC3_COD else t2.ATC3_Code end
and t1.ATC4_COD = case when t2.ATC4_Code is null then t1.ATC4_COD else t2.ATC4_Code end
and t1.APP1_COD = case when t2.NFC1_Code is null then t1.APP1_COD else t2.NFC1_Code end
and t1.APP2_COD = case when t2.NFC2_Code is null then t1.APP2_COD else t2.NFC2_Code end
and t1.APP3_COD = case when t2.NFC3_Code is null then t1.APP3_COD else t2.NFC3_Code end
and t1.CORP_COD = case when t2.corporation_code is null then t1.CORP_COD else t2.corporation_code end
and t1.MANU_COD = case when t2.Manufacturer_Code is null then t1.MANU_COD else t2.Manufacturer_Code end
and t1.PROD_COD = case when t2.Product_Code is null then t1.PROD_COD else t2.Product_Code end
and t1.PACK_COD = case when t2.Pack_Code is null then t1.PACK_COD else t2.Pack_Code end
and t1.STGH_DES = case when t2.Strength is null then t1.STGH_DES else t2.Strength end
and t1.CMPS_COD = case when t2.Molecule_Code is null then t1.CMPS_COD else t2.Molecule_Code end
where t2.market is not null
;
-- COMMAND ----------
--！！！注意此段是反选的规则，故执行的是从已经定义好的TempMKT中删除数据
--需要看TempMKT的字段结构，没有值的字段，在Value里面放个空值
MERGE INTO tmp.tmp_ims_td_temp_market AS t1
USING tmp.tmp_ims_td_temp_market_del AS t2
on t1.ATC1_COD = t2.ATC1_COD
and t1.ATC2_COD = t2.ATC2_COD
and t1.ATC3_COD = t2.ATC3_COD
and t1.ATC4_COD = t2.ATC4_COD
and t1.APP1_COD = t2.APP1_COD
and t1.APP2_COD = t2.APP2_COD
and t1.APP3_COD = t2.APP3_COD
and t1.CORP_COD = t2.CORP_COD
and t1.MANU_COD = t2.MANU_COD
and t1.PROD_COD = t2.PROD_COD
and t1.PACK_COD = t2.PACK_COD
and t1.STGH_DES = t2.STGH_DES
and t1.CMPS_COD = t2.CMPS_COD
and t1.market = t2.market
WHEN MATCHED THEN DELETE
;
-- COMMAND ----------
---第三步：
--处理 Extend_Market 有值得数据，用已有市场，定义新的市场，带上系数
insert into tmp.tmp_ims_td_temp_market
select distinct
t2.Market,
t1.PACK_COD ,
t1.PACK_DES ,
t1.STGH_DES ,
t1.PACK_LCH ,
t1.PROD_COD ,
t1.CMPS_COD ,
t1.CMPS_DES ,
t1.ATC1_COD ,
t1.ATC2_COD ,
t1.ATC3_COD ,
t1.ATC4_COD ,
t1.APP1_COD ,
t1.APP2_COD ,
t1.APP3_COD ,
t1.BIO_DESC ,
t1.GENE_ORIG_DESC ,
t1.ETH_OTC_DESC ,
t1.NRDL_DESC ,
t1.NRDL_Entry_Date ,
t1.EDL_DESC ,
t1.TCM_DESC ,
t1.PAED_DESC ,
t1.GQCE_DESC ,
t1.VBP_DESC ,
t1.MANU_COD ,
t1.MANU_DES ,
t1.MNFL_COD ,
t1.MNFL_DES ,
t1.CORP_COD ,
t1.CORP_DES ,
t2.BU,
t1.BrandType ,
case when t2.Extend_Market_Ratio is null then '1' end AS Market_Ratio--市场的系数若没有维护，默认为1
from tmp.tmp_ims_td_temp_market t1
left join (select Market, BU,Extend_Market,Extend_Market_Ratio from dwd.dwd_gnd_ims_tblmarket WHERE Extend_Market IS NOT NULL) t2 on t1.Market=t2.Extend_Market
where t2.Market is not null
;
-- COMMAND ----------
--将，市场的系数为空值的数据，初始化为1
UPDATE tmp.tmp_ims_td_temp_market SET Market_Ratio = 1 WHERE Market_Ratio IS NULL;
-- COMMAND ----------
-- 20260119 依赖 tmp.tmp_ims_td_temp_market
-- 20260119 依赖上面补0 dwd.dwd_gnd_ims_tblkeycompetitor
--KeyCompetitor部分
--定义Key Competitor
insert overwrite table tmp.tmp_ims_td_temp_market_upd1
select distinct t2.keycompetitor as Key_Competitor,t2.no,t2.no1,t1.*
from tmp.tmp_ims_td_temp_market t1
left join (
select
ROW_NUMBER() OVER(ORDER BY
CASE
WHEN ATC1_Code IS NOT NULL THEN 1
WHEN ATC2_Code IS NOT NULL THEN 2
WHEN ATC3_Code IS NOT NULL THEN 3
WHEN ATC4_Code IS NOT NULL THEN 4
WHEN Molecule_Code IS NOT NULL THEN 5
WHEN Product_Code IS NOT NULL THEN 6
WHEN Pack_Code IS NOT NULL THEN 7
ELSE 999 END,
CASE WHEN NFC1_CODE IS NULL THEN 0 ELSE 1 END,
CASE WHEN NFC2_CODE IS NULL THEN 0 ELSE 1 END,
CASE WHEN NFC2_CODE IS NULL THEN 0 ELSE 1 END,
CASE WHEN NFC3_CODE IS NULL THEN 0 ELSE 1 END
) AS no1,* from dwd.dwd_gnd_ims_tblkeycompetitor
)t2
on t1.ATC1_COD = case when t2.ATC1_Code is null then t1.ATC1_COD else t2.ATC1_Code end
and t1.ATC2_COD = case when t2.ATC2_Code is null then t1.ATC2_COD else t2.ATC2_Code end
and t1.ATC3_COD = case when t2.ATC3_Code is null then t1.ATC3_COD else t2.ATC3_Code end
and t1.ATC4_COD = case when t2.ATC4_Code is null then t1.ATC4_COD else t2.ATC4_Code end
and t1.APP1_COD = case when t2.NFC1_Code is null then t1.APP1_COD else t2.NFC1_Code end
and t1.APP2_COD = case when t2.NFC2_Code is null then t1.APP2_COD else t2.NFC2_Code end
and t1.APP3_COD = case when t2.NFC3_Code is null then t1.APP3_COD else t2.NFC3_Code end
and t1.CORP_COD = case when t2.corporation_code is null then t1.CORP_COD else t2.corporation_code end
and t1.MANU_COD = case when t2.Manufacturer_Code is null then t1.MANU_COD else t2.Manufacturer_Code end
and t1.PROD_COD = case when t2.Product_Code is null then t1.PROD_COD else t2.Product_Code end
and t1.PACK_COD = case when t2.Pack_Code is null then t1.PACK_COD else t2.Pack_Code end
and t1.STGH_DES = case when t2.Strength is null then t1.STGH_DES else t2.Strength end
and t1.CMPS_COD = case when t2.Molecule_Code is null then t1.CMPS_COD else t2.Molecule_Code end
and t1.market = case when t2.market is null then t1.market else t2.market end
;
-- COMMAND ----------
--删除被重复定义的Key Competitor，删除写入时间早的（No 小的）的对应行即可，保留No最大的一行即最新的规则
insert overwrite table tmp.tmp_ims_td_temp_market_upd2
select * from(
select row_number() over(partition by market,pack_cod,prod_cod order by no1 desc,no desc) as id,*
from tmp.tmp_ims_td_temp_market_upd1) where id>1
;
-- COMMAND ----------
MERGE INTO tmp.tmp_ims_td_temp_market_upd1 AS t1
USING tmp.tmp_ims_td_temp_market_upd2 AS t2
on t1.ATC1_COD = t2.ATC1_COD
and t1.ATC2_COD = t2.ATC2_COD
and t1.ATC3_COD = t2.ATC3_COD
and t1.ATC4_COD = t2.ATC4_COD
and t1.APP1_COD = t2.APP1_COD
and t1.APP2_COD = t2.APP2_COD
and t1.APP3_COD = t2.APP3_COD
and t1.CORP_COD = t2.CORP_COD
and t1.MANU_COD = t2.MANU_COD
and t1.PROD_COD = t2.PROD_COD
and t1.PACK_COD = t2.PACK_COD
and t1.STGH_DES = t2.STGH_DES
and t1.CMPS_COD = t2.CMPS_COD
and t1.market = t2.market
and t1.Key_Competitor = t2.Key_Competitor
and t1.no=t2.no
WHEN MATCHED THEN DELETE
;
-- COMMAND ----------
--没有挂上KeyCompetitor的，更新为Others ,最后去掉No，写入临时表备用
update tmp.tmp_ims_td_temp_market_upd1 set Key_Competitor = 'Others' where Key_Competitor is NULL;
-- COMMAND ----------
insert overwrite table tmp.tmp_ims_td_temp_market_upd
select
Key_Competitor,market,PACK_COD,PACK_DES,STGH_DES,PACK_LCH,PROD_COD,CMPS_COD,CMPS_DES,
ATC1_COD,ATC2_COD,ATC3_COD,ATC4_COD,APP1_COD,APP2_COD,APP3_COD,BIO_DESC,GENE_ORIG_DESC,
ETH_OTC_DESC,NRDL_DESC,NRDL_Entry_Date,EDL_DESC,TCM_DESC,PAED_DESC,GQCE_DESC,VBP_DESC,
MANU_COD,MANU_DES,MNFL_COD,MNFL_DES,CORP_COD,CORP_DES,BrandType,bu,Market_Ratio
from tmp.tmp_ims_td_temp_market_upd1
;
-- COMMAND ----------
-- 20260119 市场维度信息 我感觉可以拉到前面去
--市场维度信息
insert overwrite table dws.dws_ims_td_market
select distinct market,PACK_COD,PACK_DES,STGH_DES,PACK_LCH,PROD_COD,CMPS_COD,CMPS_DES,
ATC1_COD,ATC2_COD,ATC3_COD,ATC4_COD,APP1_COD,APP2_COD,APP3_COD,BIO_DESC,GENE_ORIG_DESC,
ETH_OTC_DESC,NRDL_DESC,NRDL_Entry_Date,EDL_DESC,TCM_DESC,PAED_DESC,GQCE_DESC,VBP_DESC,
MANU_COD,MANU_DES,MNFL_COD,MNFL_DES,CORP_COD,CORP_DES,BrandType,bu,Market_Ratio,Key_Competitor,
from_utc_timestamp(current_timestamp(),'UTC+8'),
from_utc_timestamp(current_timestamp(),'UTC+8')
from tmp.tmp_ims_td_temp_market_upd
;

7
CHPA/02 dws_ims_td_market_ta.sql Normal file
View File

@@ -0,0 +1,7 @@
-- Databricks notebook source
-- 20260119 市场&TA 关系信息 我感觉可以拉到前面去
--市场&TA 关系信息
insert overwrite table dws.dws_ims_td_market_ta
select *
from dwd.dwd_gnd_ims_tblmarket_ta_map
;

22
CHPA/02 dws_ims_td_nfc_cn.sql Normal file
View File

@@ -0,0 +1,22 @@
-- Databricks notebook source
--NFC挂中文名
insert overwrite table dws.dws_ims_td_nfc_cn
SELECT DISTINCT T1.NFC1_CODE AS APP1_COD ,
T1.NFC1_DES AS APP1_DES,
case when NFC1.APP1_DES_C is null then T1.NFC1_DES else NFC1.APP1_DES_C end AS APP1_DES_C,
T1.NFC2_CODE AS APP2_COD ,
T1.NFC2_DES AS APP2_DES,
case when NFC2.APP2_DES_C is null then T1.NFC2_DES else NFC2.APP2_DES_C end AS APP2_DES_C,
T1.NFC3_CODE AS APP3_COD,
T1.NFC3_DES AS APP3_DES,
case when NFC3.APP3_DES_C is null then T1.NFC3_DES else NFC3.APP3_DES_C end AS APP3_DES_C,
from_utc_timestamp(current_timestamp(),'UTC+8'),
from_utc_timestamp(current_timestamp(),'UTC+8')
FROM dwd.dwd_ims_NFC_Hierarchy T1
LEFT JOIN (SELECT DISTINCT APP3_COD,APP3_DES_C FROM dwd.dwd_gnd_ims_tblAPP) NFC3
ON T1.NFC3_CODE = NFC3.APP3_COD
LEFT JOIN (SELECT DISTINCT APP2_COD,APP2_DES_C FROM dwd.dwd_gnd_ims_tblAPP) NFC2
ON T1.NFC2_CODE = NFC2.APP2_COD
LEFT JOIN (SELECT DISTINCT APP1_COD,APP1_DES_C FROM dwd.dwd_gnd_ims_tblAPP) NFC1
ON T1.NFC1_CODE = NFC1.APP1_COD
;

7
CHPA/02 dws_ims_td_prod_cn.sql Normal file
View File

@@ -0,0 +1,7 @@
-- Databricks notebook source
--产品挂中文名
insert overwrite table dws.dws_ims_td_prod_cn
SELECT distinct RIGHT(concat('0000000000',prodcode),9) AS PROD_COD,ename as PROD_DES,namec as PROD_DES_C, gene_name as CMPS_DES_C,1 AS RANK_TYPE,
from_utc_timestamp(current_timestamp(),'UTC+8'),
from_utc_timestamp(current_timestamp(),'UTC+8')
FROM dwd.dwd_gnd_ims_tblprodcn

32
CHPA/02 tmp_ims_td_prod_tmp.sql Normal file
View File

@@ -0,0 +1,32 @@
-- Databricks notebook source
-- DBTITLE 1,产品挂中文名
--Product中文名字,若有超过5个CORP或者MANU生产，则将其报告上排序的优先级调低
insert overwrite table tmp.tmp_ims_td_prod_tmp
SELECT DISTINCT PROD.PROD_COD
FROM dws.dws_ims_td_prod_cn PROD
LEFT JOIN dwd.dwd_ims_td_pack_property PACK
ON PROD.PROD_COD = PACK.PROD_COD
LEFT JOIN dws.dws_ims_td_corp_cn CORP
ON PACK.CORP_COD = CORP.CORP_COD
WHERE PROD.PROD_DES_C IN
(
--先计算出超过10个厂家生产的Prod_CN
--先计算出有多个厂家生产的Prod_CN
SELECT PROD.PROD_DES_C
FROM dwd.dwd_ims_td_pack_property PACK
LEFT JOIN dws.dws_ims_td_prod_cn PROD
ON PACK.PROD_COD = PROD.PROD_COD
GROUP BY PROD.PROD_DES_C
HAVING ( COUNT(DISTINCT CORP_DES) > 5 OR COUNT(DISTINCT MANU_DES) > 5 )
AND PROD.PROD_DES_C IS NOT NULL
);
-- COMMAND ----------
UPDATE dws.dws_ims_td_prod_cn SET RANK_TYPE = 0 where PROD_COD in (
SELECT DISTINCT PROD_COD FROM tmp.tmp_ims_td_prod_tmp
);

317
CHPA/02 tmp_ims_tf_fact_sales.sql Normal file
View File

@@ -0,0 +1,317 @@
-- Databricks notebook source
-- 20260130 临时方案，解决测试环境有NULL值问题
create or replace temporary view dwd_ims_td_pack
as
select
distinct
pack_id,
pack_code
from dwd.dwd_ims_td_pack
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW FACT_CHPA_SALES_TEMP_WITH_PREVIOUS
AS
SELECT
CAST(YM AS INT) AS YM,
CAST(YEAR AS INT) AS YEAR,
CAST(REPLACE(YM,YEAR,'') AS INT) AS MONTH,
CAST(VALUE AS DECIMAL(38,10)) AS VALUE,
CAST(COUNTINGUNIT AS DECIMAL(38,10)) AS COUNTINGUNIT,
CAST(TOTALUNIT AS DECIMAL(38,10)) AS TOTALUNIT,
PROVINCE_C,
PHCD,
conversion_ratio
FROM dwd.dwd_gnd_pharbers_prov_fact
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW DIM_PROVINCE
AS
select
-- t2.geo_key ,
t2.province_code,
CASE
WHEN
t2.province_name
IN ('上海','北京','天津','重庆')
THEN
CONCAT(t2.province_name,'市')
WHEN
t2.province_name
IN ('内蒙古','西藏')
THEN
CONCAT(t2.province_name,'自治区')
WHEN
t2.province_name = '宁夏'
THEN CONCAT(t2.province_name,'回族自治区')
WHEN
t2.province_name = '广西'
THEN CONCAT(t2.province_name,'壮族自治区')
WHEN
t2.province_name = '新疆'
THEN CONCAT(t2.province_name,'维吾尔自治区')
ELSE CONCAT(t2.province_name,'省')
END AS province_name
from dm.dm_td_geography t2
where t2.geo_key = t2.province_code
and province_name not in ('台湾','澳门','香港')
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW DIM_CHPA_PACK_INFO
AS
SELECT
PACK_COD,
MAX(IQVIA_PACK_CODE) AS IQVIA_PACK_CODE,
MAX(countingunit) as COUNTINGUNIT_RATIO
FROM
dwd.dwd_gnd_dept_pack_property
GROUP BY
PACK_COD
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW FACT_CHPA_SALES
AS
SELECT
YM,
YEAR,
REPLACE(YM,YEAR,'') AS MONTH,
T3.IQVIA_PACK_CODE AS PACK_CODE,
T2.PROVINCE_CODE AS AUDIT_COD,
VALUE AS MTH00LC,
0 AS MTH00LCLY,
CASE WHEN T3.COUNTINGUNIT_RATIO IS NULL THEN TOTALUNIT*T1.conversion_ratio ELSE TOTALUNIT*T3.COUNTINGUNIT_RATIO END AS MTH00CN,
0 AS MTH00CNLY,
TOTALUNIT AS MTH00UN,
0 AS MTH00UNLY
FROM FACT_CHPA_SALES_TEMP_WITH_PREVIOUS AS T1
LEFT JOIN DIM_PROVINCE AS T2 ON T1.PROVINCE_C = T2.PROVINCE_NAME
LEFT JOIN DIM_CHPA_PACK_INFO AS T3 ON T1.PHCD = T3.PACK_COD
UNION ALL
SELECT
CAST((YEAR+1)*100+REPLACE(YM,YEAR,'') AS INT) AS YM,
YEAR+1,
CAST(REPLACE(YM,YEAR,'') AS INT) AS MONTH,
T3.IQVIA_PACK_CODE AS PACK_CODE,
T2.PROVINCE_CODE AS AUDIT_COD,
0 AS MTH00LC,
VALUE AS MTH00LCLY,
0 AS MTH00CN,
CASE WHEN T3.COUNTINGUNIT_RATIO IS NULL THEN TOTALUNIT*T1.conversion_ratio ELSE TOTALUNIT*T3.COUNTINGUNIT_RATIO END AS MTH00CNLY,
0 AS MTH00UN,
TOTALUNIT AS MTH00UNLY
FROM FACT_CHPA_SALES_TEMP_WITH_PREVIOUS AS T1
LEFT JOIN DIM_PROVINCE AS T2 ON T1.PROVINCE_C = T2.PROVINCE_NAME
LEFT JOIN DIM_CHPA_PACK_INFO AS T3 ON T1.PHCD = T3.PACK_COD
-- COMMAND ----------
-- 20260119 dwd.dwd_ims_td_pack、dwd.dwd_ims_tf_fact_sales 不依赖
--删除最近五年的dim pack数据，将最新的dim pack写入
delete from dws.dws_ims_td_pack_ym
where ym + 500 >
(select max(year * 100 + month) from dwd.dwd_ims_tf_fact_sales);
insert into dws.dws_ims_td_pack_ym
select
distinct ym,
pack_id,
pack_code,
from_utc_timestamp(current_timestamp(), 'UTC+8'),
from_utc_timestamp(current_timestamp(), 'UTC+8')
from
(
select
distinct year * 100 + month as ym
from
dwd.dwd_ims_tf_fact_sales
where
year * 100 + month + 500 > (
select max(year * 100 + month) from dwd.dwd_ims_tf_fact_sales
)
) full
-- join dwd.dwd_ims_td_pack
join dwd_ims_td_pack
;
-- COMMAND ----------
-- --IMS 原始数据预处理
-- --FACT SALES 处理
-- insert overwrite table tmp.tmp_ims_tf_fact_sales
-- select distinct
-- FACT.YM,AUDIT.Audit_Code AS AUDIT_COD,FACT.Pack_Code AS PACK_COD,
-- sum(MTH00LC) as MTH00LC,sum(MTH00LCLY) as MTH00LCLY,
-- sum(MTH00CN) as MTH00CN,sum(MTH00CNLY) as MTH00CNLY,
-- sum(MTH00UN) as MTH00UN,sum(MTH00UNLY) as MTH00UNLY
-- from(
-- select
-- Year*100+Month as YM,
-- Year ,
-- Month ,
-- t2.pack_code ,
-- Audit_ID ,
-- SalesValue_LC as MTH00LC,
-- 0 as MTH00LCLY,
-- Counting_Unit as MTH00CN,
-- 0 as MTH00CNLY,
-- Sales_Unit as MTH00UN,
-- 0 as MTH00UNLY
-- from dwd.dwd_ims_tf_fact_sales t1
-- --补充历史的PACK ID 和 PACK CODE的关联关系，带上YM将最新dim pack表full join 最近五年
-- left join dws.dws_ims_td_pack_ym t2 on t1.Pack_ID=t2.pack_id and t1.Year*100+t1.Month=t2.ym
-- union all
-- select
-- (Year+1)*100+Month as YM,
-- Year+1 ,
-- Month ,
-- t2.pack_code ,
-- Audit_ID ,
-- 0 as MTH00LC,
-- SalesValue_LC as MTH00LCLY,
-- 0 as MTH00CN,
-- Counting_Unit as MTH00CNLY,
-- 0 as MTH00UN,
-- Sales_Unit as MTH00UNLY
-- from dwd.dwd_ims_tf_fact_sales t1
-- --补充历史的PACK ID 和 PACK CODE的关联关系，带上YM将最新dim pack表full join 最近五年
-- left join dws.dws_ims_td_pack_ym t2 on t1.Pack_ID=t2.pack_id and t1.Year*100+t1.Month=t2.ym
-- )FACT
-- LEFT JOIN dwd.dwd_ims_td_audit AUDIT
-- ON FACT.Audit_ID = AUDIT.Audit_ID
-- where FACT.YM<=(select max(Year*100+Month) from dwd.dwd_ims_tf_fact_sales)
-- ----------20260320 chenwu CHPA的数据只保留202201之后的
-- and FACT.YM>=202201
-- ----------
-- AND AUDIT.Audit_Code = 'CHT'
-- group by FACT.YM,AUDIT.Audit_Code,FACT.Pack_Code
-- -- 20260120新增
-- -- 以前 全国和省份数据放在一起，现在全国是上面，省份是新的，单独放在下面。
-- union all
-- select distinct
-- CAST(YM AS INT) AS YM,
-- AUDIT_COD,Pack_Code AS PACK_COD,
-- sum(MTH00LC) as MTH00LC,sum(MTH00LCLY) as MTH00LCLY,
-- sum(MTH00CN) as MTH00CN,sum(MTH00CNLY) as MTH00CNLY,
-- sum(MTH00UN) as MTH00UN,sum(MTH00UNLY) as MTH00UNLY
-- from
-- FACT_CHPA_SALES
-- where
-- YM<=(select max(YM) from dwd.dwd_gnd_pharbers_prov_fact)
-- group by YM,Audit_Cod,Pack_Code
-- ;
-- COMMAND ----------
-- IMS 原始数据预处理 —— FACT SALES 处理
INSERT OVERWRITE TABLE tmp.tmp_ims_tf_fact_sales
WITH yearmont_range (--防止翘角导致数据存在差异
select
min(ym) YM
from
(
select
max(ym) ym
from
dwd.dwd_gnd_pharbers_prov_fact
union all
select
max(Year * 100 + Month) ym
from
dwd.dwd_ims_tf_fact_sales
)
)
-- ════════════════════════════════════════════
-- Part 1：全国数据（CHT）
-- ════════════════════════════════════════════
SELECT DISTINCT
FACT.YM,
AUDIT.Audit_Code AS AUDIT_COD,
FACT.Pack_Code AS PACK_COD,
SUM(MTH00LC) AS MTH00LC,
SUM(MTH00LCLY) AS MTH00LCLY,
SUM(MTH00CN) AS MTH00CN,
SUM(MTH00CNLY) AS MTH00CNLY,
SUM(MTH00UN) AS MTH00UN,
SUM(MTH00UNLY) AS MTH00UNLY
FROM (
-- 当期销售数据
SELECT
Year * 100 + Month AS YM,
Year,
Month,
t2.pack_code,
Audit_ID,
SalesValue_LC AS MTH00LC,
0 AS MTH00LCLY,
Counting_Unit AS MTH00CN,
0 AS MTH00CNLY,
Sales_Unit AS MTH00UN,
0 AS MTH00UNLY
FROM dwd.dwd_ims_tf_fact_sales t1
-- 补充历史 Pack_ID → Pack_Code 映射（带 YM，关联最近五年快照）
LEFT JOIN dws.dws_ims_td_pack_ym t2
ON t1.Pack_ID = t2.pack_id
AND t1.Year * 100 + t1.Month = t2.ym
UNION ALL
-- 同期数据（当期行 YM+1 年，作为去年同期值）
SELECT
(Year + 1) * 100 + Month AS YM,
Year + 1,
Month,
t2.pack_code,
Audit_ID,
0 AS MTH00LC,
SalesValue_LC AS MTH00LCLY,
0 AS MTH00CN,
Counting_Unit AS MTH00CNLY,
0 AS MTH00UN,
Sales_Unit AS MTH00UNLY
FROM dwd.dwd_ims_tf_fact_sales t1
-- 补充历史 Pack_ID → Pack_Code 映射（带 YM，关联最近五年快照）
LEFT JOIN dws.dws_ims_td_pack_ym t2
ON t1.Pack_ID = t2.pack_id
AND t1.Year * 100 + t1.Month = t2.ym
) FACT
LEFT JOIN dwd.dwd_ims_td_audit AUDIT
ON FACT.Audit_ID = AUDIT.Audit_ID
WHERE FACT.YM <= (SELECT YM FROM yearmont_range)
AND FACT.YM >= 202201 -- 20260320 chenwu CHPA 数据只保留 202201 之后
AND AUDIT.Audit_Code = 'CHT'
GROUP BY
FACT.YM,
AUDIT.Audit_Code,
FACT.Pack_Code
UNION ALL
-- ════════════════════════════════════════════
-- Part 2：省份数据（CHPA）
-- 20260120 新增：全国与省份拆分，省份数据单独维护
-- ════════════════════════════════════════════
SELECT DISTINCT
CAST(YM AS INT) AS YM,
AUDIT_COD,
Pack_Code AS PACK_COD,
SUM(MTH00LC) AS MTH00LC,
SUM(MTH00LCLY) AS MTH00LCLY,
SUM(MTH00CN) AS MTH00CN,
SUM(MTH00CNLY) AS MTH00CNLY,
SUM(MTH00UN) AS MTH00UN,
SUM(MTH00UNLY) AS MTH00UNLY
FROM FACT_CHPA_SALES
WHERE YM <= (SELECT YM FROM yearmont_range)
GROUP BY
YM,
Audit_Cod,
Pack_Code
;

160
CHPA/03 DM_TD_EXT_CHPA_MARKET_BRAND_RATIO.sql Normal file
View File

@@ -0,0 +1,160 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_CHPA_MARKET_BRAND_RATIO (
-- MARKET STRING,
-- PACK_CODE STRING,
-- VALUE_BRAND_RATIO FLOAT,
-- VALUE_BRAND_RATIO_START STRING,
-- VALUE_BRAND_RATIO_END STRING,
-- UNIT_BRAND_RATIO FLOAT,
-- UNIT_BRAND_START STRING,
-- UNIT_BRAND_END STRING,
-- COUNTINGUNIT_BRAND_RATIO FLOAT,
-- COUNTINGUNIT_BRAND_START STRING,
-- COUNTINGUNIT_BRAND_END STRING,
-- PDOT_BRAND_RATIO FLOAT,
-- PDOT_BRAND_START STRING,
-- PDOT_BRAND_END STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chpa_market_brand_ratio';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chpa_market_brand_ratio';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_CHPA_MARKET_BRAND_RATIO
SELECT
DISTINCT
T1.MARKET,
T1.PACK_CODE,
CASE
WHEN T2.RATIO IS NOT NULL THEN T2.RATIO
ELSE NVL(T5.RATIO, 1) END AS VALUE_BRAND_RATIO,
CASE
WHEN T2.RATIO IS NOT NULL THEN T2.STARTTIME
ELSE NVL(T5.STARTTIME, 200001) END AS VALUE_BRAND_RATIO_START,
CASE
WHEN T2.RATIO IS NOT NULL THEN T2.ENDTIME
ELSE NVL(T5.ENDTIME, 299912) END AS VALUE_BRAND_RATIO_END,
CASE
WHEN T3.RATIO IS NOT NULL THEN T3.RATIO
ELSE NVL(T6.RATIO, 1) END AS UNIT_BRAND_RATIO,
CASE
WHEN T3.RATIO IS NOT NULL THEN T3.STARTTIME
ELSE NVL(T6.STARTTIME, 200001) END AS UNIT_BRAND_START,
CASE
WHEN T3.RATIO IS NOT NULL THEN T3.ENDTIME
ELSE NVL(T6.ENDTIME, 299912) END AS UNIT_BRAND_END,
1 AS COUNTINGUNIT_BRAND_RATIO,
200001 AS COUNTINGUNIT_BRAND_START,
299912 AS COUNTINGUNIT_BRAND_END,
CASE
WHEN T4.RATIO IS NOT NULL THEN T4.RATIO
ELSE NVL(T7.RATIO, 1) END AS PDOT_BRAND_RATIO,
CASE
WHEN T4.RATIO IS NOT NULL THEN T4.STARTTIME
ELSE NVL(T7.STARTTIME, 200001) END AS PDOT_BRAND_START,
CASE
WHEN T4.RATIO IS NOT NULL THEN T4.ENDTIME
ELSE NVL(T7.ENDTIME, 299912) END AS PDOT_BRAND_END,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_CHPA_MARKET_RATIO T1
LEFT JOIN ( --VALUE--PACK
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'VALUE' AND UPPER(LEVEL) = 'PACK'
) T2 ON T1.MARKET = T2.MARKET AND T1.PACK_CODE = T2.PACK_COD
LEFT JOIN ( --UNIT--PACK
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'UNIT' AND UPPER(LEVEL) = 'PACK'
) T3 ON T1.MARKET = T3.MARKET AND T1.PACK_CODE = T3.PACK_COD
LEFT JOIN ( --COUNTINGUNIT--PACK
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'COUNTINGUNIT' AND UPPER(LEVEL) = 'PACK'
) T4 ON T1.MARKET = T4.MARKET AND T1.PACK_CODE = T4.PACK_COD
LEFT JOIN ( --VALUE--MOLECULE
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'VALUE' AND UPPER(LEVEL) = 'MOLECULE'
) T5 ON T1.MARKET = T5.MARKET AND T1.CMPS_CODE = T5.CMPS_COD
LEFT JOIN ( --UNIT--MOLECULE
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'UNIT' AND UPPER(LEVEL) = 'MOLECULE'
) T6 ON T1.MARKET = T6.MARKET AND T1.CMPS_CODE = T6.CMPS_COD
LEFT JOIN ( --COUNTINGUNIT--MOLECULE
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'COUNTINGUNIT' AND UPPER(LEVEL) = 'MOLECULE'
) T7 ON T1.MARKET = T7.MARKET
AND T1.CMPS_CODE = T7.CMPS_COD

71
CHPA/03 DM_TD_EXT_CHPA_MARKET_PACK_MAPPING.sql Normal file
View File

@@ -0,0 +1,71 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_CHPA_MARKET_PACK_MAPPING
-- (
-- PACK_CODE STRING,
-- MARKET STRING,
-- CLASS STRING,
-- KEY_COMPETITOR STRING,
-- MARKET_RATIO STRING,
-- STARTTIME STRING,
-- ENDTIME STRING,
-- VALUE_BRAND_RATIO STRING,
-- VALUE_BRAND_RATIO_START STRING,
-- VALUE_BRAND_RATIO_END STRING,
-- UNIT_BRAND_RATIO STRING,
-- UNIT_BRAND_START STRING,
-- UNIT_BRAND_END STRING,
-- COUNTINGUNIT_BRAND_RATIO STRING,
-- COUNTINGUNIT_BRAND_START STRING,
-- COUNTINGUNIT_BRAND_END STRING,
-- PDOT_BRAND_RATIO STRING,
-- PDOT_BRAND_START STRING,
-- PDOT_BRAND_END STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP
-- )
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chpa_market_pack_mapping';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chpa_market_pack_mapping';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_CHPA_MARKET_PACK_MAPPING
SELECT DISTINCT
T1.PACK_CODE,
T1.MARKET,
T4.CLASS,
T5.KEY_COMPETITOR,
T2.MARKET_RATIO,
T2.STARTTIME,
T2.ENDTIME,
T3.VALUE_BRAND_RATIO,
T3.VALUE_BRAND_RATIO_START,
T3.VALUE_BRAND_RATIO_END,
T3.UNIT_BRAND_RATIO,
T3.UNIT_BRAND_START,
T3.UNIT_BRAND_END,
T3.COUNTINGUNIT_BRAND_RATIO,
T3.COUNTINGUNIT_BRAND_START,
T3.COUNTINGUNIT_BRAND_END,
T3.PDOT_BRAND_RATIO,
T3.PDOT_BRAND_START,
T3.PDOT_BRAND_END,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
FROM
DM.DM_TD_EXT_CHPA_PACK2MARKET T1
LEFT JOIN DM.DM_TD_EXT_CHPA_MARKET_RATIO T2
ON T1.MARKET = T2.MARKET
AND T1.PACK_CODE = T2.PACK_CODE
LEFT JOIN DM.DM_TD_EXT_CHPA_MARKET_BRAND_RATIO T3
ON T1.MARKET = T3.MARKET
AND T1.PACK_CODE = T3.PACK_CODE
LEFT JOIN DM.DM_TD_EXT_CHPA_PACK2MARKET_CLASS T4
ON T1.PACK_CODE = T4.PACK_CODE
AND T1.MARKET = T4.MARKET
LEFT JOIN DM.DM_TD_EXT_CHPA_PACK2MARKET_KEYCOMPETITOR T5
ON T1.PACK_CODE = T5.PACK_CODE
AND T1.MARKET = T5.MARKET
WHERE
UPPER(T1.MARKET) NOT LIKE "%ALL%"

30
CHPA/03 DM_TD_EXT_CHPA_MARKET_RATIO.sql Normal file
View File

@@ -0,0 +1,30 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_CHPA_MARKET_RATIO (
-- MARKET STRING,
-- PACK_CODE STRING,
-- CMPS_CODE STRING,
-- STARTTIME STRING,
-- ENDTIME STRING,
-- MARKET_RATIO STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chpa_market_ratio';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chpa_market_ratio';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_CHPA_MARKET_RATIO
SELECT
DISTINCT
MARKET,
PACK_CODE,
CMPS_CODE,
STARTTIME,
ENDTIME,
MARKET_RATIO,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_CHPA_PACK2MARKET
WHERE UPPER(MARKET) NOT LIKE '%ALL%'

327
CHPA/03 DM_TD_EXT_CHPA_PACK2MARKET.sql Normal file
View File

@@ -0,0 +1,327 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_CHPA_PACK2MARKET (
-- MARKET STRING,
-- PACK_CODE STRING,
-- PACK_DESC STRING,
-- STGH_DESC STRING,
-- PACK_LCH STRING,
-- PROD_CODE STRING,
-- CMPS_CODE STRING,
-- CMPS_DESC STRING,
-- ATC1_CODE STRING,
-- ATC2_CODE STRING,
-- ATC3_CODE STRING,
-- ATC4_CODE STRING,
-- APP1_CODE STRING,
-- APP2_CODE STRING,
-- APP3_CODE STRING,
-- BIO_DESC STRING,
-- GENE_ORIG_DESC STRING,
-- ETH_OTC_DESC STRING,
-- NRDL_DESC STRING,
-- NRDL_ENTRY_DATE STRING,
-- EDL_DESC STRING,
-- TCM_DESC STRING,
-- PAED_DESC STRING,
-- GQCE_DESC STRING,
-- VBP_DESC STRING,
-- MANU_CODE STRING,
-- MANU_DESC STRING,
-- MNFL_CODE STRING,
-- MNFL_DESC STRING,
-- CORP_CODE STRING,
-- CORP_DESC STRING,
-- BRANDTYPE STRING,
-- BU STRING,
-- STARTTIME STRING,
-- ENDTIME STRING,
-- MARKET_RATIO STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chpa_pack2market';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chpa_pack2market';
-- COMMAND ----------
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_CHPA_PACK2MARKET_KEYCOMPETITOR (
-- KEY_COMPETITOR STRING,
-- MARKET STRING,
-- PACK_CODE STRING)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chpa_pack2market_keycompetitor';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chpa_pack2market_keycompetitor';
-- COMMAND ----------
-- CREATE or REPLACE TABLE DM.DM_TD_EXT_CHPA_PACK2MARKET_CLASS (
-- MARKET STRING,
-- PACK_CODE STRING,
-- CLASS STRING)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chpa_pack2market_class';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chpa_pack2market_class';
-- COMMAND ----------
-- 使用 EXCEPT（集合差操作）简化逻辑
INSERT OVERWRITE TABLE DM.DM_TD_EXT_CHPA_PACK2MARKET
-- 包含规则
SELECT DISTINCT
T2.MARKET,
T1.PACK_CODE,
T1.PACK_DESC,
T1.STGH_DESC,
T1.PACK_LCH,
T1.PROD_CODE,
T1.CMPS_CODE,
T1.CMPS_DESC,
T1.ATC1_CODE,
T1.ATC2_CODE,
T1.ATC3_CODE,
T1.ATC4_CODE,
T1.APP1_CODE,
T1.APP2_CODE,
T1.APP3_CODE,
T1.BIO_DESC,
T1.GENE_ORIG_DESC AS GENE_ORIG,
T1.ETH_OTC_DESC AS ETH_OTC_D,
T1.NRDL_DESC,
T1.NRDL_ENTRY_DATE AS NRDL_ENTR,
T1.EDL_DESC,
T1.TCM_DESC,
T1.PAED_DESC,
T1.GQCE_DESC,
T1.VBP_DESC,
T1.MANU_CODE,
T1.MANU_DESC,
T1.MNFL_CODE,
T1.MNFL_DESC,
T1.CORP_CODE,
T1.CORP_DESC,
T1.BRANDTYPE,
T2.BU,
T2.STARTTIME,
T2.ENDTIME,
COALESCE(T2.EXTEND_MARKET_RATIO, '1') AS MARKET_RATIO,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_CHPA_PACK_PROPERTY AS T1
LEFT JOIN (
SELECT
MARKET,
BU,
STARTTIME,
ENDTIME,
EXTEND_MARKET_RATIO,
ATC1_CODE,
ATC2_CODE,
ATC3_CODE,
ATC4_CODE,
NFC1_CODE,
NFC2_CODE,
NFC3_CODE,
CORPORATION_CODE,
MANUFACTURER_CODE,
PRODUCT_CODE,
PACK_CODE,
STRENGTH,
MOLECULE_CODE
FROM DWD.DWD_GND_IMS_TBLMARKET_BYMONTH
WHERE EXTEND_MARKET IS NULL AND (NOT_IN_FLAG IS NULL OR NOT_IN_FLAG = '1')
) AS T2
ON (T2.ATC1_CODE IS NULL OR T1.ATC1_CODE = T2.ATC1_CODE)
AND (T2.ATC2_CODE IS NULL OR T1.ATC2_CODE = T2.ATC2_CODE)
AND (T2.ATC3_CODE IS NULL OR T1.ATC3_CODE = T2.ATC3_CODE)
AND (T2.ATC4_CODE IS NULL OR T1.ATC4_CODE = T2.ATC4_CODE)
AND (T2.NFC1_CODE IS NULL OR T1.APP1_CODE = T2.NFC1_CODE)
AND (T2.NFC2_CODE IS NULL OR T1.APP2_CODE = T2.NFC2_CODE)
AND (T2.NFC3_CODE IS NULL OR T1.APP3_CODE = T2.NFC3_CODE)
AND (T2.CORPORATION_CODE IS NULL OR T1.CORP_CODE = T2.CORPORATION_CODE)
AND (T2.MANUFACTURER_CODE IS NULL OR T1.MANU_CODE = T2.MANUFACTURER_CODE)
AND (T2.PRODUCT_CODE IS NULL OR T1.PROD_CODE = T2.PRODUCT_CODE)
AND (T2.PACK_CODE IS NULL OR T1.PACK_CODE = T2.PACK_CODE)
AND (T2.STRENGTH IS NULL OR T1.STGH_DESC = T2.STRENGTH)
AND (T2.MOLECULE_CODE IS NULL OR T1.CMPS_CODE = T2.MOLECULE_CODE)
WHERE T2.MARKET IS NOT NULL
EXCEPT
-- 排除规则
SELECT DISTINCT
T2.MARKET,
T1.PACK_CODE,
T1.PACK_DESC,
T1.STGH_DESC,
T1.PACK_LCH,
T1.PROD_CODE,
T1.CMPS_CODE,
T1.CMPS_DESC,
T1.ATC1_CODE,
T1.ATC2_CODE,
T1.ATC3_CODE,
T1.ATC4_CODE,
T1.APP1_CODE,
T1.APP2_CODE,
T1.APP3_CODE,
T1.BIO_DESC,
T1.GENE_ORIG_DESC AS GENE_ORIG,
T1.ETH_OTC_DESC AS ETH_OTC_D,
T1.NRDL_DESC,
T1.NRDL_ENTRY_DATE AS NRDL_ENTR,
T1.EDL_DESC,
T1.TCM_DESC,
T1.PAED_DESC,
T1.GQCE_DESC,
T1.VBP_DESC,
T1.MANU_CODE,
T1.MANU_DESC,
T1.MNFL_CODE,
T1.MNFL_DESC,
T1.CORP_CODE,
T1.CORP_DESC,
T1.BRANDTYPE,
T2.BU,
T2.STARTTIME,
T2.ENDTIME,
COALESCE(T2.EXTEND_MARKET_RATIO, '1') AS MARKET_RATIO,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_CHPA_PACK_PROPERTY AS T1
LEFT JOIN (
SELECT
MARKET,
BU,
STARTTIME,
ENDTIME,
EXTEND_MARKET_RATIO,
ATC1_CODE,
ATC2_CODE,
ATC3_CODE,
ATC4_CODE,
NFC1_CODE,
NFC2_CODE,
NFC3_CODE,
CORPORATION_CODE,
MANUFACTURER_CODE,
PRODUCT_CODE,
PACK_CODE,
STRENGTH,
MOLECULE_CODE
FROM DWD.DWD_GND_IMS_TBLMARKET_BYMONTH
WHERE EXTEND_MARKET IS NULL AND NOT_IN_FLAG = '0'
) AS T2
ON (T2.ATC1_CODE IS NULL OR T1.ATC1_CODE = T2.ATC1_CODE)
AND (T2.ATC2_CODE IS NULL OR T1.ATC2_CODE = T2.ATC2_CODE)
AND (T2.ATC3_CODE IS NULL OR T1.ATC3_CODE = T2.ATC3_CODE)
AND (T2.ATC4_CODE IS NULL OR T1.ATC4_CODE = T2.ATC4_CODE)
AND (T2.NFC1_CODE IS NULL OR T1.APP1_CODE = T2.NFC1_CODE)
AND (T2.NFC2_CODE IS NULL OR T1.APP2_CODE = T2.NFC2_CODE)
AND (T2.NFC3_CODE IS NULL OR T1.APP3_CODE = T2.NFC3_CODE)
AND (T2.CORPORATION_CODE IS NULL OR T1.CORP_CODE = T2.CORPORATION_CODE)
AND (T2.MANUFACTURER_CODE IS NULL OR T1.MANU_CODE = T2.MANUFACTURER_CODE)
AND (T2.PRODUCT_CODE IS NULL OR T1.PROD_CODE = T2.PRODUCT_CODE)
AND (T2.PACK_CODE IS NULL OR T1.PACK_CODE = T2.PACK_CODE)
AND (T2.STRENGTH IS NULL OR T1.STGH_DESC = T2.STRENGTH)
AND (T2.MOLECULE_CODE IS NULL OR T1.CMPS_CODE = T2.MOLECULE_CODE)
WHERE T2.MARKET IS NOT NULL;
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_CHPA_PACK2MARKET_KEYCOMPETITOR
WITH kc AS (
SELECT
keycompetitor, no,
ROW_NUMBER() OVER (ORDER BY
CASE
WHEN ATC1_Code IS NOT NULL THEN 1
WHEN ATC2_Code IS NOT NULL THEN 2
WHEN ATC3_Code IS NOT NULL THEN 3
WHEN ATC4_Code IS NOT NULL THEN 4
WHEN Molecule_Code IS NOT NULL THEN 5
WHEN Product_Code IS NOT NULL THEN 6
WHEN Pack_Code IS NOT NULL THEN 7
ELSE 999
END,
CASE WHEN NFC1_CODE IS NULL THEN 0 ELSE 1 END,
CASE WHEN NFC2_CODE IS NULL THEN 0 ELSE 1 END,
CASE WHEN NFC3_CODE IS NULL THEN 0 ELSE 1 END
) AS no1,
ATC1_Code, ATC2_Code, ATC3_Code, ATC4_Code,
NFC1_Code, NFC2_Code, NFC3_Code,
corporation_code, Manufacturer_Code,
Product_Code, Pack_Code, Strength, Molecule_Code, market
FROM dwd.dwd_gnd_ims_tblkeycompetitor
),
joined AS (
SELECT
/*+ BROADCAST(kc) */
COALESCE(kc.keycompetitor, 'OTHERS') AS Key_Competitor, -- ✅ UPDATE 逻辑内联
t1.market, t1.PACK_CODE
/*bug 调试区
,
kc.no,
kc.no1,
t1.PROD_COD, t1.CMPS_COD, t1.CMPS_DES,
t1.ATC1_COD, t1.ATC2_COD, t1.ATC3_COD, t1.ATC4_COD,
t1.APP1_COD, t1.APP2_COD, t1.APP3_COD,
t1.BIO_DESC, t1.GENE_ORIG_DESC, t1.ETH_OTC_DESC,
t1.NRDL_DESC, t1.NRDL_Entry_Date, t1.EDL_DESC, t1.TCM_DESC,
t1.PAED_DESC, t1.GQCE_DESC, t1.VBP_DESC,
t1.MANU_COD, t1.MANU_DES, t1.MNFL_COD, t1.MNFL_DES,
t1.CORP_COD, t1.CORP_DES, t1.BrandType, t1.bu, t1.Market_Ratio
*/
,ROW_NUMBER() OVER (
PARTITION BY t1.market, t1.PACK_CODE, t1.PROD_CODE
ORDER BY kc.no1 DESC, kc.no DESC
) AS _rn
FROM DM.DM_TD_EXT_CHPA_PACK2MARKET t1
LEFT JOIN kc
ON (kc.ATC1_Code IS NULL OR t1.ATC1_CODE = kc.ATC1_Code)
AND (kc.ATC2_Code IS NULL OR t1.ATC2_CODE = kc.ATC2_Code)
AND (kc.ATC3_Code IS NULL OR t1.ATC3_CODE = kc.ATC3_Code)
AND (kc.ATC4_Code IS NULL OR t1.ATC4_CODE = kc.ATC4_Code)
AND (kc.NFC1_Code IS NULL OR t1.APP1_CODE = kc.NFC1_Code)
AND (kc.NFC2_Code IS NULL OR t1.APP2_CODE = kc.NFC2_Code)
AND (kc.NFC3_Code IS NULL OR t1.APP3_CODE = kc.NFC3_Code)
AND (kc.corporation_code IS NULL OR t1.CORP_CODE = kc.corporation_code)
AND (kc.Manufacturer_Code IS NULL OR t1.MANU_CODE = kc.Manufacturer_Code)
AND (kc.Product_Code IS NULL OR t1.PROD_CODE = kc.Product_Code)
AND (kc.Pack_Code IS NULL OR t1.PACK_CODE = kc.Pack_Code)
AND (kc.Strength IS NULL OR t1.STGH_DESC = kc.Strength)
AND (kc.Molecule_Code IS NULL OR t1.CMPS_CODE = kc.Molecule_Code)
AND (kc.market IS NULL OR t1.market = kc.market)
)
SELECT distinct Key_Competitor,market,pack_code
FROM joined
WHERE _rn = 1;
-- COMMAND ----------
-- DBTITLE 1,CLASS
INSERT OVERWRITE TABLE DM.DM_TD_EXT_CHPA_PACK2MARKET_CLASS
SELECT
/*+ BROADCAST(T2) */
DISTINCT
T1.MARKET,
T1.PACK_CODE,
COALESCE( T2.CLASS, 'OTHERS') AS CLASS
FROM DM.DM_TD_EXT_CHPA_PACK2MARKET T1
LEFT JOIN DWD.DWD_GND_TBLCLASS T2
ON (T2.MARKET IS NULL OR T1.MARKET = T2.MARKET)
AND (T2.PACK_CODE IS NULL OR T1.PACK_CODE = T2.PACK_CODE)
AND (T2.PRODUCT_CODE IS NULL OR T1.PROD_CODE = T2.PRODUCT_CODE)
AND (T2.MOLECULE_CODE IS NULL OR T1.CMPS_CODE = T2.MOLECULE_CODE)
AND (T2.CORPORATION_CODE IS NULL OR T1.CORP_CODE = T2.CORPORATION_CODE)
AND (T2.MANUFACTURER_CODE IS NULL OR T1.MANU_CODE = T2.MANUFACTURER_CODE)
AND (T2.ATC1_CODE IS NULL OR T1.ATC1_CODE = T2.ATC1_CODE)
AND (T2.ATC2_CODE IS NULL OR T1.ATC2_CODE = T2.ATC2_CODE)
AND (T2.ATC3_CODE IS NULL OR T1.ATC3_CODE = T2.ATC3_CODE)
AND (T2.ATC4_CODE IS NULL OR T1.ATC4_CODE = T2.ATC4_CODE)
AND (T2.NFC1_CODE IS NULL OR T1.APP1_CODE = T2.NFC1_CODE)
AND (T2.NFC2_CODE IS NULL OR T1.APP2_CODE = T2.NFC2_CODE)
AND (T2.NFC3_CODE IS NULL OR T1.APP3_CODE = T2.NFC3_CODE)
AND (T2.STRENGTH IS NULL OR T1.STGH_DESC = T2.STRENGTH)

147
CHPA/03 DM_TD_EXT_CHPA_PACK_PROPERTY.sql Normal file
View File

@@ -0,0 +1,147 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_CHPA_PACK_PROPERTY
-- (
-- -- MARKET_PACK_KEY STRING,
-- PACK_CODE STRING,
-- PACK_DESC STRING,
-- STGH_DESC STRING,
-- PACK_LCH STRING,
-- FAMILY_CODE STRING,
-- FAMILY_NAME STRING,
-- PROD_CODE STRING,
-- PROD_DESC STRING,
-- PROD_DESC_C STRING,
-- CMPS_CODE STRING,
-- CMPS_DESC STRING,
-- CMPS_DESC_C STRING,
-- ATC1_CODE STRING,
-- ATC2_CODE STRING,
-- ATC3_CODE STRING,
-- ATC4_CODE STRING,
-- APP1_CODE STRING,
-- APP2_CODE STRING,
-- APP3_CODE STRING,
-- BIO_DESC STRING,
-- GENE_ORIG_DESC STRING,
-- ETH_OTC_DESC STRING,
-- NRDL_DESC STRING,
-- NRDL_ENTRY_DATE STRING,
-- EDL_DESC STRING,
-- TCM_DESC STRING,
-- PAED_DESC STRING,
-- GQCE_DESC STRING,
-- VBP_DESC_V STRING,
-- VBP_DESC STRING,
-- MANU_CODE STRING,
-- MANU_DESC STRING,
-- MANU_DESC_C STRING,
-- MNFL_CODE STRING,
-- MNFL_DESC STRING,
-- CORP_CODE STRING,
-- CORP_DESC STRING,
-- CORP_DESC_C STRING,
-- BRANDTYPE STRING,
-- -- MARKET STRING,
-- -- KEY_COMPETITOR STRING,
-- IS_AZ STRING,
-- AZ_MAIN STRING,
-- -- AZ_RELATED STRING,
-- ATC1_DESC STRING,
-- ATC1_DESC_C STRING,
-- ATC2_DESC STRING,
-- ATC2_DESC_C STRING,
-- ATC3_DESC STRING,
-- ATC3_DESC_C STRING,
-- ATC4_DESC STRING,
-- ATC4_DESC_C STRING,
-- APP1_DESC STRING,
-- APP1_DESC_C STRING,
-- APP2_DESC STRING,
-- APP2_DESC_C STRING,
-- APP3_DESC STRING,
-- APP3_DESC_C STRING,
-- -- CLASS STRING,
-- -- MARKET_RATIO STRING,
-- COUNTINGUNIT STRING,
-- VBP_BRAND STRING,
-- REPLENISH_FALG STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP
-- )
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chpa_pack_property';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chpa_pack_property';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_CHPA_PACK_PROPERTY
SELECT
-- MARKET_PACK_KEY,
PACK_COD AS PACK_CODE,
PACK_DES AS PACK_DESC,
STGH_DES AS STGH_DESC,
PACK_LCH,
FAMILY_CODE,
FAMILY_NAME,
PROD_COD AS PROD_CODE,
PROD_DES AS PROD_DESC,
PROD_DES_C AS PROD_DESC_C,
CMPS_COD AS CMPS_CODE,
CMPS_DES AS CMPS_DESC,
CMPS_DES_C AS CMPS_DESC_C,
ATC1_COD AS ATC1_CODE,
ATC2_COD AS ATC2_CODE,
ATC3_COD AS ATC3_CODE,
ATC4_COD AS ATC4_CODE,
APP1_COD AS APP1_CODE,
APP2_COD AS APP2_CODE,
APP3_COD AS APP3_CODE,
BIO_DESC,
GENE_ORIG_DESC,
ETH_OTC_DESC,
NRDL_DESC,
NRDL_ENTRY_DATE,
EDL_DESC,
TCM_DESC,
PAED_DESC,
GQCE_DESC,
VBP_DESC_V,
VBP_DESC,
MANU_COD AS MANU_CODE,
MANU_DES AS MANU_DESC,
MANU_DES_C AS MANU_DESC_C,
MNFL_COD AS MNFL_CODE,
MNFL_DES AS MNFL_DESC,
CORP_COD AS CORP_CODE,
CORP_DES AS CORP_DESC,
CORP_DES_C AS CORP_DESC_C,
BRANDTYPE,
-- MARKET,
-- KEY_COMPETITOR,
IS_AZ,
AZ_MAIN,
-- AZ_RELATED,
ATC1_DES AS ATC1_DESC,
ATC1_DES_C AS ATC1_DESC_C,
ATC2_DES AS ATC2_DESC,
ATC2_DES_C AS ATC2_DESC_C,
ATC3_DES AS ATC3_DESC,
ATC3_DES_C AS ATC3_DESC_C,
ATC4_DES AS ATC4_DESC,
ATC4_DES_C AS ATC4_DESC_C,
APP1_DES AS APP1_DESC,
APP1_DES_C AS APP1_DESC_C,
APP2_DES AS APP2_DESC,
APP2_DES_C AS APP2_DESC_C,
APP3_DES AS APP3_DESC,
APP3_DES_C AS APP3_DESC_C,
-- CLASS,
-- NULL AS MARKET_RATIO,
NULL AS COUNTINGUNIT,
NULL AS VBP_BRAND,
NULL AS REPLENISH_FALG,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_IMS_TD_PACK_PROPERTY
;

166
CHPA/03 DM_TF_EXT_CHPA_SALES.sql Normal file
View File

@@ -0,0 +1,166 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TF_EXT_CHPA_SALES (
-- YYYYMM STRING,
-- PACK_CODE STRING,
-- CORP_CODE STRING,
-- AUDIT_CODE STRING,
-- PLATFORM_TYPE STRING,
-- STORE_NAME STRING,
-- STORE_TYPE STRING,
-- REGION_TYPE STRING,
-- PACK_FLAG INT,
-- PROD_FLAG INT,
-- DTP_FLAG INT,
-- SALES_UNIT_CAL DECIMAL(38,10),
-- SALES_UNIT_CAL_LY DECIMAL(38,10),
-- SALES_VALUE_CAL DECIMAL(38,10),
-- SALES_VALUE_CAL_LY DECIMAL(38,10),
-- CONUTING_UNIT DECIMAL(38,10),
-- CONUTING_UNIT_LY DECIMAL(38,10),
-- DATA_SOURCE STRING,
-- INST_CODE STRING COMMENT '内部机构编码',
-- CMPS_FLAG STRING COMMENT '分子式标签',
-- DEPT_NAME STRING COMMENT '科室名称',
-- PRESCRIPTION DECIMAL(38,10) COMMENT '处方张数',
-- PRESCRIPTION_LY DECIMAL(38,10) COMMENT '去年同期处方张数',
-- NEW_CODE STRING COMMENT '主数据关联CODE',
-- AREA STRING COMMENT '城市',
-- H_LEVEL STRING COMMENT '医院类型',
-- REIMBURSE STRING COMMENT '报销情况',
-- REIMBURSE_TYPE STRING COMMENT '报销类型',
-- PRESCRIPTION_SOURCE STRING COMMENT '处方来源',
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_chpa_sales';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_chpa_sales';
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW CHPA_SALES AS
SELECT
A.YYYYMM,
A.PACK_COD AS PACK_CODE,
A.AUDIT_COD AS AUDIT_CODE,
'IQVIA-CHPA(Monthly)' DATA_SOURCE,
-------------------------------------
SUM(SALES_UNIT_CAL) SALES_UNIT_CAL,
SUM(SALES_UNIT_CAL_LY) as SALES_UNIT_CAL_LY,
SUM(SALES_VALUE_CAL) SALES_VALUE_CAL,
SUM(SALES_VALUE_CAL_LY) as SALES_VALUE_CAL_LY,
SUM(CONUTING_UNIT) CONUTING_UNIT,
SUM(CONUTING_UNIT_LY) as CONUTING_UNIT_LY,
null as prescription,
null as prescription_ly,
-------------------------------------
--Retail藏数逻辑标签------------------
1 PACK_FLAG,
1 PROD_FLAG,
0 DTP_FLAG,
null as cmps_flag,
-------------------------------------
--EC数据标签--------------------------
'' PLATFORM_TYPE,
'' STORE_NAME,
'' STORE_TYPE,
-------------------------------------
--COUNTY数据标签----------------------
'' REGION_TYPE,
-------------------------------------
--AIA数据标签-------------------------
null inst_code,
-------------------------------------
--XIE HE 数据标签---------------------
'' as dept_name,
'' as new_code,
'' as area,
'' as h_level,
'' as reimburse,
'' as reimburse_type,
'' as prescription_source,
-------------------------------------
'' as CORP_CODE --可以移除，报告没有使用
-------------------------------------
FROM
DM.DM_IMS_TF_SALES A
where
A.YYYYMM >= '202201'
-- AND A.MARKET = 'IMS ALL Market'
GROUP BY
A.YYYYMM,
A.PACK_COD,
A.AUDIT_COD
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TF_EXT_CHPA_SALES (
YYYYMM,
PACK_CODE,
AUDIT_CODE,
DATA_SOURCE,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
PRESCRIPTION,
PRESCRIPTION_LY,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
CMPS_FLAG,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
INST_CODE,
DEPT_NAME,
NEW_CODE,
AREA,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE,
CORP_CODE,
ETL_INSERT_DT,
ETL_UPDATE_DT
)
SELECT
YYYYMM,
NVL (
NULLIF(PACK_CODE, ''),
CONCAT ('PACK_CODE_', DATA_SOURCE)
) AS PACK_CODE,
NVL (NULLIF(AUDIT_CODE, ''), 'ROC') AS AUDIT_CODE,
DATA_SOURCE,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
PRESCRIPTION,
PRESCRIPTION_LY,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
CMPS_FLAG,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
INST_CODE,
DEPT_NAME,
NEW_CODE,
AREA,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE,
CORP_CODE,
FROM_UTC_TIMESTAMP (CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP (CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
FROM
CHPA_SALES;

58
CHPA/03 dm_ims_td_calendar.sql Normal file
View File

@@ -0,0 +1,58 @@
-- Databricks notebook source
-- 源表：DWS.DWS_IMS_TD_DATE
-- 目标表：DM.DM_IMS_TD_CALENDAR,DM.DM_IMS_S_DATE
-- COMMAND ----------
--DM 逻辑
--DATE
INSERT OVERWRITE TABLE DM.DM_IMS_TD_CALENDAR
--当前年销量日期日历表缺少的用去年补全
SELECT
CAST(YM+100 AS INT) AS YYYYMM,
CAST(YEAR+1 AS INT) AS YEAR,
MONTH,
QUARTER,
CONCAT(CAST(LEFT(YQ,4) AS INT)+1,QUARTER) AS YQ ,
CAST(CONCAT(CAST(YEAR AS INT)+1,'-',MONTH,'-01') AS DATE) AS DAY_DATE,
CASE WHEN YM+100= (SELECT MAX(YM) FROM TMP.TMP_IMS_TF_FACT_SALES) THEN 'R' ELSE DATE_FLAG END,
CONCAT(CAST(LEFT(HALF_YEAR,4) AS INT)+1,RIGHT(HALF_YEAR,2)) AS HALF_YEAR,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') ETL_UPDATE_DT
FROM DWS.DWS_IMS_TD_DATE
WHERE YEAR+1=(SELECT MAX(YEAR) FROM DWS.DWS_IMS_TD_DATE)
--前四年的日历表
UNION ALL
SELECT
CAST(YM AS INT) AS YYYYMM,
CAST(YEAR AS INT),
MONTH,
QUARTER,
YQ,
CAST(CONCAT(YEAR,'-',MONTH,'-01') AS DATE) AS DAY_DATE,
DATE_FLAG,
HALF_YEAR,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') ETL_UPDATE_DT
FROM DWS.DWS_IMS_TD_DATE
WHERE YEAR<(SELECT MAX(YEAR) FROM DWS.DWS_IMS_TD_DATE)
AND YEAR+9>(SELECT MAX(YEAR) FROM DWS.DWS_IMS_TD_DATE)
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_IMS_S_DATE
SELECT
YYYYMM,
YEAR,
MONTH,
QUARTER,
YQ,
DAY_DATE,
DATE_FLAG,
HALF_YEAR,
ETL_INSERT_DT,
ETL_UPDATE_DT
FROM DM.DM_IMS_TD_CALENDAR

67
CHPA/03 dm_ims_td_geo.sql Normal file
View File

@@ -0,0 +1,67 @@
-- Databricks notebook source
-- 源表：DM.DM_TD_GEOGRAPHY,DWS.DWS_IMS_TD_GEO
-- 目标表：DM.DM_IMS_TD_GEO
-- COMMAND ----------
-- 20260119 地理数据 我感觉可以拉到前面去
--地理维度信息 AUDIT
-- 20260122 因为城市数据不再提供,改为硬编码省份数据. BY YUSUO/CHENWU
INSERT OVERWRITE TABLE DWS.DWS_IMS_TD_GEO
SELECT
AUDIT_COD,
AUDIT_DES,
AUDIT_DES_C,
AUDIT_TYPE,
CITY_TIER,
AZ_CITY_TIER,
PROVINCE,
PROVINCE_C,
REGIONCENTER,
ETL_INSERT_DT,
ETL_UPDATE_DT
FROM
TMP.TMP_PROVINCE_RAWDATA
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW DIM_PROVINCE
AS
SELECT
T2.PROVINCE_CODE,
CASE
WHEN T2.PROVINCE_NAME IN ('上海', '北京', '天津', '重庆') THEN CONCAT (T2.PROVINCE_NAME, '市')
WHEN T2.PROVINCE_NAME IN ('内蒙古', '西藏') THEN CONCAT (T2.PROVINCE_NAME, '自治区')
WHEN T2.PROVINCE_NAME = '宁夏' THEN CONCAT (T2.PROVINCE_NAME, '回族自治区')
WHEN T2.PROVINCE_NAME = '广西' THEN CONCAT (T2.PROVINCE_NAME, '壮族自治区')
WHEN T2.PROVINCE_NAME = '新疆' THEN CONCAT (T2.PROVINCE_NAME, '维吾尔自治区')
ELSE CONCAT (T2.PROVINCE_NAME, '省')
END AS PROVINCE_NAME
FROM
DM.DM_TD_GEOGRAPHY T2
WHERE
T2.GEO_KEY = T2.PROVINCE_CODE
AND PROVINCE_NAME NOT IN ('台湾', '澳门', '香港')
-- COMMAND ----------
--DM 地理纬度表
INSERT OVERWRITE TABLE DM.DM_IMS_TD_GEO
SELECT DISTINCT
T1.AUDIT_COD,
T1.AUDIT_DES AS CITY,
T1.AUDIT_DES_C AS CITY_C,
T1.AUDIT_TYPE AS CITY_TYPE,
T1.CITY_TIER,
T1.AZ_CITY_TIER,
T1.PROVINCE,
T1.PROVINCE_C,
T2.PROVINCE_CODE,
T2.PROVINCE_NAME AS CITY_MAP,
T2.PROVINCE_NAME,
T1.REGIONCENTER AS REGION_CENTER,
T1.ETL_INSERT_DT,
T1.ETL_UPDATE_DT
FROM
DWS.DWS_IMS_TD_GEO AS T1
LEFT JOIN DIM_PROVINCE AS T2 ON T1.AUDIT_COD = T2.PROVINCE_CODE

35
CHPA/03 dm_ims_td_market_property.sql Normal file
View File

@@ -0,0 +1,35 @@
-- Databricks notebook source
-- 源表：DWD.DWD_TD_IMS_MARKET_TA_ORDER
-- 目标表：DM.DM_IMS_TD_MARKET_PROPERTY
-- COMMAND ----------
update dwd.dwd_td_ims_market_ta_order
set
market = 'Pulmicort VBP Benchmark Market'
where
market = 'Pumicort VBP Benchmark Market'
-- COMMAND ----------
--dm market property 市场相关属性和market&ta 的排序 by qingli 20230425
insert overwrite table dm.dm_ims_td_market_property
select
market_order,
bu as bu,
ta as ta_key,
case
when ta = 'ONC' then 'ONCO'
else ta
end as ta,
market as market_key,
case
when market = 'ONC Market' then 'ONCO Market'
else market
end as market,
null as svg,
ta_order,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_INSERT_DT,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_UPDATE_DT
from
dwd.dwd_td_ims_market_ta_order

615
CHPA/03 dm_ims_td_org.sql Normal file
View File

@@ -0,0 +1,615 @@
-- Databricks notebook source
-- 源表：DM.dm_td_org,dm.dm_td_territory_indication,dm.dm_td_date_config,dm.dm_ims_tf_sales
-- 目标表：dm.dm_ims_td_org
-- COMMAND ----------
-- MAGIC %run ../../../Common/config
-- COMMAND ----------
-- MAGIC %python
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dim_product_wide`').createOrReplaceTempView('cdw_dwd_dim_product_wide')
-- COMMAND ----------
create or replace temporary view dm_td_org_tmp as
SELECT
org_key,
mr_trty_tag,
district_code,
rsm_name,
rc_name,
p2.region_name,
p2.rsd_kcode,
p2.rsd_name,
CASE
WHEN p2.yyyymm >= '202601'
and p2.bu_code = 'PC12' then 'PC09'
else p2.bu_code
end bu_code,
CASE
WHEN p2.yyyymm >= '202601'
and p2.bu_report = 'EC' THEN 'Retail'
ELSE p2.bu_report
end bu_report,
CASE
WHEN p2.yyyymm >= '202601'
and p2.sub_bu_code = 'NA27' then 'NA06'
else p2.sub_bu_code
end sub_bu_code,
CASE
WHEN p2.yyyymm >= '202601'
and p2.sub_bu_name = 'EC' then 'Retail'
else p2.sub_bu_name
end sub_bu_name,
CASE
WHEN p2.yyyymm >= '202601'
and p2.sub_bu_name_en = 'EC' then 'Retail'
else p2.sub_bu_name_en
end sub_bu_name_en,
CASE
WHEN p2.yyyymm >= '202601'
and p2.bu_name = 'EC' then 'Retail'
else p2.bu_name
end bu_name,
CASE
WHEN p2.yyyymm >= '202601'
and p2.bu_name_en = 'EC' then 'Retail'
else p2.bu_name_en
end bu_name_en,
CASE
WHEN p2.yyyymm >= '202601'
and p2.nsd_kcode = 'ND_EC0001_V' then 'KSGX559'
else p2.nsd_kcode
end nsd_kcode,
CASE
WHEN p2.yyyymm >= '202601'
and p2.bu_name = 'EC'
and p2.nsd_name = 'Vacant' then '李白杨'
else p2.nsd_name
end nsd_name,
CASE
WHEN p2.yyyymm >= '202601'
and p2.nsd_trty_code = 'ND_EC0001' then 'ND_RT0001'
else p2.nsd_trty_code
end nsd_trty_code,
CASE
WHEN p2.yyyymm >= '202601'
and p2.bsd_kcode = 'VP_EC0001_V' then 'VP_RT0001_V'
else p2.bsd_kcode
end bsd_kcode,
CASE
WHEN p2.yyyymm >= '202601'
and p2.bsd_trty_code = 'VP_EC0001' then 'VP_RT0001'
else p2.bsd_trty_code
end bsd_trty_code,
p2.rsm_kcode,
p2.rc_code,
p2.rc_name_en,
p2.region_code,
p2.region_name_en,
BU_CATEGORY,
yyyymm,
mr_trty_code,
mr_kcode,
dsm_trty_code,
rsm_trty_code,
dsm_kcode,
rsd_trty_code
FROM
DM.dm_td_org P2
-- COMMAND ----------
--dm_org --使用和performance cvh 的org架构
-- insert overwrite table tmp.tmp_ims_td_org
create or replace temporary view tmp_ims_td_org as
select distinct
rsm_trty_code,
rsm_kcode,
case
when rsm_name = 'Vacant' then concat (rsm_trty_code, '_V')
else rsm_name
end as rsm_name,
rsd_trty_code,
rsd_kcode,
case
when rsd_name = 'Vacant' then concat (rsd_trty_code, '_V')
else rsd_name
end as rsd_name,
nsd_trty_code,
nsd_kcode,
case
when nsd_name = 'Vacant' then concat (nsd_trty_code, '_V')
else nsd_name
end as nsd_name,
region_code,
region_name,
region_name_en,
rc_code,
rc_name,
rc_name_en,
bu_code,
bu_name,
bu_name_en,
bu_category,
sub_bu_code,
sub_bu_name,
sub_bu_name_en,
t2.city_name,
null as ims_pack_cod,
t3.market,
t2.province_code as ims_audit_cod,
1 as ims_city_ratio
from
dm_td_org_tmp t1
left join (
select distinct
t1.yyyymm,
t1.trty_code,
t2.city_name,
-- 20260210 Yusuo ADD
t2.province_code,
t4.ims_prod_cod,
t4.ims_pack_cod
from
dm.dm_td_territory_indication t1
left join dwd.dwd_td_institution t2 on t1.inst_code = t2.inst_code
left join cdw_dwd_dim_product_wide t3 on t3.prd_family_cd = t1.family_code
left join dwd.dwd_td_imscode_azcode t4 on t3.prd_brand_cd = t4.brandcode
where
t1.bu not in ('CUBE', 'KA')
) t2 on t2.trty_code = t1.mr_trty_code
and t2.yyyymm = t1.yyyymm
left join dws.dws_ims_td_market t3 on t2.ims_pack_cod = t3.pack_cod
--使用和performance cvh 的org架构
where
t1.yyyymm in (
select
key_value
from
dm.dm_td_date_config
where
key_code = 'pfmc_cvh'
)
-- COMMAND ----------
--将还没有做出来AZ产品关系的market挂给所有的对应BU
-- insert into tmp_ims_td_org
create or replace temporary view tmp_ims_td_org1 as
select distinct
'Vacant' as rsm_trty_code,
'Vacant' as rsm_kcode,
'Vacant' as rsm_name,
'Vacant' as rsd_trty_code,
'Vacant' as rsd_kcode,
'Vacant' as rsd_name,
'Vacant' as nsd_trty_code,
'Vacant' as nsd_kcode,
'Vacant' as nsd_name,
'Non AZ Region' as region_code,
'Non AZ Region' as region_name,
'Non AZ Region' as region_name_en,
'Non AZ Region' as rc_code,
'Non AZ Region' as rc_name,
'Non AZ Region' as rc_name_en,
bu_code,
bu_name,
bu_name_en,
bu_category,
sub_bu_code,
sub_bu_name,
sub_bu_name_en,
city_name,
null as ims_pack_cod,
t2.market,
ims_audit_cod,
ims_city_ratio
from
tmp_ims_td_org t1
left join (
select distinct
t2.market,
t2.bu
from
tmp_ims_td_org t1
right join dwd.dwd_td_ims_market_ta_order t2 on t1.market = t2.market
where
t1.market is null
and t2.market <> 'IMS ALL Market'
) t2 on t1.bu_name_en = t2.bu
where
t2.market is not null
-- COMMAND ----------
--将所有的ROC的市场数据补全，同时根据市场找到对应BU
-- insert into tmp_ims_td_org
create or replace temporary view tmp_ims_td_org2 as
select distinct
rsm_trty_code,
rsm_kcode,
rsm_name,
rsd_trty_code,
rsd_kcode,
rsd_name,
nsd_trty_code,
nsd_kcode,
nsd_name,
region_code,
region_name,
region_name_en,
rc_code,
rc_name,
rc_name_en,
t2.bu_code,
t2.bu_name,
t2.bu_name_en,
t2.bu_category,
t2.sub_bu_code,
t2.sub_bu_name,
t2.sub_bu_name_en,
t2.city_name,
null as ims_pack_cod,
t2.market,
t2.ims_audit_cod,
t2.ims_city_ratio
from
(
select
'Vacant' as rsm_trty_code,
'Vacant' as rsm_kcode,
'Vacant' as rsm_name,
'Vacant' as rsd_trty_code,
'Vacant' as rsd_kcode,
'Vacant' as rsd_name,
'Vacant' as nsd_trty_code,
'Vacant' as nsd_kcode,
'Vacant' as nsd_name,
'Other Low Tiers' as region_code,
'Other Low Tiers' as region_name,
'Other Low Tiers' as region_name_en,
'Other Low Tiers' as rc_code,
'Other Low Tiers' as rc_name,
'Other Low Tiers' as rc_name_en,
'Other Low Tiers' as city_name,
'ROC' as audit_cod
) t1
full join (
select distinct
bu_code,
bu_name,
bu_name_en,
bu_category,
sub_bu_code,
sub_bu_name,
sub_bu_name_en,
market,
ims_city_ratio,
'Other Low Tiers' as city_name,
'ROC' as ims_audit_cod
from
tmp_ims_td_org1
where
bu_code is not null
--有销量的ROC数据
and concat (market, '_ROC') in (
select distinct
concat(market,'_ROC' ) market_audit_key from
dm.dm_tf_ext_unionall_market_sales
where DATA_SOURCE = 'IQVIA-CHPA(Monthly)' and
audit_cod = 'ROC'
)
) t2 on t1.audit_cod = t2.ims_audit_cod
-- COMMAND ----------
--将所有未能挂上Org的市场数据补全，同时根据市场找到对应BU
-- insert into tmp_ims_td_org
create or replace temporary view tmp_ims_td_org3 as
select distinct
rsm_trty_code,
rsm_kcode,
rsm_name,
rsd_trty_code,
rsd_kcode,
rsd_name,
nsd_trty_code,
nsd_kcode,
nsd_name,
region_code,
region_name,
region_name_en,
rc_code,
rc_name,
rc_name_en,
t2.bu_code,
t2.bu_name,
t2.bu_name_en,
t2.bu_category,
t2.sub_bu_code,
t2.sub_bu_name,
t2.sub_bu_name_en,
t2.city_name,
null as ims_pack_cod,
t2.market,
t2.ims_audit_cod,
t2.ims_city_ratio
from
(
select
'Vacant' as rsm_trty_code,
'Vacant' as rsm_kcode,
'Vacant' as rsm_name,
'Vacant' as rsd_trty_code,
'Vacant' as rsd_kcode,
'Vacant' as rsd_name,
'Vacant' as nsd_trty_code,
'Vacant' as nsd_kcode,
'Vacant' as nsd_name,
'Non AZ Region' as region_code,
'Non AZ Region' as region_name,
'Non AZ Region' as region_name_en,
'Non AZ Region' as rc_code,
'Non AZ Region' as rc_name,
'Non AZ Region' as rc_name_en
) t1
full join (
select distinct
bu_code,
bu_name,
bu_name_en,
bu_category,
sub_bu_code,
sub_bu_name,
sub_bu_name_en,
-- 'ALL' as sub_bu_code,'ALL' as sub_bu_name,'ALL' as sub_bu_name_en,
t2.audit_cod as ims_audit_cod,
null as city_name,
t2.market,
ims_city_ratio
from
tmp_ims_td_org2 t1
inner join (
--筛选出挂不上Org的市场城市，同时又有销量的
select distinct
t2.market,
t2.audit_cod,
t3.bu
from
tmp_ims_td_org2 t1
right join (
select distinct
audit_cod,
market
from
dm.dm_tf_ext_unionall_market_sales
where DATA_SOURCE = 'IQVIA-CHPA(Monthly)'
) t2 on concat (t1.market, '_', t1.ims_audit_cod) = concat (t2.market, '_', t2.audit_cod)
left join dwd.dwd_td_ims_market_ta_order t3 on t2.market = t3.market
where
concat (t1.market, '_', t1.ims_audit_cod) is null
and t2.market <> 'IMS ALL Market'
) t2 on t1.bu_name_en = t2.bu
) t2
where
t2.bu_code is not null
-- COMMAND ----------
create or replace temporary view tmp_ims_td_org4 as
select
rsm_trty_code,
rsm_kcode,
rsm_name,
rsd_trty_code,
rsd_kcode,
rsd_name,
nsd_trty_code,
nsd_kcode,
nsd_name,
region_code,
region_name,
region_name_en,
rc_code,
rc_name,
rc_name_en,
bu_code,
bu_name,
bu_name_en,
bu_category,
sub_bu_code,
sub_bu_name,
sub_bu_name_en,
city_name,
ims_pack_cod,
market,
ims_audit_cod,
ims_city_ratio
from
tmp_ims_td_org
union all
select
rsm_trty_code,
rsm_kcode,
rsm_name,
rsd_trty_code,
rsd_kcode,
rsd_name,
nsd_trty_code,
nsd_kcode,
nsd_name,
region_code,
region_name,
region_name_en,
rc_code,
rc_name,
rc_name_en,
bu_code,
bu_name,
bu_name_en,
bu_category,
sub_bu_code,
sub_bu_name,
sub_bu_name_en,
city_name,
ims_pack_cod,
market,
ims_audit_cod,
ims_city_ratio
from
tmp_ims_td_org1
union all
select
rsm_trty_code,
rsm_kcode,
rsm_name,
rsd_trty_code,
rsd_kcode,
rsd_name,
nsd_trty_code,
nsd_kcode,
nsd_name,
region_code,
region_name,
region_name_en,
rc_code,
rc_name,
rc_name_en,
bu_code,
bu_name,
bu_name_en,
bu_category,
sub_bu_code,
sub_bu_name,
sub_bu_name_en,
city_name,
ims_pack_cod,
market,
ims_audit_cod,
ims_city_ratio
from
tmp_ims_td_org2
union all
select
rsm_trty_code,
rsm_kcode,
rsm_name,
rsd_trty_code,
rsd_kcode,
rsd_name,
nsd_trty_code,
nsd_kcode,
nsd_name,
region_code,
region_name,
region_name_en,
rc_code,
rc_name,
rc_name_en,
bu_code,
bu_name,
bu_name_en,
bu_category,
sub_bu_code,
sub_bu_name,
sub_bu_name_en,
city_name,
ims_pack_cod,
market,
ims_audit_cod,
ims_city_ratio
from
tmp_ims_td_org3
-- COMMAND ----------
insert overwrite table dm.dm_ims_td_org
select distinct
concat (t1.market, '_', t2.audit_cod) as ORG_KEY,
t1.rsm_trty_code,
t1.rsm_kcode,
t1.rsm_name,
t1.rsd_trty_code,
t1.rsd_kcode,
t1.rsd_name,
t1.nsd_trty_code,
t1.nsd_kcode,
t1.nsd_name,
t1.region_code,
t1.region_name,
t1.region_name_en,
t1.rc_code,
t1.rc_name,
t1.rc_name_en,
t1.bu_code,
t1.bu_name,
t1.bu_name_en,
t1.bu_category,
t1.sub_bu_code,
t1.sub_bu_name,
t1.sub_bu_name_en,
t1.sub_bu_code as sub_bu_code_auth, --前端权限配置需要的字段
t1.bu_code as bu_code_auth, --前端权限配置需要的字段
t2.audit_cod,
t2.audit_des,
t2.audit_des_c,
t2.Province,
t2.Province_C,
t2.RegionCenter,
t1.market,
concat (t1.market, '_', t2.audit_cod) as market_audit_key,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_INSERT_DT,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_UPDATE_DT
from
tmp_ims_td_org4 t1
left join dws.dws_ims_td_geo t2 on t1.ims_audit_cod = t2.audit_cod
--where t1.market is not null and t1.ims_audit_cod is not null
/* 20260409 chenwu IMS ALL Market 不需要了
union all
select distinct
concat ('IMS ALL Market_', t2.audit_cod) as ORG_KEY,
t1.rsm_trty_code,
t1.rsm_kcode,
t1.rsm_name,
t1.rsd_trty_code,
t1.rsd_kcode,
t1.rsd_name,
t1.nsd_trty_code,
t1.nsd_kcode,
t1.nsd_name,
t1.region_code,
t1.region_name,
t1.region_name_en,
t1.rc_code,
t1.rc_name,
t1.rc_name_en,
t1.bu_code,
t1.bu_name,
t1.bu_name_en,
t1.bu_category,
t1.sub_bu_code,
t1.sub_bu_name,
t1.sub_bu_name_en,
'ALL' as sub_bu_code_auth,
'ALL' as bu_code_auth,
t2.audit_cod,
t2.audit_des,
t2.audit_des_c,
t2.Province,
t2.Province_C,
t2.RegionCenter,
'IMS ALL Market' as market,
concat ('IMS ALL Market_', t2.audit_cod) as market_audit_key,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_INSERT_DT,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_UPDATE_DT
from
tmp_ims_td_org4 t1
left join dws.dws_ims_td_geo t2 on t1.ims_audit_cod = t2.audit_cod
where
t1.market is not null
and t1.ims_audit_cod is not null
*/

463
CHPA/03 dm_ims_td_org_hvh.sql Normal file
View File

@@ -0,0 +1,463 @@
-- Databricks notebook source
-- MAGIC %run ../../../Common/config
-- COMMAND ----------
-- MAGIC %python
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dim_product_wide`').createOrReplaceTempView('cdw_dwd_dim_product_wide')
-- COMMAND ----------
--dm_org --使用和performance cvh 的org架构
insert overwrite table tmp.tmp_ims_td_org_hvh
select distinct
t1.yyyymm,
dsm_trty_code,
dsm_kcode,
case
when dsm_name = 'Vacant' then concat (dsm_trty_code, '_V')
else dsm_name
end as dsm_name,
rsm_trty_code,
rsm_kcode,
case
when rsm_name = 'Vacant' then concat (rsm_trty_code, '_V')
else rsm_name
end as rsm_name,
rsd_trty_code,
rsd_kcode,
case
when rsd_name = 'Vacant' then concat (rsd_trty_code, '_V')
else rsd_name
end as rsd_name,
nsd_trty_code,
nsd_kcode,
case
when nsd_name = 'Vacant' then concat (nsd_trty_code, '_V')
else nsd_name
end as nsd_name,
district_code,
district_name,
district_name_en,
region_code,
region_name,
region_name_en,
rc_code,
rc_name,
rc_name_en,
bu_code,
bu_name,
bu_name_en,
bu_category,
sub_bu_code,
sub_bu_name,
sub_bu_name_en,
t2.city_name,
null as ims_pack_cod,
t3.market,
t2.province_code as ims_audit_cod,
--福厦泉 苏锡 珠三角 对应多个city的情况的拆分比
-- 20260210 Yusuo Update
-- case when t4.ratio is null then 1 else t4.ratio end as ims_city_ratio
1 as ims_city_ratio
from
dm.dm_td_org t1
left join (
select distinct
t1.yyyymm,
t1.trty_code,
t2.city_name,
t2.province_code,
t4.ims_prod_cod,
t4.ims_pack_cod
from
dm.dm_td_territory_indication t1
left join dwd.dwd_td_institution t2 on t1.inst_code = t2.inst_code
left join cdw_dwd_dim_product_wide t3 on t3.prd_family_cd = t1.family_code
left join dwd.dwd_td_imscode_azcode t4 on t3.prd_brand_cd = t4.brandcode
where
t1.bu not in ('CUBE', 'KA')
) t2 on t2.trty_code = t1.mr_trty_code
and t2.yyyymm = t1.yyyymm
left join dws.dws_ims_td_market t3 on t2.ims_pack_cod = t3.pack_cod
-- left join dws.dws_td_ims_city_mapping t4 on t2.city_name=concat(t4.city_c_d,'市')
--使用和performance cvh 的org架构
where
left (t1.yyyymm, 4) in (
select
left (key_value, 4)
from
dm.dm_td_date_config
where
key_code = 'pfmc_cvh'
)
-- COMMAND ----------
--将还没有做出来AZ产品关系的market挂给所有的对应BU
insert into
tmp.tmp_ims_td_org_hvh
select distinct
t1.yyyymm,
'Vacant' as dsm_trty_code,
'Vacant' as dsm_kcode,
'Vacant' as dsm_name,
'Vacant' as rsm_trty_code,
'Vacant' as rsm_kcode,
'Vacant' as rsm_name,
'Vacant' as rsd_trty_code,
'Vacant' as rsd_kcode,
'Vacant' as rsd_name,
'Vacant' as nsd_trty_code,
'Vacant' as nsd_kcode,
'Vacant' as nsd_name,
'Non AZ Region' as district_code,
'Non AZ Region' as district_name,
'Non AZ Region' as district_name_en,
'Non AZ Region' as region_code,
'Non AZ Region' as region_name,
'Non AZ Region' as region_name_en,
'Non AZ Region' as rc_code,
'Non AZ Region' as rc_name,
'Non AZ Region' as rc_name_en,
bu_code,
bu_name,
bu_name_en,
bu_category,
sub_bu_code,
sub_bu_name,
sub_bu_name_en,
city_name,
null as ims_pack_cod,
t2.market,
ims_audit_cod,
ims_city_ratio
from
tmp.tmp_ims_td_org_hvh t1
left join (
select distinct
t2.market,
t2.bu
from
tmp.tmp_ims_td_org_hvh t1
right join dwd.dwd_td_ims_market_ta_order t2 on t1.market = t2.market
where
t1.market is null
and t2.market <> 'IMS ALL Market'
) t2 on t1.bu_name_en = t2.bu
where
t2.market is not null
-- COMMAND ----------
--将所有的ROC的市场数据补全，同时根据市场找到对应BU
insert into
tmp.tmp_ims_td_org_hvh
select distinct
yyyymm,
dsm_trty_code,
dsm_kcode,
dsm_name,
rsm_trty_code,
rsm_kcode,
rsm_name,
rsd_trty_code,
rsd_kcode,
rsd_name,
nsd_trty_code,
nsd_kcode,
nsd_name,
district_code,
district_name,
district_name_en,
region_code,
region_name,
region_name_en,
rc_code,
rc_name,
rc_name_en,
t2.bu_code,
t2.bu_name,
t2.bu_name_en,
t2.bu_category,
t2.sub_bu_code,
t2.sub_bu_name,
t2.sub_bu_name_en,
t2.city_name,
null as ims_pack_cod,
t2.market,
t2.ims_audit_cod,
t2.ims_city_ratio
from
(
select
'Vacant' as dsm_trty_code,
'Vacant' as dsm_kcode,
'Vacant' as dsm_name,
'Vacant' as rsm_trty_code,
'Vacant' as rsm_kcode,
'Vacant' as rsm_name,
'Vacant' as rsd_trty_code,
'Vacant' as rsd_kcode,
'Vacant' as rsd_name,
'Vacant' as nsd_trty_code,
'Vacant' as nsd_kcode,
'Vacant' as nsd_name,
'Non AZ Region' as district_code,
'Non AZ Region' as district_name,
'Non AZ Region' as district_name_en,
'Other Low Tiers' as region_code,
'Other Low Tiers' as region_name,
'Other Low Tiers' as region_name_en,
'Other Low Tiers' as rc_code,
'Other Low Tiers' as rc_name,
'Other Low Tiers' as rc_name_en,
'Other Low Tiers' as city_name,
'ROC' as audit_cod
) t1
full join (
select distinct
bu_code,
bu_name,
bu_name_en,
bu_category,
sub_bu_code,
sub_bu_name,
sub_bu_name_en,
-- 'ALL' as sub_bu_code,'ALL' as sub_bu_name,'ALL' as sub_bu_name_en,
market,
ims_city_ratio,
yyyymm,
'Other Low Tiers' as city_name,
'ROC' as ims_audit_cod
from
tmp.tmp_ims_td_org_hvh
-- )t2 --on t1.bu=t2.bu_name_en
where
bu_code is not null
--有销量的ROC数据
and concat (market, '_ROC') in (
select distinct
market_audit_key
from
dm.dm_ims_tf_sales
where
audit_cod = 'ROC'
)
) t2 on t1.audit_cod = t2.ims_audit_cod
-- COMMAND ----------
--将所有未能挂上Org的市场数据补全，同时根据市场找到对应BU
insert into
tmp.tmp_ims_td_org_hvh
select distinct
yyyymm,
dsm_trty_code,
dsm_kcode,
dsm_name,
rsm_trty_code,
rsm_kcode,
rsm_name,
rsd_trty_code,
rsd_kcode,
rsd_name,
nsd_trty_code,
nsd_kcode,
nsd_name,
district_code,
district_name,
district_name_en,
region_code,
region_name,
region_name_en,
rc_code,
rc_name,
rc_name_en,
t2.bu_code,
t2.bu_name,
t2.bu_name_en,
t2.bu_category,
t2.sub_bu_code,
t2.sub_bu_name,
t2.sub_bu_name_en,
t2.city_name,
null as ims_pack_cod,
t2.market,
t2.ims_audit_cod,
t2.ims_city_ratio
from
(
select
'Vacant' as dsm_trty_code,
'Vacant' as dsm_kcode,
'Vacant' as dsm_name,
'Vacant' as rsm_trty_code,
'Vacant' as rsm_kcode,
'Vacant' as rsm_name,
'Vacant' as rsd_trty_code,
'Vacant' as rsd_kcode,
'Vacant' as rsd_name,
'Vacant' as nsd_trty_code,
'Vacant' as nsd_kcode,
'Vacant' as nsd_name,
'Non AZ Region' as district_code,
'Non AZ Region' as district_name,
'Non AZ Region' as district_name_en,
'Non AZ Region' as region_code,
'Non AZ Region' as region_name,
'Non AZ Region' as region_name_en,
'Non AZ Region' as rc_code,
'Non AZ Region' as rc_name,
'Non AZ Region' as rc_name_en
) t1
full join (
select distinct
bu_code,
bu_name,
bu_name_en,
bu_category,
sub_bu_code,
sub_bu_name,
sub_bu_name_en,
-- 'ALL' as sub_bu_code,'ALL' as sub_bu_name,'ALL' as sub_bu_name_en,
t2.audit_cod as ims_audit_cod,
null as city_name,
t1.yyyymm,
t2.market,
ims_city_ratio
from
tmp.tmp_ims_td_org_hvh t1
inner join (
--筛选出挂不上Org的市场城市，同时又有销量的
select distinct
t2.market,
t2.audit_cod,
t3.bu
from
tmp.tmp_ims_td_org_hvh t1
right join (
select distinct
audit_cod,
market
from
dm.dm_ims_tf_sales
) t2 on concat (t1.market, '_', t1.ims_audit_cod) = concat (t2.market, '_', t2.audit_cod)
left join dwd.dwd_td_ims_market_ta_order t3 on t2.market = t3.market
where
concat (t1.market, '_', t1.ims_audit_cod) is null
and t2.market <> 'IMS ALL Market'
) t2 on t1.bu_name_en = t2.bu
) t2
where
t2.bu_code is not null
-- COMMAND ----------
insert overwrite table dm.dm_ims_td_org_hvh
select distinct
t1.yyyymm,
concat (
t1.market,
'_',
t2.audit_cod,
'_',
right (t1.yyyymm, 2)
) as ORG_KEY,
t1.dsm_trty_code,
t1.dsm_kcode,
t1.dsm_name,
t1.rsm_trty_code,
t1.rsm_kcode,
t1.rsm_name,
t1.rsd_trty_code,
t1.rsd_kcode,
t1.rsd_name,
t1.nsd_trty_code,
t1.nsd_kcode,
t1.nsd_name,
t1.district_code,
t1.district_name,
t1.district_name_en,
t1.region_code,
t1.region_name,
t1.region_name_en,
t1.rc_code,
t1.rc_name,
t1.rc_name_en,
t1.bu_code,
t1.bu_name,
t1.bu_name_en,
t1.bu_category,
t1.sub_bu_code,
t1.sub_bu_name,
t1.sub_bu_name_en,
t1.sub_bu_code as sub_bu_code_auth, --前端权限配置需要的字段
t1.bu_code as bu_code_auth, --前端权限配置需要的字段
t2.audit_cod,
t2.audit_des,
t2.audit_des_c,
t2.Province,
t2.Province_C,
t2.RegionCenter,
t1.market,
concat (t1.market, '_', t2.audit_cod) as market_audit_key,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_INSERT_DT,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_UPDATE_DT
from
tmp.tmp_ims_td_org_hvh t1
left join dws.dws_ims_td_geo t2 on t1.ims_audit_cod = t2.audit_cod
--where t1.market is not null and t1.ims_audit_cod is not null
union all
select distinct
t1.yyyymm,
concat (
'IMS ALL Market',
'_',
t2.audit_cod,
'_',
right (t1.yyyymm, 2)
) as ORG_KEY,
t1.dsm_trty_code,
t1.dsm_kcode,
t1.dsm_name,
t1.rsm_trty_code,
t1.rsm_kcode,
t1.rsm_name,
t1.rsd_trty_code,
t1.rsd_kcode,
t1.rsd_name,
t1.nsd_trty_code,
t1.nsd_kcode,
t1.nsd_name,
t1.district_code,
t1.district_name,
t1.district_name_en,
t1.region_code,
t1.region_name,
t1.region_name_en,
t1.rc_code,
t1.rc_name,
t1.rc_name_en,
t1.bu_code,
t1.bu_name,
t1.bu_name_en,
t1.bu_category,
t1.sub_bu_code,
t1.sub_bu_name,
t1.sub_bu_name_en,
'ALL' as sub_bu_code_auth,
'ALL' as bu_code_auth,
t2.audit_cod,
t2.audit_des,
t2.audit_des_c,
t2.Province,
t2.Province_C,
t2.RegionCenter,
'IMS ALL Market' as market,
concat ('IMS ALL Market_', t2.audit_cod) as market_audit_key,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_INSERT_DT,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_UPDATE_DT
from
tmp.tmp_ims_td_org_hvh t1
left join dws.dws_ims_td_geo t2 on t1.ims_audit_cod = t2.audit_cod

1679
CHPA/03 dm_ims_td_pack_property.sql Normal file
View File

File diff suppressed because it is too large Load Diff

759
CHPA/03 dm_ims_tf_sales.sql Normal file
View File

@@ -0,0 +1,759 @@
-- Databricks notebook source
-- DBTITLE 1,建表SQL
-- CREATE OR REPLACE TABLE dm.dm_ims_tf_sales (
-- YYYYMM STRING,
-- DAY_DATE STRING,
-- PACK_COD STRING,
-- AUDIT_COD STRING,
-- Related_flag STRING,
-- ORG_KEY STRING,
-- Market STRING,
-- MARKET_PACK_KEY STRING,
-- MARKET_AUDIT_KEY STRING,
-- SALES_VALUE DECIMAL(30,10),
-- SALES_VALUE_LY DECIMAL(30,10),
-- SALES_VALUE_L2Y DECIMAL(30,10),
-- SALES_UNIT DECIMAL(30,10),
-- SALES_UNIT_LY DECIMAL(30,10),
-- SALES_UNIT_L2Y DECIMAL(30,10),
-- CONUTING_UNIT DECIMAL(30,10),
-- CONUTING_UNIT_LY DECIMAL(30,10),
-- CONUTING_UNIT_L2Y DECIMAL(30,10),
-- PDOT DECIMAL(30,10),
-- PDOT_LY DECIMAL(30,10),
-- PDOT_L2Y DECIMAL(30,10),
-- SALES_VALUE_CAL DECIMAL(30,10),
-- SALES_VALUE_CAL_LY DECIMAL(30,10),
-- SALES_UNIT_CAL DECIMAL(30,10),
-- SALES_UNIT_CAL_LY DECIMAL(30,10),
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_ims_tf_sales';
-- COMMAND ----------
-- DBTITLE 1,tmp_ims_tf_fact_sales_l2y
--提前算好前端需要两年前的数据
-- tmp.tmp_ims_tf_fact_sales 整张表对应的notebook path:/Workspace/Users/kthc245@azc.partner.onmschina.cn/20251125Rebuild MA/rebuildMA/DWS/dws_ims_tbl
CREATE
OR REPLACE TEMPORARY VIEW tmp_ims_tf_fact_sales_l2y AS
select
YM,
AUDIT_COD,
PACK_COD,
sum(MTH00LC) as MTH00LC,
sum(MTH00LCLY) as MTH00LCLY,
sum(MTH00LCL2Y) as MTH00LCL2Y,
sum(MTH00CN) as MTH00CN,
sum(MTH00CNLY) as MTH00CNLY,
sum(MTH00CNL2Y) as MTH00CNL2Y,
sum(MTH00UN) as MTH00UN,
sum(MTH00UNLY) as MTH00UNLY,
sum(MTH00UNL2Y) as MTH00UNL2Y
from
(
select
YM,
AUDIT_COD,
PACK_COD,
MTH00LC,
MTH00LCLY,
0 as MTH00LCL2Y,
MTH00CN,
MTH00CNLY,
0 as MTH00CNL2Y,
MTH00UN,
MTH00UNLY,
0 as MTH00UNL2Y
from
tmp.tmp_ims_tf_fact_sales
union all
select
YM + 200 as YM,
AUDIT_COD,
PACK_COD,
0 as MTH00LC,
0 as MTH00LCLY,
MTH00LC as MTH00LCL2Y,
0 as MTH00CN,
0 as MTH00CNLY,
MTH00CN as MTH00CNL2Y,
0 as MTH00UN,
0 as MTH00UNLY,
MTH00UN as MTH00UNL2Y
from
tmp.tmp_ims_tf_fact_sales
where
YM + 200 in (
select distinct
YM
from
tmp.tmp_ims_tf_fact_sales
)
)
group by
YM,
AUDIT_COD,
PACK_COD
-- COMMAND ----------
-- DBTITLE 1,tmp_ims_tf_sales_total
--提前算好需要IMS 折算的value和不折算的value
CREATE
OR REPLACE TEMPORARY VIEW tmp_ims_tf_sales_total AS
-- select distinct
-- t1.YM as YYYYMM,
-- t1.PACK_COD,
-- t1.AUDIT_COD,
-- 'ALL' AS RELATE_FLAG,
-- t1.AUDIT_COD as ORG_KEY,
-- case
-- when t2.market is null then 'Non Market'
-- else t2.market
-- end Market,
-- case
-- when t2.market is null then concat ('Non Market_', t1.pack_cod)
-- else concat (t2.market, '_', t2.pack_cod)
-- end as MARKET_PACK_KEY,
-- case
-- when t2.market is null then concat ('Non Market_', t1.audit_cod)
-- else concat (t2.market, '_', t1.audit_cod)
-- end as MARKET_AUDIT_KEY,
-- t1.MTH00LC * (
-- case
-- when t2.Value_Market_Ratio is null then 1
-- else t2.Value_Market_Ratio
-- end
-- ) as SALES_VALUE,
-- t1.MTH00LCLY * (
-- case
-- when t2.Value_Market_Ratio is null then 1
-- else t2.Value_Market_Ratio
-- end
-- ) as SALES_VALUE_LY,
-- t1.MTH00LCL2Y * (
-- case
-- when t2.Value_Market_Ratio is null then 1
-- else t2.Value_Market_Ratio
-- end
-- ) as SALES_VALUE_L2Y,
-- t1.MTH00UN * (
-- case
-- when t2.Unit_Market_Ratio is null then 1
-- else t2.Unit_Market_Ratio
-- end
-- ) as SALES_UNIT,
-- t1.MTH00UNLY * (
-- case
-- when t2.Unit_Market_Ratio is null then 1
-- else t2.Unit_Market_Ratio
-- end
-- ) as SALES_UNIT_LY,
-- t1.MTH00UNL2Y * (
-- case
-- when t2.Unit_Market_Ratio is null then 1
-- else t2.Unit_Market_Ratio
-- end
-- ) as SALES_UNIT_L2Y,
-- t1.MTH00CN as CONUTING_UNIT,
-- t1.MTH00CNLY as CONUTING_UNIT_LY,
-- t1.MTH00CNL2Y as CONUTING_UNIT_L2Y,
-- t1.MTH00CN * (
-- case
-- when t2.CountingUnit_Market_Ratio is null then 1
-- else t2.CountingUnit_Market_Ratio
-- end
-- ) as PDOT,
-- t1.MTH00CNLY * (
-- case
-- when t2.CountingUnit_Market_Ratio is null then 1
-- else t2.CountingUnit_Market_Ratio
-- end
-- ) as PDOT_LY,
-- t1.MTH00CNL2Y * (
-- case
-- when t2.CountingUnit_Market_Ratio is null then 1
-- else t2.CountingUnit_Market_Ratio
-- end
-- ) as PDOT_L2Y,
-- t1.MTH00LC as SALES_VALUE_CAL,
-- t1.MTH00LCLY as SALES_VALUE_CAL_LY,
-- t1.MTH00LCL2Y as SALES_VALUE_CAL_L2Y,
-- t1.MTH00UN as SALES_UNIT_CAL,
-- t1.MTH00UNLY as SALES_UNIT_CAL_LY,
-- t1.MTH00UNL2Y as SALES_UNIT_CAL_L2Y,
-- from_utc_timestamp (current_timestamp(), 'UTC+8') as ETL_INSERT_DT,
-- from_utc_timestamp (current_timestamp(), 'UTC+8') as ETL_UPDATE_DT
-- from
-- tmp_ims_tf_fact_sales_l2y t1
-- left join (
-- select distinct
-- YM,
-- market,
-- pack_cod,
-- Value_Market_Ratio,
-- Unit_Market_Ratio,
-- CountingUnit_Market_Ratio
-- from
-- dws.dws_ims_td_market_ratio
-- ) t2 on t1.pack_cod = t2.pack_cod
-- and t1.YM = t2.YM
-- where
-- -- t1.YM in (select YYYYMM from dm.dm_ims_td_calendar)and
-- t2.market is not null
-- union all
-- -- IMS ALL Market 为所有的IMS市场 不折算的数据的汇总，即不折算的所有市场的city粒度数据
select distinct
t1.YM as YYYYMM,
t1.PACK_COD,
t1.AUDIT_COD,
-- 'ALL' AS RELATE_FLAG,
t1.AUDIT_COD as ORG_KEY,
-- 'IMS ALL Market' Market,
-- concat ('IMS ALL Market_', t1.pack_cod) as MARKET_PACK_KEY,
-- concat ('IMS ALL Market_', t1.audit_cod) as MARKET_AUDIT_KEY,
t1.MTH00LC SALES_VALUE,
t1.MTH00LCLY SALES_VALUE_LY,
t1.MTH00LCL2Y SALES_VALUE_L2Y,
t1.MTH00UN SALES_UNIT,
t1.MTH00UNLY SALES_UNIT_LY,
t1.MTH00UNL2Y SALES_UNIT_L2Y,
t1.MTH00CN CONUTING_UNIT,
t1.MTH00CNLY CONUTING_UNIT_LY,
t1.MTH00CNL2Y CONUTING_UNIT_L2Y,
t1.MTH00CN PDOT,
t1.MTH00CNLY PDOT_LY,
t1.MTH00CNL2Y PDOT_L2Y,
t1.MTH00LC SALES_VALUE_CAL,
t1.MTH00LCLY SALES_VALUE_CAL_LY,
t1.MTH00LCL2Y SALES_VALUE_CAL_L2Y,
t1.MTH00UN SALES_UNIT_CAL,
t1.MTH00UNLY SALES_UNIT_CAL_LY,
t1.MTH00UNL2Y SALES_UNIT_CAL_L2Y,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_INSERT_DT,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_UPDATE_DT
from
tmp_ims_tf_fact_sales_l2y t1
-- where t1.YM in (select YYYYMM from dm.dm_ims_td_calendar)
;
-- COMMAND ----------
-- DBTITLE 1,tmp_ims_tf_sales_tamarket
-- --在原本的数据上加上AZ Related Market，此为AZ 相关的六大市场数据汇总
-- --以及年月日的日期格式标签
-- CREATE
-- OR REPLACE TEMPORARY VIEW tmp_ims_tf_sales_tamarket AS
-- select distinct
-- YYYYMM,
-- concat (left (YYYYMM, 4), '-', right (YYYYMM, 2), '-01') as DAY_DATE,
-- PACK_COD,
-- AUDIT_COD,
-- RELATE_FLAG,
-- ORG_KEY,
-- Market,
-- MARKET_PACK_KEY,
-- MARKET_AUDIT_KEY,
-- SALES_VALUE,
-- SALES_VALUE_LY,
-- SALES_VALUE_L2Y,
-- SALES_UNIT,
-- SALES_UNIT_LY,
-- SALES_UNIT_L2Y,
-- CONUTING_UNIT,
-- CONUTING_UNIT_LY,
-- CONUTING_UNIT_L2Y,
-- PDOT,
-- PDOT_LY,
-- PDOT_L2Y,
-- SALES_VALUE_CAL,
-- SALES_VALUE_CAL_LY,
-- SALES_VALUE_CAL_L2Y,
-- SALES_UNIT_CAL,
-- SALES_UNIT_CAL_LY,
-- SALES_UNIT_CAL_L2Y,
-- from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_INSERT_DT,
-- from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_UPDATE_DT
-- from
-- tmp_ims_tf_sales_total
-- COMMAND ----------
-- DBTITLE 1,插入逻辑1-需保留
-- dm.dm_ims_tf_sales_china 和 dm.dm_ims_tf_sales 区别就在于 audit_cod = 'CHT' 或者 t1.audit_cod <> 'CHT'
-- insert overwrite table dm.dm_ims_tf_sales_china
-- from tmp_ims_tf_sales_tamarket t1
-- where t1.audit_cod = 'CHT' and (sales_value<>0 or sales_value_ly<>0) and t1.market not in ('Primary Prevention Market-XZK','Iressa 1L TKI Market' )
-- union all
-- from tmp_ims_tf_sales_tamarket t1
-- where t1.audit_cod = 'CHT' and (sales_value<>0 or sales_value_ly<>0) and t1.market in ('Primary Prevention Market-XZK','Iressa 1L TKI Market')
-- insert overwrite table dm.dm_ims_tf_sales
-- from tmp_ims_tf_sales_tamarket t1
-- where t1.audit_cod <> 'CHT' and t1.market not in ('Primary Prevention Market-XZK','Iressa 1L TKI Market')and (sales_value<>0 or sales_value_ly<>0)
-- union all
-- from tmp_ims_tf_sales_tamarket t1
-- where t1.audit_cod <> 'CHT' and t1.market in ('Primary Prevention Market-XZK','Iressa 1L TKI Market')and (sales_value<>0 or sales_value_ly<>0)
-- COMMAND ----------
-- CREATE TABLE catalog_e2_ngbi_qas.dm.dm_ims_tf_sales_china (
-- YYYYMM INT,
-- PACK_COD STRING,
-- AUDIT_COD STRING,
-- SALES_VALUE DECIMAL(38,10),
-- SALES_VALUE_LY DECIMAL(38,10),
-- SALES_VALUE_L2Y DECIMAL(38,10),
-- SALES_UNIT DECIMAL(38,10),
-- SALES_UNIT_LY DECIMAL(38,10),
-- SALES_UNIT_L2Y DECIMAL(38,10),
-- CONUTING_UNIT DECIMAL(38,10),
-- CONUTING_UNIT_LY DECIMAL(38,10),
-- CONUTING_UNIT_L2Y DECIMAL(38,10),
-- PDOT DECIMAL(38,10),
-- PDOT_LY DECIMAL(38,10),
-- PDOT_L2Y DECIMAL(38,10),
-- SALES_VALUE_CAL DECIMAL(38,10),
-- SALES_VALUE_CAL_LY DECIMAL(38,10),
-- SALES_UNIT_CAL DECIMAL(38,10),
-- SALES_UNIT_CAL_LY DECIMAL(38,10),
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_ims_tf_sales_china'
-- COMMAND ----------
-- DBTITLE 1,overwrite dm.dm_ims_tf_sales_china
--SALES
--CHINA CHPA数据
-- t1.market not in ('Primary Prevention Market-XZK','Iressa 1L TKI Market')
-- ,CONUTING_UNIT
-- ,CONUTING_UNIT_LY
-- ,CONUTING_UNIT_L2Y
-- ,SALES_VALUE_CAL
-- ,SALES_VALUE_CAL_LY
-- ,SALES_UNIT_CAL
-- ,SALES_UNIT_CAL_LY
-- from tmp_ims_tf_sales_tamarket t1
-- where t1.audit_cod = 'CHT' and (sales_value<>0 or sales_value_ly<>0) and t1.market not in ('Primary Prevention Market-XZK','Iressa 1L TKI Market' )
-- t1.market in ('Primary Prevention Market-XZK','Iressa 1L TKI Market')
-- PDOT as CONUTING_UNIT
-- ,PDOT_LY as CONUTING_UNIT_LY
-- ,PDOT_L2Y as CONUTING_UNIT_L2Y
-- ,SALES_VALUE as SALES_VALUE_CAL
-- ,SALES_VALUE_LY as SALES_VALUE_CAL_LY
-- ,SALES_UNIT as SALES_UNIT_CAL
-- ,SALES_UNIT_LY as SALES_UNIT_CAL_LY
-- from tmp_ims_tf_sales_tamarket t1
-- where t1.audit_cod = 'CHT' and (sales_value<>0 or sales_value_ly<>0) and t1.market in ('Primary Prevention Market-XZK','Iressa 1L TKI Market')
-- create or replace table dm.dm_ims_tf_sales_china
insert overwrite table dm.dm_ims_tf_sales_china
select distinct
YYYYMM,
-- DAY_DATE,
PACK_COD,
AUDIT_COD,
-- RELATE_FLAG,
-- ORG_KEY,
-- Market,
-- MARKET_PACK_KEY,
-- MARKET_AUDIT_KEY,
SALES_VALUE,
SALES_VALUE_LY,
SALES_VALUE_L2Y,
SALES_UNIT,
SALES_UNIT_LY,
SALES_UNIT_L2Y,
CONUTING_UNIT,
CONUTING_UNIT_LY,
CONUTING_UNIT_L2Y,
PDOT,
PDOT_LY,
PDOT_L2Y,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_INSERT_DT,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_UPDATE_DT
from
tmp_ims_tf_sales_total t1
where
t1.audit_cod = 'CHT'
and (
sales_value <> 0
or sales_value_ly <> 0
)
-- and t1.market not in (
-- 'Primary Prevention Market-XZK',
-- 'Iressa 1L TKI Market'
-- )
-- union all
-- select distinct
-- YYYYMM,
-- DAY_DATE,
-- PACK_COD,
-- AUDIT_COD,
-- RELATE_FLAG,
-- ORG_KEY,
-- Market,
-- MARKET_PACK_KEY,
-- MARKET_AUDIT_KEY,
-- SALES_VALUE,
-- SALES_VALUE_LY,
-- SALES_VALUE_L2Y,
-- SALES_UNIT,
-- SALES_UNIT_LY,
-- SALES_UNIT_L2Y,
-- PDOT as CONUTING_UNIT,
-- PDOT_LY as CONUTING_UNIT_LY,
-- PDOT_L2Y as CONUTING_UNIT_L2Y,
-- PDOT,
-- PDOT_LY,
-- PDOT_L2Y,
-- SALES_VALUE as SALES_VALUE_CAL,
-- SALES_VALUE_LY as SALES_VALUE_CAL_LY,
-- SALES_UNIT as SALES_UNIT_CAL,
-- SALES_UNIT_LY as SALES_UNIT_CAL_LY,
-- from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_INSERT_DT,
-- from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_UPDATE_DT
-- from
-- tmp_ims_tf_sales_tamarket t1
-- where
-- t1.audit_cod = 'CHT'
-- and (
-- sales_value <> 0
-- or sales_value_ly <> 0
-- )
-- and t1.market in (
-- 'Primary Prevention Market-XZK',
-- 'Iressa 1L TKI Market'
-- )
-- COMMAND ----------
-- CREATE or REPLACE TABLE dm.dm_ims_tf_sales (
-- YYYYMM INT,
-- PACK_COD STRING,
-- AUDIT_COD STRING,
-- SALES_VALUE DECIMAL(38,10),
-- SALES_VALUE_LY DECIMAL(38,10),
-- SALES_VALUE_L2Y DECIMAL(38,10),
-- SALES_UNIT DECIMAL(38,10),
-- SALES_UNIT_LY DECIMAL(38,10),
-- SALES_UNIT_L2Y DECIMAL(38,10),
-- CONUTING_UNIT DECIMAL(38,10),
-- CONUTING_UNIT_LY DECIMAL(38,10),
-- CONUTING_UNIT_L2Y DECIMAL(38,10),
-- PDOT DECIMAL(38,10),
-- PDOT_LY DECIMAL(38,10),
-- PDOT_L2Y DECIMAL(38,10),
-- SALES_VALUE_CAL DECIMAL(38,10),
-- SALES_VALUE_CAL_LY DECIMAL(38,10),
-- SALES_UNIT_CAL DECIMAL(38,10),
-- SALES_UNIT_CAL_LY DECIMAL(38,10),
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_ims_tf_sales'
-- COMMAND ----------
-- DBTITLE 1,overwrite dm.dm_ims_tf_sales
--SALES
--CITY
insert overwrite table dm.dm_ims_tf_sales
select distinct
YYYYMM,
-- DAY_DATE,
PACK_COD,
AUDIT_COD,
-- RELATE_FLAG,
-- ORG_KEY,
-- Market,
-- MARKET_PACK_KEY,
-- MARKET_AUDIT_KEY,
SALES_VALUE,
SALES_VALUE_LY,
SALES_VALUE_L2Y,
SALES_UNIT,
SALES_UNIT_LY,
SALES_UNIT_L2Y,
CONUTING_UNIT,
CONUTING_UNIT_LY,
CONUTING_UNIT_L2Y,
PDOT,
PDOT_LY,
PDOT_L2Y,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_INSERT_DT,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_UPDATE_DT
from
tmp_ims_tf_sales_total t1
where
t1.audit_cod <> 'CHT'
-- and t1.market not in (
-- 'Primary Prevention Market-XZK',
-- 'Iressa 1L TKI Market'
-- )
and (
sales_value <> 0
or sales_value_ly <> 0
)
-- union all
-- --'Primary Prevention Market-XZK','Iressa 1L TKI Market' 两个市场特殊处理，所有的value都是折算过的
-- select distinct
-- YYYYMM,
-- DAY_DATE,
-- PACK_COD,
-- AUDIT_COD,
-- RELATE_FLAG,
-- ORG_KEY,
-- Market,
-- MARKET_PACK_KEY,
-- MARKET_AUDIT_KEY,
-- SALES_VALUE,
-- SALES_VALUE_LY,
-- SALES_VALUE_L2Y,
-- SALES_UNIT,
-- SALES_UNIT_LY,
-- SALES_UNIT_L2Y,
-- PDOT as CONUTING_UNIT,
-- PDOT_LY as CONUTING_UNIT_LY,
-- PDOT_L2Y as CONUTING_UNIT_L2Y,
-- PDOT,
-- PDOT_LY,
-- PDOT_L2Y,
-- SALES_VALUE as SALES_VALUE_CAL,
-- SALES_VALUE_LY as SALES_VALUE_CAL_LY,
-- SALES_UNIT as SALES_UNIT_CAL,
-- SALES_UNIT_LY as SALES_UNIT_CAL_LY,
-- from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_INSERT_DT,
-- from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_UPDATE_DT
-- from
-- tmp_ims_tf_sales_tamarket t1
-- where
-- t1.audit_cod <> 'CHT'
-- -- and t1.market in (
-- -- 'Primary Prevention Market-XZK',
-- -- 'Iressa 1L TKI Market'
-- -- )
-- -- and (
-- -- sales_value <> 0
-- -- or sales_value_ly <> 0
-- -- )
-- COMMAND ----------
-- DBTITLE 1,插入逻辑2-需保留
-- -- 关键词
-- audit_cod <> 'CHT' -》 dm.dm_ims_tf_sales
-- audit_cod = 'CHT' -》dm.dm_ims_tf_sales_china
-- KCT 所有城市的IMS数据 基于 dm.dm_ims_tf_sales
-- ROC 其他城市的IMS数据计算 ROC = CHT-KCT
-- COMMAND ----------
-- DBTITLE 1,overwrite tmp.tmp_ims_tf_sales_kct
-- 所有城市汇总的IMS数据计算 KCT=sum(all city)
-- create or replace table tmp.tmp_ims_tf_sales_kct
insert overwrite table tmp.tmp_ims_tf_sales_kct
select
YYYYMM,
PACK_COD,
'KCT' as AUDIT_COD,
-- Related_flag,
-- 'KCT' as ORG_KEY,
-- Market,
-- MARKET_PACK_KEY,
-- concat (Market, '_KCT') as MARKET_AUDIT_KEY,
sum(SALES_VALUE) as SALES_VALUE,
sum(SALES_VALUE_LY) as SALES_VALUE_LY,
sum(SALES_VALUE_L2Y) as SALES_VALUE_L2Y,
sum(SALES_UNIT) as SALES_UNIT,
sum(SALES_UNIT_LY) as SALES_UNIT_LY,
sum(SALES_UNIT_L2Y) as SALES_UNIT_L2Y,
sum(CONUTING_UNIT) as CONUTING_UNIT,
sum(CONUTING_UNIT_LY) as CONUTING_UNIT_LY,
sum(CONUTING_UNIT_L2Y) as CONUTING_UNIT_L2Y,
sum(PDOT) as PDOT,
sum(PDOT_LY) as PDOT_LY,
sum(PDOT_LY) as PDOT_L2Y,
sum(SALES_VALUE_CAL) as SALES_VALUE_CAL,
sum(SALES_VALUE_CAL_LY) as SALES_VALUE_CAL_LY,
sum(SALES_UNIT_CAL) as SALES_UNIT_CAL,
sum(SALES_UNIT_CAL_LY) as SALES_UNIT_CAL_LY
from
dm.dm_ims_tf_sales
group by
YYYYMM,
PACK_COD
-- Related_flag,
-- Market,
-- MARKET_PACK_KEY;
-- COMMAND ----------
-- DBTITLE 1,overwrite tmp.tmp_ims_tf_sales_roc
-- 其他城市的IMS数据计算 ROC=CHT-KCT
insert overwrite table tmp.tmp_ims_tf_sales_roc
WITH
DIM_YM_PACK_MARKT (
SELECT
YYYYMM as CHINA_YYYYMM,
PACK_COD as CHINA_PACK_COD
-- ,CHINA_Market
from
dm.dm_ims_tf_sales_china
group by
YYYYMM,
PACK_COD
-- ,CHINA_Market
UNION
SELECT
YYYYMM,
PACK_COD
-- ,Market
from
tmp.tmp_ims_tf_sales_kct
group by
YYYYMM,
PACK_COD
-- ,Market
),
CHPA_SALES (
select
YYYYMM,
PACK_COD,
-- CHINA_Market,
sum(SALES_VALUE) as CHINA_SALES_VALUE,
sum(SALES_VALUE_LY) as CHINA_SALES_VALUE_LY,
sum(SALES_VALUE_L2Y) as CHINA_SALES_VALUE_L2Y,
sum(SALES_UNIT) as CHINA_SALES_UNIT,
sum(SALES_UNIT_LY) as CHINA_SALES_UNIT_LY,
sum(SALES_UNIT_L2Y) as CHINA_SALES_UNIT_L2Y,
sum(CONUTING_UNIT) as CHINA_CONUTING_UNIT,
sum(CONUTING_UNIT_LY) as CHINA_CONUTING_UNIT_LY,
sum(CONUTING_UNIT_L2Y) as CHINA_CONUTING_UNIT_L2Y,
sum(PDOT) as CHINA_PDOT,
sum(PDOT_LY) as CHINA_PDOT_LY,
sum(PDOT_L2Y) as CHINA_PDOT_L2Y,
sum(SALES_VALUE_CAL) as SALES_VALUE_CAL,
sum(SALES_VALUE_CAL_LY) as SALES_VALUE_CAL_LY,
sum(SALES_UNIT_CAL) as SALES_UNIT_CAL,
sum(SALES_UNIT_CAL_LY) as SALES_UNIT_CAL_LY
from
dm.dm_ims_tf_sales_china
group by
YYYYMM,
PACK_COD
-- ,CHINA_Market
),
FABO_SALES (
select
YYYYMM,
PACK_COD,
-- Market,
sum(SALES_VALUE) as SALES_VALUE,
sum(SALES_VALUE_LY) as SALES_VALUE_LY,
sum(SALES_VALUE_L2Y) as SALES_VALUE_L2Y,
sum(SALES_UNIT) as SALES_UNIT,
sum(SALES_UNIT_LY) as SALES_UNIT_LY,
sum(SALES_UNIT_L2Y) as SALES_UNIT_L2Y,
sum(CONUTING_UNIT) as CONUTING_UNIT,
sum(CONUTING_UNIT_LY) as CONUTING_UNIT_LY,
sum(CONUTING_UNIT_L2Y) as CONUTING_UNIT_L2Y,
sum(PDOT) as PDOT,
sum(PDOT_LY) as PDOT_LY,
sum(PDOT_L2Y) as PDOT_L2Y,
sum(SALES_VALUE_CAL) as SALES_VALUE_CAL,
sum(SALES_VALUE_CAL_LY) as SALES_VALUE_CAL_LY,
sum(SALES_UNIT_CAL) as SALES_UNIT_CAL,
sum(SALES_UNIT_CAL_LY) as SALES_UNIT_CAL_LY
from
tmp.tmp_ims_tf_sales_kct
group by
YYYYMM,
PACK_COD
-- ,Market
)
SELECT
t1.CHINA_YYYYMM as YYYYMM,
t1.CHINA_PACK_COD as PACK_COD,
'ROC' as AUDIT_COD,
-- null as Related_flag,
-- 'ROC' as ORG_KEY,
-- t1.CHINA_Market as Market,
-- concat (t1.CHINA_Market, '_', t1.china_pack_cod) as MARKET_PACK_KEY,
-- concat (t1.CHINA_Market, '_ROC') as MARKET_AUDIT_KEY,
nvl (t2.CHINA_SALES_VALUE, 0) - nvl (t3.SALES_VALUE, 0) as SALES_VALUE,
nvl (t2.CHINA_SALES_VALUE_LY, 0) - nvl (t3.SALES_VALUE_LY, 0) as SALES_VALUE_LY,
nvl (t2.CHINA_SALES_VALUE_L2Y, 0) - nvl (t3.SALES_VALUE_L2Y, 0) as SALES_VALUE_L2Y,
nvl (t2.CHINA_SALES_UNIT, 0) - nvl (t3.SALES_UNIT, 0) as SALES_UNIT,
nvl (t2.CHINA_SALES_UNIT_L2Y, 0) - nvl (t3.SALES_UNIT_L2Y, 0) as SALES_UNIT_LY,
nvl (t2.CHINA_SALES_UNIT_LY, 0) - nvl (t3.SALES_UNIT_LY, 0) as SALES_UNIT_L2Y,
nvl (t2.CHINA_CONUTING_UNIT, 0) - nvl (t3.CONUTING_UNIT, 0) as CONUTING_UNIT,
nvl (t2.CHINA_CONUTING_UNIT_LY, 0) - nvl (t3.CONUTING_UNIT_LY, 0) as CONUTING_UNIT_LY,
nvl (t2.CHINA_CONUTING_UNIT_L2Y, 0) - nvl (t3.CONUTING_UNIT_L2Y, 0) as CONUTING_UNIT_L2Y,
nvl (t2.CHINA_PDOT, 0) - nvl (t3.PDOT, 0) as PDOT,
nvl (t2.CHINA_PDOT_LY, 0) - nvl (t3.PDOT_LY, 0) as PDOT_LY,
nvl (t2.CHINA_PDOT_L2Y, 0) - nvl (t3.PDOT_L2Y, 0) as PDOT_L2Y,
nvl (t2.SALES_VALUE_CAL, 0) - nvl (t3.SALES_VALUE_CAL, 0) as SALES_VALUE_CAL,
nvl (t2.SALES_VALUE_CAL_LY, 0) - nvl (t3.SALES_VALUE_CAL_LY, 0) as SALES_VALUE_CAL_LY,
nvl (t2.SALES_UNIT_CAL, 0) - nvl (t3.SALES_UNIT_CAL, 0) as SALES_UNIT_CAL,
nvl (t2.SALES_UNIT_CAL_LY, 0) - nvl (t3.SALES_UNIT_CAL_LY, 0) as SALES_UNIT_CAL_LY
FROM
dim_ym_pack_markt t1
LEFT JOIN chpa_sales t2 on
-- t1.china_market = t2.china_market and
t1.china_yyyymm = t2.yyyymm
and t1.china_pack_cod = t2.pack_cod
LEFT JOIN fabo_sales t3 on
-- t1.china_market = t3.market and
t1.china_yyyymm = t3.yyyymm
and t1.china_pack_cod = t3.pack_cod
-- COMMAND ----------
-- DBTITLE 1,insert into dm.dm_ims_tf_sales
--将ROC的数据写入dm.dm_ims_tf_sales
insert into
dm.dm_ims_tf_sales
select distinct
YYYYMM,
-- concat (left (YYYYMM, 4), '-', right (YYYYMM, 2), '-01') as DAY_DATE,
PACK_COD,
AUDIT_COD,
-- RELATED_FLAG,
-- ORG_KEY,
-- Market,
-- MARKET_PACK_KEY,
-- MARKET_AUDIT_KEY,
SALES_VALUE,
SALES_VALUE_LY,
SALES_VALUE_L2Y,
SALES_UNIT,
SALES_UNIT_LY,
SALES_UNIT_L2Y,
CONUTING_UNIT,
CONUTING_UNIT_LY,
CONUTING_UNIT_L2Y,
PDOT,
PDOT_LY,
PDOT_L2Y,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_INSERT_DT,
from_utc_timestamp (current_timestamp(), 'UTC+8') ETL_UPDATE_DT
from
tmp.tmp_ims_tf_sales_roc
where
sales_value <> 0
or sales_value_ly <> 0;

14
CHPA/03 dm_td_chpa_market_definition.sql Normal file
View File

@@ -0,0 +1,14 @@
-- Databricks notebook source
-- 源表：DWD.DWD_GND_IMS_MARKET_DEFINITION
-- 目标表：DM.DM_TD_CHPA_MARKET_DEFINITION
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_CHPA_MARKET_DEFINITION
SELECT
TA,
MARKET,
MARKET_DEFINITION_EN,
MARKET_DEFINITION_CN
FROM
DWD.DWD_GND_IMS_MARKET_DEFINITION;

30
CHPA/03 dm_td_ims_city_mapping.sql Normal file
View File

@@ -0,0 +1,30 @@
-- Databricks notebook source
-- 源表：DWD.DWD_TD_IMS_CITY_MAPPING
-- 目标表：DM.DM_TD_IMS_CITY_MAPPING,DWS.DWS_TD_GEOGRAPHY
-- COMMAND ----------
--dwd to dws
--ims city和标准城市名以及拆分比mapping表
insert overwrite table dws.dws_td_ims_city_mapping
select distinct
audit_cod ,
city_c ,
city_c_d ,
ratio ,
from_utc_timestamp(current_timestamp(),'UTC+8'),
from_utc_timestamp(current_timestamp(),'UTC+8')
from dwd.dwd_td_ims_city_mapping
;
--dws to dm
insert overwrite table dm.dm_td_ims_city_mapping
select distinct
audit_cod ,
city_c ,
city_c_d ,
city_code,
ratio ,
from_utc_timestamp(current_timestamp(),'UTC+8'),
from_utc_timestamp(current_timestamp(),'UTC+8')
from dws.dws_td_ims_city_mapping t1
left join dws.dws_td_geography t2 on concat(t1.city_c_d,'市')=t2.city_name

1544
EC/03 ec_load_data.sql Normal file
View File

File diff suppressed because it is too large Load Diff

146
EC/04 DM_TD_EXT_EC_PACK_PROPERTY.sql Normal file
View File

@@ -0,0 +1,146 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_EC_PACK_PROPERTY
-- (
-- MARKET_PACK_KEY STRING,
-- PACK_CODE STRING,
-- PACK_DESC STRING,
-- STGH_DESC STRING,
-- PACK_LCH STRING,
-- FAMILY_CODE STRING,
-- FAMILY_NAME STRING,
-- PROD_CODE STRING,
-- PROD_DESC STRING,
-- PROD_DESC_C STRING,
-- CMPS_CODE STRING,
-- CMPS_DESC STRING,
-- CMPS_DESC_C STRING,
-- ATC1_CODE STRING,
-- ATC2_CODE STRING,
-- ATC3_CODE STRING,
-- ATC4_CODE STRING,
-- APP1_CODE STRING,
-- APP2_CODE STRING,
-- APP3_CODE STRING,
-- BIO_DESC STRING,
-- GENE_ORIG_DESC STRING,
-- ETH_OTC_DESC STRING,
-- NRDL_DESC STRING,
-- NRDL_ENTRY_DATE STRING,
-- EDL_DESC STRING,
-- TCM_DESC STRING,
-- PAED_DESC STRING,
-- GQCE_DESC STRING,
-- VBP_DESC_V STRING,
-- VBP_DESC STRING,
-- MANU_CODE STRING,
-- MANU_DESC STRING,
-- MANU_DESC_C STRING,
-- MNFL_CODE STRING,
-- MNFL_DESC STRING,
-- CORP_CODE STRING,
-- CORP_DESC STRING,
-- CORP_DESC_C STRING,
-- BRANDTYPE STRING,
-- MARKET STRING,
-- KEY_COMPETITOR STRING,
-- IS_AZ STRING,
-- AZ_MAIN STRING,
-- AZ_RELATED STRING,
-- ATC1_DESC STRING,
-- ATC1_DESC_C STRING,
-- ATC2_DESC STRING,
-- ATC2_DESC_C STRING,
-- ATC3_DESC STRING,
-- ATC3_DESC_C STRING,
-- ATC4_DESC STRING,
-- ATC4_DESC_C STRING,
-- APP1_DESC STRING,
-- APP1_DESC_C STRING,
-- APP2_DESC STRING,
-- APP2_DESC_C STRING,
-- APP3_DESC STRING,
-- APP3_DESC_C STRING,
-- CLASS STRING,
-- MARKET_RATIO STRING,
-- COUNTINGUNIT STRING,
-- VBP_BRAND STRING,
-- REPLENISH_FALG STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP
-- )
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_ec_pack_property';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_ec_pack_property';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_EC_PACK_PROPERTY
SELECT
MARKET_PACK_KEY,
iqvia_pack_code AS PACK_CODE,
pack_des AS PACK_DESC,
stgh_des AS STGH_DESC,
pack_lch AS PACK_LCH,
Family_Code AS FAMILY_CODE,
Family_Name AS FAMILY_NAME,
prod_cod AS PROD_CODE,
prod_des AS PROD_DESC,
prod_des_c AS PROD_DESC_C,
cmps_cod AS CMPS_CODE,
cmps_des AS CMPS_DESC,
cmps_des_c AS CMPS_DESC_C,
atc1_cod AS ATC1_CODE,
atc2_cod AS ATC2_CODE,
atc3_cod AS ATC3_CODE,
atc4_cod AS ATC4_CODE,
app1_cod AS APP1_CODE,
app2_cod AS APP2_CODE,
app3_cod AS APP3_CODE,
bio_desc AS BIO_DESC,
gene_orig_desc AS GENE_ORIG_DESC,
eth_otc_desc AS ETH_OTC_DESC,
nrdl_desc AS NRDL_DESC,
NRDL_ENTRY_DATE,
edl_desc AS EDL_DESC,
tcm_desc AS TCM_DESC,
paed_desc AS PAED_DESC,
gqce_desc AS GQCE_DESC,
NULL AS VBP_DESC_V,
vbp_desc AS VBP_DESC,
manu_cod AS MANU_CODE,
manu_des AS MANU_DESC,
manu_des_c AS MANU_DESC_C,
mnfl_cod AS MNFL_CODE,
mnfl_des AS MNFL_DESC,
corp_cod AS CORP_CODE,
corp_des AS CORP_DESC,
CORP_DES_C AS CORP_DESC_C,
BrandType AS BRANDTYPE,
market AS MARKET,
KEY_COMPETITOR,
is_az AS IS_AZ,
AZ_MAIN,
AZ_Related AS AZ_RELATED,
atc1_des AS ATC1_DESC,
atc1_des_c AS ATC1_DESC_C,
atc2_des AS ATC2_DESC,
atc2_des_c AS ATC2_DESC_C,
atc3_des AS ATC3_DESC,
atc3_des_c AS ATC3_DESC_C,
atc4_des AS ATC4_DESC,
atc4_des_c AS ATC4_DESC_C,
app1_des AS APP1_DESC,
app1_des_c AS APP1_DESC_C,
app2_des AS APP2_DESC,
app2_des_c AS APP2_DESC_C,
app3_des AS APP3_DESC,
app3_des_c AS APP3_DESC_C,
class AS CLASS,
extend_market_ratio AS MARKET_RATIO,
counting_unit AS COUNTINGUNIT,
NULL AS VBP_BRAND,
NULL AS REPLENISH_FALG,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_ZK_EC_PACK_PROPERTY;

173
EC/05 DM_TF_EXT_EC_SALES.sql Normal file
View File

@@ -0,0 +1,173 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TF_EXT_EC_SALES (
-- YYYYMM STRING,
-- PACK_CODE STRING,
-- CORP_CODE STRING,
-- AUDIT_CODE STRING,
-- PLATFORM_TYPE STRING,
-- STORE_NAME STRING,
-- STORE_TYPE STRING,
-- REGION_TYPE STRING,
-- PACK_FLAG INT,
-- PROD_FLAG INT,
-- DTP_FLAG INT,
-- SALES_UNIT_CAL DECIMAL(38,10),
-- SALES_UNIT_CAL_LY DECIMAL(38,10),
-- SALES_VALUE_CAL DECIMAL(38,10),
-- SALES_VALUE_CAL_LY DECIMAL(38,10),
-- CONUTING_UNIT DECIMAL(38,10),
-- CONUTING_UNIT_LY DECIMAL(38,10),
-- DATA_SOURCE STRING,
-- INST_CODE STRING COMMENT '内部机构编码',
-- CMPS_FLAG STRING COMMENT '分子式标签',
-- DEPT_NAME STRING COMMENT '科室名称',
-- PRESCRIPTION DECIMAL(38,10) COMMENT '处方张数',
-- PRESCRIPTION_LY DECIMAL(38,10) COMMENT '去年同期处方张数',
-- NEW_CODE STRING COMMENT '主数据关联CODE',
-- AREA STRING COMMENT '城市',
-- H_LEVEL STRING COMMENT '医院类型',
-- REIMBURSE STRING COMMENT '报销情况',
-- REIMBURSE_TYPE STRING COMMENT '报销类型',
-- PRESCRIPTION_SOURCE STRING COMMENT '处方来源',
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_ec_sales';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_ec_sales';
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW EC_SALES
AS
SELECT
A.YYYYMM,
A.iqvia_pack_code AS PACK_CODE,
A.AUDIT_COD AS AUDIT_CODE,
'EC(Monthly)' DATA_SOURCE,
-------------------------------------
SUM(sales_unit) SALES_UNIT_CAL,
SUM(sales_unit_ly) SALES_UNIT_CAL_LY,
SUM(sales_value) SALES_VALUE_CAL,
SUM(sales_value_LY) SALES_VALUE_CAL_LY,
SUM(counting_unit) CONUTING_UNIT,
SUM(counting_unit_LY) CONUTING_UNIT_LY,
null as prescription,
null as prescription_ly,
-------------------------------------
--Retail藏数逻辑标签------------------
1 PACK_FLAG,
1 PROD_FLAG,
0 DTP_FLAG,
null as cmps_flag,
-------------------------------------
--EC数据标签--------------------------
A.PLATFORM AS PLATFORM_TYPE,
A.STORE_NAME,
A.STORE_TYPE,
-------------------------------------
--COUNTY数据标签----------------------
'' REGION_TYPE,
-------------------------------------
--AIA数据标签-------------------------
null inst_code, --可以移除，报告没有使用
-------------------------------------
--XIE HE 数据标签---------------------
'' as dept_name,
'' as new_code,
'' as area,
'' as h_level,
'' as reimburse,
'' as reimburse_type,
'' as prescription_source,
-------------------------------------
'' as CORP_CODE --可以移除，报告没有使用
-------------------------------------
FROM
dm.dm_zk_ec_sales A
where
A.YYYYMM >= '202001'
GROUP BY
A.YYYYMM,
A.iqvia_pack_code,
A.AUDIT_COD,
A.PLATFORM,
A.STORE_NAME,
A.STORE_TYPE,
A.PACK_FLAG,
A.PROD_FLAG
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TF_EXT_EC_SALES
(
YYYYMM,
PACK_CODE,
AUDIT_CODE,
DATA_SOURCE,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
PRESCRIPTION,
PRESCRIPTION_LY,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
CMPS_FLAG,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
INST_CODE,
DEPT_NAME,
NEW_CODE,
AREA,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE,
CORP_CODE,
ETL_INSERT_DT,
ETL_UPDATE_DT
)
SELECT
YYYYMM,
NVL (
NULLIF(PACK_CODE, ''),
CONCAT ('PACK_CODE_', DATA_SOURCE)
) AS PACK_CODE,
NVL (NULLIF(AUDIT_CODE, ''), 'ROC') AS AUDIT_CODE,
DATA_SOURCE,
SALES_UNIT_CAL,
SALES_UNIT_CAL_LY,
SALES_VALUE_CAL,
SALES_VALUE_CAL_LY,
CONUTING_UNIT,
CONUTING_UNIT_LY,
PRESCRIPTION,
PRESCRIPTION_LY,
PACK_FLAG,
PROD_FLAG,
DTP_FLAG,
CMPS_FLAG,
PLATFORM_TYPE,
STORE_NAME,
STORE_TYPE,
REGION_TYPE,
INST_CODE,
DEPT_NAME,
NEW_CODE,
AREA,
H_LEVEL,
REIMBURSE,
REIMBURSE_TYPE,
PRESCRIPTION_SOURCE,
CORP_CODE,
FROM_UTC_TIMESTAMP (CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP (CURRENT_TIMESTAMP(), 'UTC+8') AS ETL_UPDATE_DT
FROM
EC_SALES
;

208
EC/06 DM_TD_EXT_EC_PACK2MARKET.sql Normal file
View File

@@ -0,0 +1,208 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_EC_PACK2MARKET (
-- MARKET STRING,
-- PACK_CODE STRING,
-- PACK_DESC STRING,
-- STGH_DESC STRING,
-- PACK_LCH STRING,
-- PROD_CODE STRING,
-- CMPS_CODE STRING,
-- CMPS_DESC STRING,
-- ATC1_CODE STRING,
-- ATC2_CODE STRING,
-- ATC3_CODE STRING,
-- ATC4_CODE STRING,
-- APP1_CODE STRING,
-- APP2_CODE STRING,
-- APP3_CODE STRING,
-- BIO_DESC STRING,
-- GENE_ORIG_DESC STRING,
-- ETH_OTC_DESC STRING,
-- NRDL_DESC STRING,
-- NRDL_ENTRY_DATE STRING,
-- EDL_DESC STRING,
-- TCM_DESC STRING,
-- PAED_DESC STRING,
-- GQCE_DESC STRING,
-- VBP_DESC STRING,
-- MANU_CODE STRING,
-- MANU_DESC STRING,
-- MNFL_CODE STRING,
-- MNFL_DESC STRING,
-- CORP_CODE STRING,
-- CORP_DESC STRING,
-- BRANDTYPE STRING,
-- BU STRING,
-- STARTTIME STRING,
-- ENDTIME STRING,
-- MARKET_RATIO STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_ec_pack2market';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_ec_pack2market';
-- COMMAND ----------
-- 使用 EXCEPT（集合差操作）简化逻辑
INSERT OVERWRITE TABLE DM.DM_TD_EXT_EC_PACK2MARKET
-- 包含规则
SELECT DISTINCT
T2.MARKET,
T1.PACK_CODE,
T1.PACK_DESC,
T1.STGH_DESC,
T1.PACK_LCH,
T1.PROD_CODE,
T1.CMPS_CODE,
T1.CMPS_DESC,
T1.ATC1_CODE,
T1.ATC2_CODE,
T1.ATC3_CODE,
T1.ATC4_CODE,
T1.APP1_CODE,
T1.APP2_CODE,
T1.APP3_CODE,
T1.BIO_DESC,
T1.GENE_ORIG_DESC AS GENE_ORIG,
T1.ETH_OTC_DESC AS ETH_OTC_D,
T1.NRDL_DESC,
T1.NRDL_ENTRY_DATE AS NRDL_ENTR,
T1.EDL_DESC,
T1.TCM_DESC,
T1.PAED_DESC,
T1.GQCE_DESC,
T1.VBP_DESC,
T1.MANU_CODE,
T1.MANU_DESC,
T1.MNFL_CODE,
T1.MNFL_DESC,
T1.CORP_CODE,
T1.CORP_DESC,
T1.BRANDTYPE,
T2.BU,
T2.STARTTIME,
T2.ENDTIME,
COALESCE(T2.EXTEND_MARKET_RATIO, '1') AS MARKET_RATIO,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_EC_PACK_PROPERTY AS T1
LEFT JOIN (
SELECT
MARKET,
BU,
STARTTIME,
ENDTIME,
EXTEND_MARKET_RATIO,
ATC1_CODE,
ATC2_CODE,
ATC3_CODE,
ATC4_CODE,
NFC1_CODE,
NFC2_CODE,
NFC3_CODE,
CORPORATION_CODE,
MANUFACTURER_CODE,
PRODUCT_CODE,
PACK_CODE,
STRENGTH,
MOLECULE_CODE
FROM DWD.DWD_GND_IMS_TBLMARKET_BYMONTH
WHERE EXTEND_MARKET IS NULL AND (NOT_IN_FLAG IS NULL OR NOT_IN_FLAG = '1')
) AS T2
ON (T2.ATC1_CODE IS NULL OR T1.ATC1_CODE = T2.ATC1_CODE)
AND (T2.ATC2_CODE IS NULL OR T1.ATC2_CODE = T2.ATC2_CODE)
AND (T2.ATC3_CODE IS NULL OR T1.ATC3_CODE = T2.ATC3_CODE)
AND (T2.ATC4_CODE IS NULL OR T1.ATC4_CODE = T2.ATC4_CODE)
AND (T2.NFC1_CODE IS NULL OR T1.APP1_CODE = T2.NFC1_CODE)
AND (T2.NFC2_CODE IS NULL OR T1.APP2_CODE = T2.NFC2_CODE)
AND (T2.NFC3_CODE IS NULL OR T1.APP3_CODE = T2.NFC3_CODE)
AND (T2.CORPORATION_CODE IS NULL OR T1.CORP_CODE = T2.CORPORATION_CODE)
AND (T2.MANUFACTURER_CODE IS NULL OR T1.MANU_CODE = T2.MANUFACTURER_CODE)
AND (T2.PRODUCT_CODE IS NULL OR T1.PROD_CODE = T2.PRODUCT_CODE)
AND (T2.PACK_CODE IS NULL OR T1.PACK_CODE = T2.PACK_CODE)
AND (T2.STRENGTH IS NULL OR T1.STGH_DESC = T2.STRENGTH)
AND (T2.MOLECULE_CODE IS NULL OR T1.CMPS_CODE = T2.MOLECULE_CODE)
WHERE T2.MARKET IS NOT NULL
EXCEPT
-- 排除规则
SELECT DISTINCT
T2.MARKET,
T1.PACK_CODE,
T1.PACK_DESC,
T1.STGH_DESC,
T1.PACK_LCH,
T1.PROD_CODE,
T1.CMPS_CODE,
T1.CMPS_DESC,
T1.ATC1_CODE,
T1.ATC2_CODE,
T1.ATC3_CODE,
T1.ATC4_CODE,
T1.APP1_CODE,
T1.APP2_CODE,
T1.APP3_CODE,
T1.BIO_DESC,
T1.GENE_ORIG_DESC AS GENE_ORIG,
T1.ETH_OTC_DESC AS ETH_OTC_D,
T1.NRDL_DESC,
T1.NRDL_ENTRY_DATE AS NRDL_ENTR,
T1.EDL_DESC,
T1.TCM_DESC,
T1.PAED_DESC,
T1.GQCE_DESC,
T1.VBP_DESC,
T1.MANU_CODE,
T1.MANU_DESC,
T1.MNFL_CODE,
T1.MNFL_DESC,
T1.CORP_CODE,
T1.CORP_DESC,
T1.BRANDTYPE,
T2.BU,
T2.STARTTIME,
T2.ENDTIME,
COALESCE(T2.EXTEND_MARKET_RATIO, '1') AS MARKET_RATIO,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_EC_PACK_PROPERTY AS T1
LEFT JOIN (
SELECT
MARKET,
BU,
STARTTIME,
ENDTIME,
EXTEND_MARKET_RATIO,
ATC1_CODE,
ATC2_CODE,
ATC3_CODE,
ATC4_CODE,
NFC1_CODE,
NFC2_CODE,
NFC3_CODE,
CORPORATION_CODE,
MANUFACTURER_CODE,
PRODUCT_CODE,
PACK_CODE,
STRENGTH,
MOLECULE_CODE
FROM DWD.DWD_GND_IMS_TBLMARKET_BYMONTH
WHERE EXTEND_MARKET IS NULL AND NOT_IN_FLAG = '0'
) AS T2
ON (T2.ATC1_CODE IS NULL OR T1.ATC1_CODE = T2.ATC1_CODE)
AND (T2.ATC2_CODE IS NULL OR T1.ATC2_CODE = T2.ATC2_CODE)
AND (T2.ATC3_CODE IS NULL OR T1.ATC3_CODE = T2.ATC3_CODE)
AND (T2.ATC4_CODE IS NULL OR T1.ATC4_CODE = T2.ATC4_CODE)
AND (T2.NFC1_CODE IS NULL OR T1.APP1_CODE = T2.NFC1_CODE)
AND (T2.NFC2_CODE IS NULL OR T1.APP2_CODE = T2.NFC2_CODE)
AND (T2.NFC3_CODE IS NULL OR T1.APP3_CODE = T2.NFC3_CODE)
AND (T2.CORPORATION_CODE IS NULL OR T1.CORP_CODE = T2.CORPORATION_CODE)
AND (T2.MANUFACTURER_CODE IS NULL OR T1.MANU_CODE = T2.MANUFACTURER_CODE)
AND (T2.PRODUCT_CODE IS NULL OR T1.PROD_CODE = T2.PRODUCT_CODE)
AND (T2.PACK_CODE IS NULL OR T1.PACK_CODE = T2.PACK_CODE)
AND (T2.STRENGTH IS NULL OR T1.STGH_DESC = T2.STRENGTH)
AND (T2.MOLECULE_CODE IS NULL OR T1.CMPS_CODE = T2.MOLECULE_CODE)
WHERE T2.MARKET IS NOT NULL;

30
EC/07 DM_TD_EXT_EC_MARKET_RATIO.sql Normal file
View File

@@ -0,0 +1,30 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_EC_MARKET_RATIO (
-- MARKET STRING,
-- PACK_CODE STRING,
-- CMPS_CODE STRING,
-- STARTTIME STRING,
-- ENDTIME STRING,
-- MARKET_RATIO STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_ec_market_ratio';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_ec_market_ratio';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_EC_MARKET_RATIO
SELECT
DISTINCT
MARKET,
PACK_CODE,
CMPS_CODE,
STARTTIME,
ENDTIME,
MARKET_RATIO,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_EC_PACK2MARKET
WHERE UPPER(MARKET) NOT LIKE '%ALL%'

163
EC/08 DM_TD_EXT_EC_MARKET_BRAND_RATIO.sql Normal file
View File

@@ -0,0 +1,163 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_EC_MARKET_BRAND_RATIO (
-- MARKET STRING,
-- PACK_CODE STRING,
-- VALUE_BRAND_RATIO FLOAT,
-- VALUE_BRAND_RATIO_START STRING,
-- VALUE_BRAND_RATIO_END STRING,
-- UNIT_BRAND_RATIO FLOAT,
-- UNIT_BRAND_START STRING,
-- UNIT_BRAND_END STRING,
-- COUNTINGUNIT_BRAND_RATIO FLOAT,
-- COUNTINGUNIT_BRAND_START STRING,
-- COUNTINGUNIT_BRAND_END STRING,
-- PDOT_BRAND_RATIO FLOAT,
-- PDOT_BRAND_START STRING,
-- PDOT_BRAND_END STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_ec_market_brand_ratio';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_ec_market_brand_ratio';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_EC_MARKET_BRAND_RATIO
SELECT
DISTINCT
T1.MARKET,
T1.PACK_CODE,
CASE
WHEN T2.RATIO IS NOT NULL THEN T2.RATIO
ELSE NVL(T5.RATIO, 1) END AS VALUE_BRAND_RATIO,
CASE
WHEN T2.RATIO IS NOT NULL THEN T2.STARTTIME
ELSE NVL(T5.STARTTIME, 200001) END AS VALUE_BRAND_RATIO_START,
CASE
WHEN T2.RATIO IS NOT NULL THEN T2.ENDTIME
ELSE NVL(T5.ENDTIME, 299912) END AS VALUE_BRAND_RATIO_END,
CASE
WHEN T3.RATIO IS NOT NULL THEN T3.RATIO
ELSE NVL(T6.RATIO, 1) END AS UNIT_BRAND_RATIO,
CASE
WHEN T3.RATIO IS NOT NULL THEN T3.STARTTIME
ELSE NVL(T6.STARTTIME, 200001) END AS UNIT_BRAND_START,
CASE
WHEN T3.RATIO IS NOT NULL THEN T3.ENDTIME
ELSE NVL(T6.ENDTIME, 299912) END AS UNIT_BRAND_END,
1 AS COUNTINGUNIT_BRAND_RATIO,
200001 AS COUNTINGUNIT_BRAND_START,
299912 AS COUNTINGUNIT_BRAND_END,
CASE
WHEN T4.RATIO IS NOT NULL THEN T4.RATIO
ELSE NVL(T7.RATIO, 1) END AS PDOT_BRAND_RATIO,
CASE
WHEN T4.RATIO IS NOT NULL THEN T4.STARTTIME
ELSE NVL(T7.STARTTIME, 200001) END AS PDOT_BRAND_START,
CASE
WHEN T4.RATIO IS NOT NULL THEN T4.ENDTIME
ELSE NVL(T7.ENDTIME, 299912) END AS PDOT_BRAND_END,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM DM.DM_TD_EXT_EC_MARKET_RATIO T1
LEFT JOIN ( --VALUE--PACK
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'VALUE' AND UPPER(LEVEL) = 'PACK'
) T2 ON T1.MARKET = T2.MARKET AND T1.PACK_CODE = T2.PACK_COD
LEFT JOIN ( --UNIT--PACK
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'UNIT' AND UPPER(LEVEL) = 'PACK'
) T3 ON T1.MARKET = T3.MARKET AND T1.PACK_CODE = T3.PACK_COD
LEFT JOIN ( --COUNTINGUNIT--PACK
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'COUNTINGUNIT' AND UPPER(LEVEL) = 'PACK'
) T4 ON T1.MARKET = T4.MARKET AND T1.PACK_CODE = T4.PACK_COD
LEFT JOIN ( --VALUE--MOLECULE
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'VALUE' AND UPPER(LEVEL) = 'MOLECULE'
) T5 ON T1.MARKET = T5.MARKET AND T1.CMPS_CODE = T5.CMPS_COD
LEFT JOIN ( --UNIT--MOLECULE
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'UNIT' AND UPPER(LEVEL) = 'MOLECULE'
) T6 ON T1.MARKET = T6.MARKET AND T1.CMPS_CODE = T6.CMPS_COD
LEFT JOIN ( --COUNTINGUNIT--MOLECULE
SELECT
DISTINCT
MARKET,
CMPS_COD,
CMPS_DES,
PACK_COD,
PACK_DES,
LEVEL,
TYPE,
CAST(RATIO AS FLOAT) AS RATIO,
CAST(STARTTIME AS INT) STARTTIME,
CAST(ENDTIME AS INT) ENDTIME
FROM DWD.DWD_GND_IMS_TBLBRANDRATIO_BYMONTH
WHERE UPPER(TYPE) = 'COUNTINGUNIT' AND UPPER(LEVEL) = 'MOLECULE'
) T7 ON T1.MARKET = T7.MARKET
AND T1.CMPS_CODE = T7.CMPS_COD
-- COMMAND ----------

62
EC/09 DM_TD_EXT_EC_MARKET_PACK_MAPPING.sql Normal file
View File

@@ -0,0 +1,62 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_EC_MARKET_PACK_MAPPING
-- (
-- PACK_CODE STRING,
-- MARKET STRING,
-- CLASS STRING,
-- KEY_COMPETITOR STRING,
-- MARKET_RATIO STRING,
-- STARTTIME STRING,
-- ENDTIME STRING,
-- VALUE_BRAND_RATIO STRING,
-- VALUE_BRAND_RATIO_START STRING,
-- VALUE_BRAND_RATIO_END STRING,
-- UNIT_BRAND_RATIO STRING,
-- UNIT_BRAND_START STRING,
-- UNIT_BRAND_END STRING,
-- COUNTINGUNIT_BRAND_RATIO STRING,
-- COUNTINGUNIT_BRAND_START STRING,
-- COUNTINGUNIT_BRAND_END STRING,
-- PDOT_BRAND_RATIO STRING,
-- PDOT_BRAND_START STRING,
-- PDOT_BRAND_END STRING,
-- ETL_INSERT_DT TIMESTAMP,
-- ETL_UPDATE_DT TIMESTAMP
-- )
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_ext_ec_market_pack_mapping';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_ec_market_pack_mapping';
-- COMMAND ----------
INSERT OVERWRITE TABLE DM.DM_TD_EXT_EC_MARKET_PACK_MAPPING
SELECT DISTINCT
T1.PACK_CODE,
T1.MARKET,
T1.CLASS,
T1.KEY_COMPETITOR,
T2.MARKET_RATIO,
T2.STARTTIME,
T2.ENDTIME,
T3.VALUE_BRAND_RATIO,
T3.VALUE_BRAND_RATIO_START,
T3.VALUE_BRAND_RATIO_END,
T3.UNIT_BRAND_RATIO,
T3.UNIT_BRAND_START,
T3.UNIT_BRAND_END,
T3.COUNTINGUNIT_BRAND_RATIO,
T3.COUNTINGUNIT_BRAND_START,
T3.COUNTINGUNIT_BRAND_END,
T3.PDOT_BRAND_RATIO,
T3.PDOT_BRAND_START,
T3.PDOT_BRAND_END,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM
DM.DM_TD_EXT_EC_PACK_PROPERTY T1
LEFT JOIN DM.DM_TD_EXT_EC_MARKET_RATIO T2
ON T1.MARKET = T2.MARKET AND T1.PACK_CODE = T2.PACK_CODE
LEFT JOIN DM.DM_TD_EXT_EC_MARKET_BRAND_RATIO T3
ON T1.MARKET = T3.MARKET AND T1.PACK_CODE = T3.PACK_CODE
WHERE UPPER(T1.MARKET) NOT LIKE "%ALL%"

103
EC/1 (ec)blob_to_dwd.sql Normal file
View File

@@ -0,0 +1,103 @@
-- Databricks notebook source
-- MAGIC %python
-- MAGIC import datetime
-- MAGIC from pyspark.sql.functions import current_timestamp, expr, date_format
-- MAGIC
-- MAGIC # 计算当前日期
-- MAGIC current_date_utc = datetime.datetime.utcnow()
-- MAGIC offset = datetime.timedelta(hours=8)
-- MAGIC current_date = current_date_utc + offset
-- MAGIC
-- MAGIC today_path = "{:04d}/{:02d}/{:02d}/".format(
-- MAGIC current_date.year,
-- MAGIC current_date.month,
-- MAGIC current_date.day,
-- MAGIC )
-- MAGIC # 基础路径
-- MAGIC # 测试环境
-- MAGIC base_path_0 = f"abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/ODS/GND/UserUpload/"
-- MAGIC # 生产环境
-- MAGIC # base_path_0 = f"abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/ODS/GND/UserUpload/"
-- MAGIC base_path = base_path_0 + today_path
-- MAGIC print(base_path)
-- MAGIC
-- MAGIC # 检查基础路径是否存在
-- MAGIC def check_path_exists(path):
-- MAGIC try:
-- MAGIC dbutils.fs.ls(path)
-- MAGIC return True
-- MAGIC except Exception as e:
-- MAGIC return False
-- MAGIC
-- MAGIC if check_path_exists(base_path):
-- MAGIC # 列出所有批次路径
-- MAGIC batch_paths = dbutils.fs.ls(base_path)
-- MAGIC
-- MAGIC # 从已存在的配置表中读取数据
-- MAGIC config_df = spark.table("dwd.dwd_gnd_ec_config_table")
-- MAGIC
-- MAGIC # 逐批处理
-- MAGIC for batch in batch_paths:
-- MAGIC current_batch_number = int(batch.name.strip('/'))
-- MAGIC # print(f"Checking batch {current_batch_number} at {batch.path}")
-- MAGIC
-- MAGIC files_in_batch = dbutils.fs.ls(batch.path)
-- MAGIC
-- MAGIC print("该批次中的文件:")
-- MAGIC for file in files_in_batch:
-- MAGIC print(file.name)
-- MAGIC
-- MAGIC for row in config_df.collect():
-- MAGIC file_name = row['file_name'].strip().lower()
-- MAGIC table_name = row['table_name']
-- MAGIC
-- MAGIC # 检查文件是否匹配
-- MAGIC matching_files = [f for f in files_in_batch if f.name.strip().lower() == file_name]
-- MAGIC
-- MAGIC for match in matching_files:
-- MAGIC csv_file_path = batch.path + match.name
-- MAGIC print(f"找到匹配的文件: {csv_file_path}")
-- MAGIC
-- MAGIC # 读取 CSV 文件
-- MAGIC df = spark.read.format("csv").option("header", "true").option("charset", "GBK").load(csv_file_path)
-- MAGIC
-- MAGIC # 对列进行重命名（假设所有文件的列名相同）
-- MAGIC df = df.withColumnRenamed('时间（月度）', 'time')
-- MAGIC df = df.withColumnRenamed('平台', 'platform')
-- MAGIC df = df.withColumnRenamed('店铺名称', 'store_name')
-- MAGIC df = df.withColumnRenamed('店铺类型', 'store_type')
-- MAGIC df = df.withColumnRenamed('产品ID', 'product_id')
-- MAGIC df = df.withColumnRenamed('品牌', 'brand_name')
-- MAGIC df = df.withColumnRenamed('品名', 'category_name')
-- MAGIC df = df.withColumnRenamed('商品名', 'product_name')
-- MAGIC df = df.withColumnRenamed('通用名', 'common_name')
-- MAGIC df = df.withColumnRenamed('厂家', 'factory')
-- MAGIC df = df.withColumnRenamed('集团权益', 'group_interest')
-- MAGIC df = df.withColumnRenamed('规格', 'specification')
-- MAGIC df = df.withColumnRenamed('单件包装盒数', 'pcs_per_box')
-- MAGIC df = df.withColumnRenamed('剂型', 'dosage')
-- MAGIC df = df.withColumnRenamed('细分一', 'des1')
-- MAGIC df = df.withColumnRenamed('细分二', 'des2')
-- MAGIC df = df.withColumnRenamed('细分三', 'des3')
-- MAGIC df = df.withColumnRenamed('细分四', 'des4')
-- MAGIC df = df.withColumnRenamed('销售额', 'sales_amount')
-- MAGIC df = df.withColumnRenamed('成交件数', 'sold_pcs')
-- MAGIC df = df.withColumnRenamed('平均单价（元/件）', 'aup_pices')
-- MAGIC df = df.withColumnRenamed('销售量（盒）', 'sales_qty')
-- MAGIC df = df.withColumnRenamed('平均单价（元/盒）', 'aup_box')
-- MAGIC df = df.withColumnRenamed('PROD_COD', 'PROD_COD')
-- MAGIC df = df.withColumnRenamed('PACK_COD', 'PACK_COD')
-- MAGIC df = df.withColumnRenamed('APP3_COD', 'APP3_COD')
-- MAGIC df = df.withColumnRenamed('ATC4_COD', 'ATC4_COD')
-- MAGIC
-- MAGIC # 添加 'etl_insert_dt' 列，包含当前时间并加上 8 小时的时差
-- MAGIC df = df.withColumn('etl_insert_dt', date_format(expr("current_timestamp() + INTERVAL 8 HOURS"), 'yyyy-MM-dd HH:mm:ss'))
-- MAGIC
-- MAGIC # 将数据保存到目标表
-- MAGIC df.write.mode("overwrite").saveAsTable(table_name)
-- MAGIC
-- MAGIC print(f"数据已写入表 {table_name}")
-- MAGIC else:
-- MAGIC print("未找到批次或路径不存在。")
-- MAGIC
-- MAGIC

74
EC/2 dwd_inc_gnd_ext_ec_nationnal_pack_union_all.py Normal file
View File

@@ -0,0 +1,74 @@
# Databricks notebook source
from pyspark.sql.functions import lit
from pyspark.sql.utils import AnalysisException
# pack数据自动接入 整合
#获取配置表信息(表名、brand_flag
df = spark.sql("""
SELECT DISTINCT table_name tab ,file_name brand_flag FROM dwd.dwd_gnd_ec_config_table
where type_name ='PACK'
""").collect()
# print(df)
def get_union_pack_data(df):
#初始化结果集
union_query = None
# niad_pdot_unit需特殊赋值对应表名：tmp.tmp_inc_gnd_ext_retail_nataional_niad
# niad_pdot_unit_flag = 'tmp.tmp_inc_gnd_ext_retail_nataional_niad'
for table in df:
# 选择当前表名
table_name = table['tab']
if not table_name:
print("由于table_name为空，故跳过查询")
continue
# 获取对应brand表维度对应得 market 名称
# brand_flag = str(table.brand_flag)
sql = f"""
select
t1.time,
t1.platform,
t1.store_name,
t1.store_type,
t1.product_id,
t1.brand_name,
t1.category_name,
t1.product_name,
t1.common_name,
t1.factory,
t1.group_interest,
t1.specification,
t1.pcs_per_box,
t1.dosage,
t1.des1,
t1.des2,
t1.des3,
t1.des4,
t1.sales_amount,
t1.sold_pcs,
t1.aup_pices,
t1.sales_qty,
t1.aup_box,
t1.PROD_COD,
t1.PACK_COD,
t1.APP3_COD,
t1.ATC4_COD,
t1.etl_insert_dt
from {table_name} t1
"""
# 尝试读取数据，只有在表名非空时执行
try:
current_query = spark.sql(sql)
except AnalysisException as e:
print(f"Error querying table {table_name}: {e}")
continue # 如果查询发生错误，跳过当前循环迭代
#union 数据
if union_query ==None:
union_query=current_query
else:
union_query = union_query.union(current_query)
#返回数据集
return union_query
pack_result = get_union_pack_data(df)
if pack_result is not None:
pack_result.write.mode("overwrite").saveAsTable("dwd.dwd_inc_gnd_ext_ec_nationnal_pack_union_all")
else:
print("没有数据写入这张表里")

228
Merged_Data/Merged_Data_Config_table_bkp.sql Normal file
View File

@@ -0,0 +1,228 @@
-- Databricks notebook source
create or replace temporary view brand_province_config -- brand配置表
as
select upper(`brand`) as brand, `province` as province, datasource as data_source from dwd.dwd_gnd_merge_data_brand_province
-- COMMAND ----------
create or replace temporary view market_province_config --market 配置表
as
select market, province, datasource as data_source from dwd.dwd_gnd_merge_data_brand_market
-- COMMAND ----------
insert overwrite dws.dws_external_merge_data_config_special
-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DWS/dws_external_merge_data_config_special' --prd
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DWS/dws_external_merge_data_config_special' --test
with pack_and_brand ( --brand 转 pack
select
upper(brandnamee) brand_name
,ims_pack_cod PACK_COD --取所有code带出market
from dwd.dwd_td_imscode_azcode
where ims_pack_cod is not null and startswith(ims_pack_cod,'0')
and upper(brandnamee) in (select brand from brand_province_config group by 1 )--取出在配置表中的pack
group by 1,2
)
,pack_to_market (
select
a.PACK_COD
,MARKET
from dm.dm_td_external_market_pack_mapping a
inner join pack_and_brand b on a.PACK_COD = b.pack_cod --取出配置表中的brand对应的 market
where DATA_SOURCE in ( 'AIA(Monthly)')--只取 AIA中的市场，chpa的市场太多
and lower(a.MARKET) not like '%all market%'
and market not in (
'RE Extended Market'
,'RE Market'
,'RE Market (Global)'
,'Pulmicort VBP Benchmark Market'
,'Respules Market'
)
group by 1,2
)
,market_to_pack ( -- 获取上述market中所有的pack
select
market
,pack_cod
from dm.dm_td_external_market_pack_mapping a
where DATA_SOURCE in ( 'AIA(Monthly)','IQVIA-CHPA(Monthly)')--AIA和chpa的市场
and market in (select market from pack_to_market group by 1)
group by 1,2
)
,province_to_audit (
select
PROVINCE_C
,AUDIT_COD
,DATA_SOURCE
,CASE WHEN DATA_SOURCE = 'AIA(Monthly)' THEN 'AIA'
WHEN DATA_SOURCE = 'IQVIA-CHPA(Monthly)' THEN 'CHPA'
ELSE NULL
END AS SOURCENAME
from dm.dm_td_external_geo
where DATA_SOURCE in ( 'AIA(Monthly)','IQVIA-CHPA(Monthly)')
group by 1,2,3
)
,not_selected_datasource (--因为pack2market2pack 会导致部分大市场包含所有的pack，故用配置表再删除一些不需要配置数据源的
select
a.ims_pack_cod PACK_COD --取所有code
,b.province
from dwd.dwd_td_imscode_azcode a
inner join (
select
UPPER(brand) as brand
,province
from brand_province_config
where data_source = '/'
group by 1,2
) b
on upper(a.brandnamee) = upper(b.brand )
where a.ims_pack_cod is not null and startswith(a.ims_pack_cod,'0')
group by 1,2
union
select
pack_cod,
'天津'
from market_to_pack
where market= 'EGFR TKI Market'
group by 1
)
select DATA_SOURCE,PACK_COD,AUDIT_COD
from not_selected_datasource a
left join province_to_audit b on a.province = b.province_c
group by 1,2,3
-- COMMAND ----------
INSERT OVERWRITE dws.dws_external_merge_data_config
-- create or replace table dws.dws_external_merge_data_config
/*
1通过brand找到pack，通过pack找到市场以市场下的所有pack
2通过province找到对应的省份或者机构，从而获取到audit_code
*/
----pack
with pack_and_brand ( --brand 转 pack
select
upper(brandnamee) brand_name
,ims_pack_cod PACK_COD --取所有code带出market
from dwd.dwd_td_imscode_azcode
where ims_pack_cod is not null and startswith(ims_pack_cod,'0')
and upper(brandnamee) in (select brand from brand_province_config group by 1 )--取出在配置表中的pack
group by 1,2
)
,pack_to_market (
select
a.PACK_COD
,MARKET
from dm.dm_td_external_market_pack_mapping a
inner join pack_and_brand b on a.PACK_COD = b.pack_cod --取出配置表中的brand对应的 market
where DATA_SOURCE in ( 'AIA(Monthly)')--只取 AIA中的市场，chpa的市场太多
and lower(a.MARKET) not like '%all market%'
and market not in (
'RE Extended Market'
,'RE Market'
,'RE Market (Global)'
,'Pulmicort VBP Benchmark Market'
,'Respules Market'
)
group by 1,2
)
,market_to_pack ( -- 获取上述market中所有的pack
select
market
,pack_cod
from dm.dm_td_external_market_pack_mapping a
where DATA_SOURCE in ( 'AIA(Monthly)','IQVIA-CHPA(Monthly)')--AIA和chpa的市场
and market in (select market from pack_to_market group by 1)
group by 1,2
)
,province_to_audit (
select
PROVINCE_C
,AUDIT_COD
,DATA_SOURCE
,CASE WHEN DATA_SOURCE = 'AIA(Monthly)' THEN 'AIA'
WHEN DATA_SOURCE = 'IQVIA-CHPA(Monthly)' THEN 'CHPA'
ELSE NULL
END AS SOURCENAME
from dm.dm_td_external_geo
where DATA_SOURCE in ( 'AIA(Monthly)','IQVIA-CHPA(Monthly)')
group by 1,2,3
)
,pack_result as (
select
D.pack_cod
,E.audit_cod
,A.province
,A.data_source as SOURCENAME
,E.data_source
from brand_province_config A
inner join pack_and_brand B on A.brand = B.brand_name
inner join pack_to_market C on B.pack_cod = C.pack_cod
inner join market_to_pack D on C.market = D.market
inner join province_to_audit E on A.province = E.province_c AND A.data_source = E.SOURCENAME
where A.data_source <> '/'
group by 1,2,3,4,5
)
----market
,config_market (
select
market
from market_province_config
group by 1
)
,market_to_pack_dtp (
select
PACK_COD,MARKET
from dm.dm_td_external_market_pack_mapping
where DATA_SOURCE = 'DTP(Quarterly)' and PACK_COD in (
select
PACK_COD
from dm.dm_tf_external_sales
where DTP_FLAG =1
group by 1)
)
,final_market_to_pack (--获取需要转换market成pack的部分
select PACK_COD,a.MARKET from market_to_pack a inner join config_market b on a.market = b.market
union
select PACK_COD,a.MARKET from market_to_pack_dtp a inner join config_market b on a.market = b.market
)
----market
,market_result as (
select
B.pack_cod
,C.AUDIT_COD
,A.province
,A.data_source as SOURCENAME
,C.DATA_SOURCE
from market_province_config a
left join final_market_to_pack B on A.market = B.market
left join province_to_audit C on A.province = C.province_c AND A.data_source = C.SOURCENAME
where a.data_source <> '/' AND C.AUDIT_COD IS not NULL
group by 1,2,3,4,5)
,need_to_be_changed( --算出不在pack 中的pack code，替换成 market中的pack code 的配置信息
select
pack_cod,AUDIT_COD,DATA_SOURCE,sourcename
from market_result
where pack_cod in (select pack_cod from pack_result group by 1)
)
--最终写入表
select
pack_cod,AUDIT_COD,DATA_SOURCE,sourcename, 'pack' CONFIG_DATASOURCE
from pack_result
union ALL
select
pack_cod,AUDIT_COD,DATA_SOURCE,sourcename, 'market' CONFIG_DATASOURCE
from pack_result
where pack_cod not in (select pack_cod from need_to_be_changed group by 1)
union all
select
pack_cod,AUDIT_COD,DATA_SOURCE,sourcename, 'market' CONFIG_DATASOURCE
from need_to_be_changed

213
Merged_Data/Merged_Data_Config_table_bymonth.sql Normal file
View File

@@ -0,0 +1,213 @@
-- Databricks notebook source
-- create or replace table dws.dws_td_ext_merge_data_config
-- (
-- market string,
-- audit_cod string,
-- province string,
-- sourcename string,
-- data_source string
-- )
-- USING delta
-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dws_td_ext_merge_data_config';
-- -- 上面是生产环境location，下面是测试环境location
-- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dws_td_ext_merge_data_config';
-- COMMAND ----------
create or replace temporary view brand_province_config -- brand配置表
as
select upper(`brand`) as brand, `province` as province, datasource as data_source from dwd.dwd_gnd_merge_data_brand_province
-- COMMAND ----------
create or replace temporary view market_province_config --market 配置表
as
select market, province, datasource as data_source from dwd.dwd_gnd_merge_data_brand_market
-- COMMAND ----------
-- -- insert overwrite dws.dws_td_ext_merge_data_config_special
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DWS/dws_external_merge_data_config_special' --prd
-- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DWS/dws_external_merge_data_config_special' --test
-- with pack_and_brand ( --brand 转 pack
-- select
-- upper(brandnamee) brand_name
-- ,ims_pack_cod PACK_COD --取所有code带出market
-- from dwd.dwd_td_imscode_azcode
-- where ims_pack_cod is not null and startswith(ims_pack_cod,'0')
-- and upper(brandnamee) in (select brand from brand_province_config group by 1 )--取出在配置表中的pack
-- group by 1,2
-- )
-- ,pack_to_market (
-- select
-- a.PACK_CODE
-- ,MARKET
-- from dm.dm_td_ext_unionall_market_pack_mapping a
-- inner join pack_and_brand b on a.PACK_CODE = b.pack_cod --取出配置表中的brand对应的 market
-- where a.DATASOURCE in ( 'AIA(Monthly)')--只取 AIA中的市场，chpa的市场太多
-- and lower(a.MARKET) not like '%all market%'
-- and market not in (
-- 'RE Extended Market'
-- ,'RE Market'
-- ,'RE Market (Global)'
-- ,'Pulmicort VBP Benchmark Market'
-- ,'Respules Market'
-- )
-- group by 1,2
-- )
-- -- ,market_to_pack ( -- 获取上述market中所有的pack
-- -- select
-- -- market
-- -- ,pack_cod
-- -- from dm.dm_td_external_market_pack_mapping a
-- -- where DATA_SOURCE in ( 'AIA(Monthly)','IQVIA-CHPA(Monthly)')--AIA和chpa的市场
-- -- and market in (select market from pack_to_market group by 1)
-- -- group by 1,2
-- -- )
-- ,province_to_audit (
-- select
-- PROVINCE_C
-- ,AUDIT_COD
-- ,DATA_SOURCE
-- ,CASE WHEN DATA_SOURCE = 'AIA(Monthly)' THEN 'AIA'
-- WHEN DATA_SOURCE = 'IQVIA-CHPA(Monthly)' THEN 'CHPA'
-- ELSE NULL
-- END AS SOURCENAME
-- from dm.dm_td_external_geo
-- where DATA_SOURCE in ( 'AIA(Monthly)','IQVIA-CHPA(Monthly)')
-- group by 1,2,3
-- )
-- ,not_selected_datasource (--因为pack2market2pack 会导致部分大市场包含所有的pack，故用配置表再删除一些不需要配置数据源的
-- select
-- a.ims_pack_cod PACK_COD --取所有code
-- ,b.province
-- from dwd.dwd_td_imscode_azcode a
-- inner join (
-- select
-- UPPER(brand) as brand
-- ,province
-- from brand_province_config
-- where data_source = '/'
-- group by 1,2
-- ) b
-- on upper(a.brandnamee) = upper(b.brand )
-- where a.ims_pack_cod is not null and startswith(a.ims_pack_cod,'0')
-- group by 1,2
-- union
-- select
-- pack_cod,
-- '天津'
-- from market_to_pack
-- where market= 'EGFR TKI Market'
-- group by 1
-- )
-- select DATA_SOURCE,PACK_COD,AUDIT_COD
-- from not_selected_datasource a
-- left join province_to_audit b on a.province = b.province_c
-- group by 1,2,3
-- COMMAND ----------
INSERT OVERWRITE dws.dws_td_ext_merge_data_config
-- create or replace table dws.dws_td_ext_merge_data_config
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DWS/dws_td_ext_merge_data_config' --test
/*
1通过brand找到pack，通过pack找到市场以市场下的所有pack
2通过province找到对应的省份或者机构，从而获取到audit_code
*/
----pack
with pack_and_brand ( --brand 转 pack
select
upper(brandnamee) brand_name
,ims_pack_cod PACK_COD --取所有code带出market
from dwd.dwd_td_imscode_azcode
where ims_pack_cod is not null and startswith(ims_pack_cod,'0')
and upper(brandnamee) in (select brand from brand_province_config group by 1 )--取出在配置表中的pack
group by 1,2
)
,pack_to_market (
select
a.PACK_CODE,
MARKET
from dm.dm_td_ext_unionall_market_pack_mapping a
inner join pack_and_brand b on a.PACK_CODE = b.pack_cod --取出配置表中的brand对应的 market
where a.DATASOURCE in ( 'AIA(Monthly)')--只取 AIA中的市场，chpa的市场太多
and lower(a.MARKET) not like '%all market%'
and market not in (
'RE Extended Market'
,'RE Market'
,'RE Market (Global)'
,'Pulmicort VBP Benchmark Market'
,'Respules Market'
)
group by 1,2
)
,province_to_audit (
select
PROVINCE_C
,AUDIT_COD
,DATA_SOURCE
,CASE WHEN DATA_SOURCE = 'AIA(Monthly)' THEN 'AIA'
WHEN DATA_SOURCE = 'IQVIA-CHPA(Monthly)' THEN 'CHPA'
ELSE NULL
END AS SOURCENAME
from dm.dm_td_external_geo
where DATA_SOURCE in ( 'AIA(Monthly)','IQVIA-CHPA(Monthly)')
group by 1,2,3
)
,pack_result as (
select
C.market
,E.audit_cod
,A.province
,A.data_source as SOURCENAME
,E.data_source
from brand_province_config A
inner join pack_and_brand B on A.brand = B.brand_name
inner join pack_to_market C on B.pack_cod = C.pack_code
-- inner join market_to_pack D on C.market = D.market
inner join province_to_audit E on A.province = E.province_c AND A.data_source = E.SOURCENAME
where A.data_source <> '/'
group by 1,2,3,4,5
)
---- MARKET PART
,market_result as (
select
A.MARKET
,C.AUDIT_COD
,A.province
,A.data_source as SOURCENAME
,C.DATA_SOURCE
from market_province_config a
-- left join final_market_to_pack B on A.market = B.market
left join province_to_audit C on A.province = C.province_c AND A.data_source = C.SOURCENAME
where a.data_source <> '/' AND C.AUDIT_COD IS not NULL
group by 1,2,3,4,5)
----最终结果
select
market,
audit_cod,
province,
sourcename,
data_source
FROM pack_result
where market NOT in (
select
market
from market_province_config
group by 1
)
union all
select
market,
audit_cod,
province,
sourcename,
data_source
FROM market_result

124
Merged_Data/dm_tf_exteranl_sales_merged_data_dtp_others_bkp.sql Normal file
View File

@@ -0,0 +1,124 @@
-- Databricks notebook source
-- create or replace table dm.dm_tf_exteranl_sales_merged_data_dtp_others
insert overwrite table dm.dm_tf_exteranl_sales_merged_data_dtp_others
with all_egfr_pack as (
select * from DM.dm_td_external_market_pack_mapping
where DATA_SOURCE = 'DTP(Quarterly)' and MARKET = 'EGFR TKI Market'
)
,4brand as (
select * from all_egfr_pack
where PROD_DES_C in ('阿美乐','泰瑞沙','凯美纳','艾弗沙')
)
,otherbrand as (
select PACK_COD from all_egfr_pack
where PACK_COD not in ( select PACK_COD from `4brand` group by 1)
group by 1
)
-- -- 直取表 cu - dtp4brand 的 cu = otherbrand 的cu
,dirct_cu as (
select
yyyymm
,replace(audit_key,'DTP(Quarterly)','') audit_cod
,cast(pdot_counting_unit as decimal ) pdot_counting_unit
,cast(pdot_counting_unit_ly as decimal ) pdot_counting_unit_ly
from dm.dm_tf_external_retail_special a
where yyyymm >=202401
)
--先 4个brand 的 cu 和 value
,fact_sales_dtp_4brand(
select
YYYYMM,AUDIT_COD
,sum(cast( CONUTING_UNIT*PDOT_MARKET_RATIO as decimal)) CONUTING_UNIT
,sum(cast( CONUTING_UNIT_LY*PDOT_MARKET_RATIO as decimal)) CONUTING_UNIT_LY
from dm.dm_tf_external_sales t1
inner join 4brand t2 on t1.PACK_COD = t2.PACK_COD
where t1.DATA_SOURCE = 'DTP(Quarterly)'
and DTP_FLAG = 1
and YYYYMM >=202401
group by
YYYYMM,AUDIT_COD )
-- 其他产品 的 cu 和 value
,fact_sales_dtp_other(
select
YYYYMM
,"DTPOTHERS" PACK_COD
,"DTPOTHERS" CORP_COD
,AUDIT_COD
,'' as PLATFORM_TYPE
,'' as STORE_NAME
,'' as STORE_TYPE
,'' as REGION_TYPE
,1 as PACK_FLAG
,2 as PROD_FLAG
,1 as DTP_FLAG
,sum(SALES_UNIT_CAL) SALES_UNIT_CAL
,sum(SALES_UNIT_CAL_LY) SALES_UNIT_CAL_LY
,sum(SALES_VALUE_CAL) SALES_VALUE_CAL
,sum(SALES_VALUE_CAL_LY) SALES_VALUE_CAL_LY
,sum(CONUTING_UNIT) CONUTING_UNIT
,sum(CONUTING_UNIT_LY) CONUTING_UNIT_LY
,'DTP(Quarterly)' as DATA_SOURCE
,'' as inst_code
,'' as cmps_flag
,'' as dept_name
,'' as prescription
,'' as prescription_ly
,'' as new_code
,'' as area
,'' as h_level
,'' as reimburse
,'' as reimburse_type
,'' as prescription_source
from dm.dm_tf_external_sales
where DTP_FLAG = 1
and PACK_COD in (select PACK_COD from otherbrand) --其他部分
and DATA_SOURCE = 'DTP(Quarterly)'
and YYYYMM >=202401
group by
YYYYMM,AUDIT_COD)
select
T0.YYYYMM
,t0.PACK_COD
,t0.CORP_COD
,t0.AUDIT_COD
,t0.PLATFORM_TYPE
,t0.STORE_NAME
,t0.STORE_TYPE
,t0.REGION_TYPE
,t0.PACK_FLAG
,t0.PROD_FLAG
,t0.DTP_FLAG
,t0.SALES_UNIT_CAL
,t0.SALES_UNIT_CAL_LY
,t0.SALES_VALUE_CAL
,t0.SALES_VALUE_CAL_LY
,t1.CONUTING_UNIT
,t1.CONUTING_UNIT_LY
,t0.DATA_SOURCE
,t0.inst_code
,t0.cmps_flag
,t0.dept_name
,t0.prescription
,t0.prescription_ly
,t0.new_code
,t0.area
,t0.h_level
,t0.reimburse
,t0.reimburse_type
,t0.prescription_source
from fact_sales_dtp_other t0
left join (
select
t1.yyyymm
,t1.audit_cod
,t1.pdot_counting_unit - t2.conuting_unit as CONUTING_UNIT
,t1.pdot_counting_unit_ly - t2.conuting_unit_ly as CONUTING_UNIT_LY
from dirct_cu t1
left join fact_sales_dtp_4brand t2 on t1.yyyymm = t2.yyyymm and t1.audit_cod = t2.audit_cod
) t1
on t0.audit_cod = t1.audit_cod and t0.yyyymm = t1.yyyymm
-- select * from fact_sales_dtp_4brand

289
Merged_Data/dm_tf_exteranl_sales_merged_data_dtp_others_bymonth_bkp.sql Normal file
View File

@@ -0,0 +1,289 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE dm.DM_TF_EXTERANL_SALES_MERGED_DATA_DTP_OTHERS_bymonth (
-- MARKET STRING,
-- KEY_COMPETITOR STRING,
-- CLASS STRING,
-- YYYYMM STRING,
-- PACK_COD STRING,
-- CORP_COD STRING,
-- AUDIT_COD STRING,
-- PLATFORM_TYPE STRING,
-- STORE_NAME STRING,
-- STORE_TYPE STRING,
-- REGION_TYPE STRING,
-- PACK_FLAG INT,
-- PROD_FLAG INT,
-- DTP_FLAG INT,
-- SALES_UNIT_CAL DECIMAL(38,10),
-- SALES_UNIT_CAL_LY DECIMAL(38,10),
-- SALES_VALUE_CAL DECIMAL(38,10),
-- SALES_VALUE_CAL_LY DECIMAL(38,10),
-- CONUTING_UNIT DECIMAL(21,0),
-- CONUTING_UNIT_LY DECIMAL(21,0),
-- PDOT DECIMAL(21,0),
-- PDOT_LY DECIMAL(21,0),
-- DATA_SOURCE STRING,
-- inst_code STRING,
-- cmps_flag STRING,
-- dept_name STRING,
-- prescription STRING,
-- prescription_ly STRING,
-- new_code STRING,
-- area STRING,
-- h_level STRING,
-- reimburse STRING,
-- reimburse_type STRING,
-- prescription_source STRING)
-- USING delta
-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/DM_TF_EXTERANL_SALES_MERGED_DATA_DTP_OTHERS_bymonth';
-- -- 上面是生产环境location，下面是测试环境location
-- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/DM_TF_EXTERANL_SALES_MERGED_DATA_DTP_OTHERS_bymonth';
-- COMMAND ----------
-- 这个jupyter核心逻辑是
-- 直接 cu - 4brand cu = other cu
-- 其中 维度部分,通过DM.DM_TF_EXT_UNIONALL_SALES innerjoin other
-- 其中 度量部分,通过 直接cu - 4brand cu
-- COMMAND ----------
CREATE OR REPLACE TEMPORARY VIEW RETAIL_DTP
AS
SELECT
nvl(t2.market,'ALL Market') MARKET
,nvl(t2.KEY_COMPETITOR,'OTHERS') as KEY_COMPETITOR
,nvl(t2.CLASS,'Others') as CLASS
,t1.YYYYMM
,t1.PACK_CODE
,t1.CORP_CODE
,t1.AUDIT_CODE
,t1.PLATFORM_TYPE
,t1.STORE_NAME
,t1.STORE_TYPE
,t1.REGION_TYPE
,t1.DATA_SOURCE
,t1.PACK_FLAG
,t1.PROD_FLAG
,t1.DTP_FLAG
,t1.CMPS_FLAG
,t1.NEW_CODE
,t1.INST_CODE
,NULL AS AIA_HP_FLAG
,t1.DEPT_NAME
,t1.H_LEVEL
,t1.REIMBURSE
,t1.REIMBURSE_TYPE
,PRESCRIPTION_SOURCE AS PRESCRIPTION_TYPE
,t1.PRESCRIPTION
,t1.PRESCRIPTION_LY
,nvl(nvl(t2.MARKET_RATIO,1)*t2.Value_brand_Ratio,1) as VA
,nvl(nvl(t2.MARKET_RATIO,1)*t2.Unit_brand_Ratio,1) AS UT
,nvl(nvl(t2.MARKET_RATIO,1)*t2.CountingUnit_brand_Ratio,1) AS CU
,nvl(nvl(t2.MARKET_RATIO,1)*t2.PDOT_brand_Ratio,1) AS PT
,cast(t1.SALES_UNIT_CAL * nvl(t2.Unit_brand_Ratio,1) *nvl(t2.MARKET_RATIO,1) as decimal(35,6)) AS SALES_UNIT_CAL
,cast(t1.SALES_UNIT_CAL_LY * nvl(t3.Unit_brand_Ratio,1) *nvl(t3.MARKET_RATIO,1) as decimal(35,6)) AS SALES_UNIT_CAL_LY
,cast(t1.SALES_VALUE_CAL * nvl(t2.Value_brand_Ratio,1) *nvl(t2.MARKET_RATIO,1) as decimal(35,6)) AS SALES_VALUE_CAL
,cast(t1.SALES_VALUE_CAL_LY * nvl(t3.Value_brand_Ratio,1) *nvl(t3.MARKET_RATIO,1) as decimal(35,6)) AS SALES_VALUE_CAL_LY
,cast(t1.CONUTING_UNIT * nvl(t2.CountingUnit_brand_Ratio,1) *nvl(t2.MARKET_RATIO,1) as decimal(35,6)) AS CONUTING_UNIT
,cast(t1.CONUTING_UNIT_LY * nvl(t3.CountingUnit_brand_Ratio,1) *nvl(t3.MARKET_RATIO,1) as decimal(35,6)) AS CONUTING_UNIT_LY
,cast(t1.CONUTING_UNIT * nvl(t2.PDOT_brand_Ratio,1) *nvl(t2.MARKET_RATIO,1) as decimal(35,6)) AS PDOT
,cast(t1.CONUTING_UNIT_LY * nvl(t3.PDOT_brand_Ratio,1) *nvl(t3.MARKET_RATIO,1) as decimal(35,6)) AS PDOT_LY
FROM DM.DM_TF_EXT_UNIONALL_SALES t1
LEFT JOIN DM.dm_td_ext_unionall_market_pack_mapping t2
on t1.PACK_CODE = t2.PACK_CODE and t1.DATA_SOURCE = t2.DATASOURCE
and t1.YYYYMM >=t2.starttime and t1.YYYYMM <=t2.endtime
and t1.YYYYMM >= t2.Value_brand_Ratio_start and t1.YYYYMM <=t2.Value_brand_Ratio_end
and t1.YYYYMM >= t2.Unit_brand_start and t1.YYYYMM <=t2.Unit_brand_end
and t1.YYYYMM >= t2.CountingUnit_brand_start and t1.YYYYMM <=t2.CountingUnit_brand_end
and t1.YYYYMM >= t2.PDOT_brand_start and t1.YYYYMM <=t2.PDOT_brand_end
LEFT JOIN DM.dm_td_ext_unionall_market_pack_mapping t3
on t1.PACK_CODE = t3.PACK_CODE and t1.DATA_SOURCE = t3.DATASOURCE
and (t1.YYYYMM-100) >=t3.starttime and (t1.YYYYMM-100) <=t3.endtime
and (t1.YYYYMM-100) >= t3.Value_brand_Ratio_start and (t1.YYYYMM-100) <=t3.Value_brand_Ratio_end
and (t1.YYYYMM-100) >= t3.Unit_brand_start and (t1.YYYYMM-100) <=t3.Unit_brand_end
and (t1.YYYYMM-100) >= t3.CountingUnit_brand_start and (t1.YYYYMM-100) <=t3.CountingUnit_brand_end
and (t1.YYYYMM-100) >= t3.PDOT_brand_start and (t1.YYYYMM-100) <=t3.PDOT_brand_end
and t2.market = t3.MARKET
where t1.DATA_SOURCE = 'DTP(Quarterly)' and t1.DTP_FLAG=1
-- and YYYYMM >=202501
-- COMMAND ----------
-- 所有的pack_code
create or replace temporary view all_egfr_pack
as
select MARKET,PACK_CODE from DM.DM_TD_EXT_RETAIL_MARKET_PACK_MAPPING
where MARKET = 'EGFR TKI Market'
group by MARKET,PACK_CODE
-- COMMAND ----------
-- 4brand pack_code
create or replace temporary view 4brand
as
select
MARKET,
PACK_CODE
from
all_egfr_pack
where
pack_code IN
(
SELECT
PACK_CODE
FROM
dm.dm_td_ext_retail_DTP_pack_property
WHERE
PROD_DESC_C in ('阿美乐','泰瑞沙','凯美纳','艾弗沙')
group BY
PACK_CODE
)
-- COMMAND ----------
create or replace temporary view otherbrand
as
select
PACK_CODE
from
all_egfr_pack
where
PACK_CODE not in ( select PACK_CODE from `4brand` group by PACK_CODE)
group by
PACK_CODE
-- COMMAND ----------
-- 这里是 直接 cu
create or replace temporary view dirct_cu
as
select
yyyymm,
replace(audit_key,'DTP(Quarterly)','') audit_cod,
cast(pdot_counting_unit as decimal ) pdot_counting_unit,
cast(pdot_counting_unit_ly as decimal ) pdot_counting_unit_ly
from dm.dm_tf_external_retail_special a
where yyyymm >=202401
-- COMMAND ----------
-- 这里是获取 4brand 的度量值部分
create or replace temporary view fact_sales_dtp_4brand
as
select
YYYYMM,
AUDIT_CODE AS AUDIT_COD
,sum(PDOT) CONUTING_UNIT
,sum(PDOT_LY) CONUTING_UNIT_LY
,MAX(t2.MARKET) as MARKET
from RETAIL_DTP t1
inner join 4brand t2 on t1.PACK_CODE = t2.PACK_CODE
where t1.DATA_SOURCE = 'DTP(Quarterly)'
and t1.MARKET = 'EGFR TKI Market'
and DTP_FLAG = 1
and YYYYMM >=202401
group by
YYYYMM,AUDIT_COD
-- COMMAND ----------
-- 这里是获取 other 的维度值部分
create or replace temporary view fact_sales_dtp_other
as
select
YYYYMM
,"DTPOTHERS" PACK_COD
,"DTPOTHERS" CORP_COD
,AUDIT_CODE AS AUDIT_COD
,'' as PLATFORM_TYPE
,'' as STORE_NAME
,'' as STORE_TYPE
,'' as REGION_TYPE
,1 as PACK_FLAG
,2 as PROD_FLAG
,1 as DTP_FLAG
,sum(SALES_UNIT_CAL) SALES_UNIT_CAL
,sum(SALES_UNIT_CAL_LY) SALES_UNIT_CAL_LY
,sum(SALES_VALUE_CAL) SALES_VALUE_CAL
,sum(SALES_VALUE_CAL_LY) SALES_VALUE_CAL_LY
,sum(CONUTING_UNIT) CONUTING_UNIT
,sum(CONUTING_UNIT_LY) CONUTING_UNIT_LY
,'DTP(Quarterly)' as DATA_SOURCE
,'' as inst_code
,'' as cmps_flag
,'' as dept_name
,'' as prescription
,'' as prescription_ly
,'' as new_code
,'' as area
,'' as h_level
,'' as reimburse
,'' as reimburse_type
,'' as prescription_source
from DM.DM_TF_EXT_UNIONALL_SALES
where DTP_FLAG = 1
and PACK_CODE in (select PACK_CODe from otherbrand) --其他部分
and DATA_SOURCE = 'DTP(Quarterly)'
and YYYYMM >=202401
group by
YYYYMM,AUDIT_CODE
-- COMMAND ----------
-- create or replace table dm.DM_TF_EXTERANL_SALES_MERGED_DATA_DTP_OTHERS_bymonth
insert overwrite table dm.DM_TF_EXTERANL_SALES_MERGED_DATA_DTP_OTHERS_bymonth
select
t1.MARKET
,'OTHERS' as KEY_COMPETITOR
,'Others' as CLASS
,T0.YYYYMM
,t0.PACK_COD
,t0.CORP_COD
,t0.AUDIT_COD
,t0.PLATFORM_TYPE
,t0.STORE_NAME
,t0.STORE_TYPE
,t0.REGION_TYPE
,t0.PACK_FLAG
,t0.PROD_FLAG
,t0.DTP_FLAG
,t0.SALES_UNIT_CAL
,t0.SALES_UNIT_CAL_LY
,t0.SALES_VALUE_CAL
,t0.SALES_VALUE_CAL_LY
,t1.CONUTING_UNIT
,t1.CONUTING_UNIT_LY
,t1.CONUTING_UNIT as pdot
,t1.CONUTING_UNIT_LY as pdot_ly
,t0.DATA_SOURCE
,t0.inst_code
,t0.cmps_flag
,t0.dept_name
,t0.prescription
,t0.prescription_ly
,t0.new_code
,t0.area
,t0.h_level
,t0.reimburse
,t0.reimburse_type
,t0.prescription_source
from fact_sales_dtp_other t0
left join (
select
t1.yyyymm
,t1.audit_cod
,t1.pdot_counting_unit - t2.conuting_unit as CONUTING_UNIT
,t1.pdot_counting_unit_ly - t2.conuting_unit_ly as CONUTING_UNIT_LY
,t2.market
from dirct_cu t1
left join fact_sales_dtp_4brand t2 on t1.yyyymm = t2.yyyymm and t1.audit_cod = t2.audit_cod
) t1
on t0.audit_cod = t1.audit_cod and t0.yyyymm = t1.yyyymm
-- select * from fact_sales_dtp_4brand

1154
ORG/DM_TD_EXT_AIA_ORG.sql Normal file
View File

File diff suppressed because it is too large Load Diff

235
ORG/DM_TD_EXT_CHC_ORG.sql Normal file
View File

@@ -0,0 +1,235 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_CHC_ORG (
-- REGION_CODE STRING,
-- REGION_NAME_EN STRING,
-- RSM_NAME STRING,
-- RC_CODE STRING,
-- RC_NAME_EN STRING,
-- RC_NAME STRING,
-- REGIONCENTER STRING,
-- AUDIT_COD STRING,
-- MARKET STRING,
-- BU_CATEGORY STRING,
-- BU_NAME_EN STRING,
-- SUB_BU_NAME STRING,
-- SUB_BU_CODE_AUTH STRING,
-- BU_CODE_AUTH STRING,
-- DATA_SOURCE STRING,
-- REGION_RATIO FLOAT,
-- TEAM STRING,
-- MR_TRTY_CODE STRING,
-- NSD_KCODE STRING,
-- NSD_NAME STRING,
-- RSD_KCODE STRING,
-- RSD_NAME STRING,
-- RSM_KCODE STRING,
-- DSM_KCODE STRING,
-- DSM_NAME STRING,
-- MR_KCODE STRING,
-- MR_NAME STRING)
-- USING delta
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chc_org';
-- COMMAND ----------
-- MAGIC %run ../../../Common/config
-- COMMAND ----------
-- MAGIC %python
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dim_product_wide`').createOrReplaceTempView('cdw_dwd_dim_product_wide')
-- COMMAND ----------
-- 20260130 由于DM.dm_td_org数据需要转换，新增这个CELL。
create or replace temporary view dm_td_org_temp
as
SELECT org_key,mr_trty_tag,district_code,rsm_name,rc_name,p2.region_name,p2.rsd_kcode,p2.rsd_name
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_code = 'PC12' then 'PC09' else p2.bu_code end bu_code
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_report = 'EC' THEN 'Retail' ELSE p2.bu_report end bu_report
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_code = 'NA27' then 'NA06' else p2.sub_bu_code end sub_bu_code
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_name = 'EC' then 'Retail' else p2.sub_bu_name end sub_bu_name
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_name_en = 'EC' then 'Retail' else p2.sub_bu_name_en end sub_bu_name_en
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name = 'EC' then 'Retail' else p2.bu_name end bu_name
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name_en = 'EC' then 'Retail' else p2.bu_name_en end bu_name_en
,CASE WHEN p2.yyyymm >= '202601' and p2.nsd_kcode = 'ND_EC0001_V' then 'KSGX559' else p2.nsd_kcode end nsd_kcode
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name = 'EC'and p2.nsd_name = 'Vacant' then '李白杨' else p2.nsd_name end nsd_name
,CASE WHEN p2.yyyymm >= '202601' and p2.nsd_trty_code = 'ND_EC0001' then 'ND_RT0001' else p2.nsd_trty_code end nsd_trty_code
,CASE WHEN p2.yyyymm >= '202601' and p2.bsd_kcode = 'VP_EC0001_V' then 'VP_RT0001_V' else p2.bsd_kcode end bsd_kcode
,CASE WHEN p2.yyyymm >= '202601' and p2.bsd_trty_code = 'VP_EC0001' then 'VP_RT0001' else p2.bsd_trty_code end bsd_trty_code
,mr_kcode,dsm_trty_code,rsm_trty_code,dsm_kcode,rsd_trty_code
,district_name,dsm_name,mr_name,yyyymm,mr_trty_code,REGION_CODE,rc_code,bu_category,rsm_kcode
FROM DM.dm_td_org P2
-- COMMAND ----------
----------------------------------------------------CHC------------------------------------------------------------------------------
-----------只有chc的人能看chc
insert overwrite dm.DM_TD_EXT_CHC_ORG
(REGION_CODE,region_name_en,rc_code,rc_name_en,RegionCenter,AUDIT_COD,MARKET,bu_category,bu_name_en,sub_bu_name,sub_bu_code_auth,bu_code_auth,DATA_SOURCE,REGION_RATIO,RSM_Name,RC_Name,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
)
SELECT DISTINCT T1.REGION_CODE,T1.region_name,T1.rc_code,T1.rc_name,T4.REGION_CENTER RegionCenter,T4.AUDIT_COD,T3.MARKET,T1.bu_category,T1.bu_name_en,T1.sub_bu_name,T1.sub_bu_code sub_bu_code_auth,concat(T1.bu_code,'CHC(Quarterly)') bu_code_auth,T4.DATA_SOURCE,1 REGION_RATIO,t1.RSM_Name,t1.rsd_name,
'' as TEAM,
'' as MR_TRTY_CODE,
'' as NSD_KCODE,
'' as NSD_NAME,
'' as RSD_KCODE,
'' as RSD_NAME,
'' as RSM_KCODE,
'' as DSM_KCODE,
'' as DSM_NAME,
'' as MR_KCODE,
'' as MR_NAME
from dm_td_org_temp t1
left join(
select distinct
t1.yyyymm,
t1.trty_code,
t2.city_name,
t4.ims_prod_cod,
t4.ims_pack_cod
from dm.dm_td_sd_territory_indication t1
left join dwd.dwd_td_institution t2 on t1.inst_code=t2.inst_code
left join cdw_dwd_dim_product_wide t3 on t3.prd_family_cd=t1.family_code
left join dwd.dwd_td_imscode_azcode t4 on t3.prd_brand_cd=t4.brandcode
where t1.bu not in ('CUBE','KA')
and t1.key_hp<>3 --chenwu 20250904 排除双考影响
)t2 on t2.trty_code= t1.mr_trty_code and t2.yyyymm=t1.yyyymm
left join dws.dws_ims_td_market t3 on t2.ims_pack_cod=t3.pack_cod
left join DM.dm_td_external_geo_temp T4 on t2.city_name=concat(t4.city_c,'市') AND T4.DATA_SOURCE = 'CHC(Quarterly)'
--使用和performance cvh 的org架构
where t1.yyyymm in(
select key_value from dm.dm_td_date_config where key_code='pfmc_cvh')
and t3.market is not null and t4.audit_cod is not null and t1.bu_name_en = 'CHC'
-- COMMAND ----------
-- DBTITLE 1,add
--修改时间：20240808
--修改人：FanXuJia
--修改背景：
--事实表与维度表来自于不同手工表，目前存在一部分audit_cod在事实表有销量，但是在维度表没有这个audit_cod，导致销量缺失。
--因此需要从事实表补全这部分audit_cod
with sales as (
select distinct PACK_COD,AUDIT_COD,DATA_SOURCE from dm.dm_tf_external_sales
union
select distinct PACK_COD,AUDIT_COD,DATA_SOURCE from dm.dm_tf_external_sales_thc
)
,market_pack as (
select distinct PACK_COD,MARKET,DATA_SOURCE from dm.dm_td_external_market_pack_mapping
)
,org as (
select distinct AUDIT_COD,MARKET,DATA_SOURCE from dm.DM_TD_EXT_CHC_ORG
)
,tmp_org as (
select distinct a.AUDIT_COD,b.MARKET,a.DATA_SOURCE
from sales a
left join market_pack b on a.PACK_COD = b.PACK_COD and a.DATA_SOURCE = b.DATA_SOURCE
left join org c on a.DATA_SOURCE = c.DATA_SOURCE and a.AUDIT_COD = c.AUDIT_COD and b.MARKET = c.MARKET
where C.AUDIT_COD IS NULL
)
insert into table dm.DM_TD_EXT_CHC_ORG
(
REGION_CODE,
region_name_en,
RSM_Name,
rc_code,
rc_name_en,
RC_Name,
RegionCenter,
AUDIT_COD,
MARKET,
bu_category,
bu_name_en,
sub_bu_name,
sub_bu_code_auth,
bu_code_auth,
DATA_SOURCE,
REGION_RATIO,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
)
select
'ROC' as REGION_CODE,
'ROC' as region_name_en,
'Vacant' as RSM_Name,
'ROC' as rc_code,
'ROC' as rc_name_en,
'Vacant' as RC_Name,
null as RegionCenter,
AUDIT_COD,
MARKET,
null as bu_category,
null as bu_name_en,
null as sub_bu_name,
null as sub_bu_code_auth,
DATA_SOURCE as bu_code_auth,
DATA_SOURCE,
1 as REGION_RATIO,
'ROC' as TEAM,
'Vacant' as MR_TRTY_CODE,
'Vacant' as NSD_KCODE,
'Vacant' as NSD_NAME,
'Vacant' as RSD_KCODE,
'Vacant' as RSD_NAME,
'Vacant' as RSM_KCODE,
'Vacant' as DSM_KCODE,
'Vacant' as DSM_NAME,
'Vacant' as MR_KCODE,
'Vacant' as MR_NAME
from tmp_org
where DATA_SOURCE = 'CHC(Quarterly)'
-- COMMAND ----------
-- DBTITLE 1,hidden_market
-----------------------------------------------------------
--修改时间：20241118
--修改人：Fanxujia
--修改内容：
--使用手工文件排除掉每个渠道的不需要显示的市场
-----------------------------------------------------------
with hidden_market as (
select distinct
case when upper(data_source) = 'CHPA' then 'IQVIA-CHPA(Monthly)'
when upper(data_source) = '科室市场' then 'XH Data(Quarterly)'
when upper(data_source) = 'EC' then 'EC(Monthly)'
when upper(data_source) = 'RETAIL' then 'Retail(Quarterly)'
when upper(data_source) = 'CHC' then 'CHC(Quarterly)'
when upper(data_source) = 'AIA' then 'AIA(Monthly)'
when upper(data_source) = 'THC' then 'THC(Quarterly)'
when upper(data_source) = 'COUNTY' then 'IQVIA-COUNTY(Quarterly)'
else data_source
end as data_source,
upper(hidden_market) as hidden_market
from dwd.dwd_gnd_ims_hidden_market
)
MERGE into dm.DM_TD_EXT_CHC_ORG as t1
USING hidden_market t2
on t1.data_source = t2.data_source
and upper(t1.MARKET) = upper(t2.hidden_market)
WHEN MATCHED THEN
DELETE

233
ORG/DM_TD_EXT_CHPA_ORG.sql Normal file
View File

@@ -0,0 +1,233 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_CHPA_ORG (
-- REGION_CODE STRING,
-- REGION_NAME_EN STRING,
-- RSM_NAME STRING,
-- RC_CODE STRING,
-- RC_NAME_EN STRING,
-- RC_NAME STRING,
-- REGIONCENTER STRING,
-- AUDIT_COD STRING,
-- MARKET STRING,
-- BU_CATEGORY STRING,
-- BU_NAME_EN STRING,
-- SUB_BU_NAME STRING,
-- SUB_BU_CODE_AUTH STRING,
-- BU_CODE_AUTH STRING,
-- DATA_SOURCE STRING,
-- REGION_RATIO FLOAT,
-- TEAM STRING,
-- MR_TRTY_CODE STRING,
-- NSD_KCODE STRING,
-- NSD_NAME STRING,
-- RSD_KCODE STRING,
-- RSD_NAME STRING,
-- RSM_KCODE STRING,
-- DSM_KCODE STRING,
-- DSM_NAME STRING,
-- MR_KCODE STRING,
-- MR_NAME STRING)
-- USING delta
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_chpa_org';
-- COMMAND ----------
----------------------------------------------------chpa------------------------------------------------------------------------------
------所有BU能看CHPA
insert overwrite DM.DM_TD_EXT_CHPA_ORG (
REGION_CODE,
region_name_en,
rc_code,
rc_name_en,
RegionCenter,
AUDIT_COD,
MARKET,
bu_category,
bu_name_en,
sub_bu_name,
sub_bu_code_auth,
bu_code_auth,
DATA_SOURCE,
REGION_RATIO,
RSM_Name,
RC_Name,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
)
SELECT DISTINCT
CASE WHEN bu_category IN ('All Channel','EC') then 'ROC'
when REGION_CODE IN ('Other Low Tiers', 'Non AZ Region') THEN 'ROC'
ELSE REGION_CODE
END REGION_CODE,
CASE WHEN bu_category IN ('All Channel','EC') then 'ROC'
when region_name IN ('Other Low Tiers', 'Non AZ Region') THEN 'ROC'
ELSE region_name
END region_name,
CASE WHEN bu_category IN ('All Channel','EC') then 'ROC'
when rc_code IN ('Other Low Tiers', 'Non AZ Region') THEN 'ROC'
ELSE rc_code
END rc_code,
CASE WHEN bu_category IN ('All Channel','EC') then 'ROC'
when rc_name IN ('Other Low Tiers', 'Non AZ Region') THEN 'ROC'
ELSE rc_name
END rc_name,
RegionCenter,
AUDIT_COD,
MARKET,
bu_category,
bu_name_en,
sub_bu_name,
sub_bu_code_auth,
concat(bu_code_auth, 'IQVIA-CHPA(Monthly)') bu_code_auth,
'IQVIA-CHPA(Monthly)' DATA_SOURCE,
1 REGION_RATIO,
rsm_name,
rsd_name,
'' as TEAM,
'' as MR_TRTY_CODE,
'' as NSD_KCODE,
'' as NSD_NAME,
'' as RSD_KCODE,
'' as RSD_NAME,
'' as RSM_KCODE,
'' as DSM_KCODE,
'' as DSM_NAME,
'' as MR_KCODE,
'' as MR_NAME
FROM
DM.DM_ims_TD_ORG
WHERE
MARKET <> 'IMS ALL Market'
and AUDIT_COD is not null
and ORG_KEY is not null
--AND bu_category NOT IN ('All Channel','EC')
-- COMMAND ----------
-- DBTITLE 1,add
--修改时间：20240808
--修改人：FanXuJia
--修改背景：
--事实表与维度表来自于不同手工表，目前存在一部分audit_cod在事实表有销量，但是在维度表没有这个audit_cod，导致销量缺失。
--因此需要从事实表补全这部分audit_cod
with sales as (
select distinct PACK_COD,AUDIT_COD,DATA_SOURCE from dm.dm_tf_external_sales
union
select distinct PACK_COD,AUDIT_COD,DATA_SOURCE from dm.dm_tf_external_sales_thc
)
,market_pack as (
select distinct PACK_COD,MARKET,DATA_SOURCE from dm.dm_td_external_market_pack_mapping
)
,org as (
select distinct AUDIT_COD,MARKET,DATA_SOURCE from dm.DM_TD_EXT_CHPA_ORG
)
,tmp_org as (
select distinct a.AUDIT_COD,b.MARKET,a.DATA_SOURCE
from sales a
left join market_pack b on a.PACK_COD = b.PACK_COD and a.DATA_SOURCE = b.DATA_SOURCE
left join org c on a.DATA_SOURCE = c.DATA_SOURCE and a.AUDIT_COD = c.AUDIT_COD and b.MARKET = c.MARKET
where C.AUDIT_COD IS NULL
)
insert into table dm.DM_TD_EXT_CHPA_ORG
(
REGION_CODE,
region_name_en,
RSM_Name,
rc_code,
rc_name_en,
RC_Name,
RegionCenter,
AUDIT_COD,
MARKET,
bu_category,
bu_name_en,
sub_bu_name,
sub_bu_code_auth,
bu_code_auth,
DATA_SOURCE,
REGION_RATIO,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
)
select
'ROC' as REGION_CODE,
'ROC' as region_name_en,
'Vacant' as RSM_Name,
'ROC' as rc_code,
'ROC' as rc_name_en,
'Vacant' as RC_Name,
null as RegionCenter,
AUDIT_COD,
MARKET,
null as bu_category,
null as bu_name_en,
null as sub_bu_name,
null as sub_bu_code_auth,
DATA_SOURCE as bu_code_auth,
DATA_SOURCE,
1 as REGION_RATIO,
'ROC' as TEAM,
'Vacant' as MR_TRTY_CODE,
'Vacant' as NSD_KCODE,
'Vacant' as NSD_NAME,
'Vacant' as RSD_KCODE,
'Vacant' as RSD_NAME,
'Vacant' as RSM_KCODE,
'Vacant' as DSM_KCODE,
'Vacant' as DSM_NAME,
'Vacant' as MR_KCODE,
'Vacant' as MR_NAME
from tmp_org
where DATA_SOURCE = 'IQVIA-CHPA(Monthly)'
-- COMMAND ----------
-- DBTITLE 1,hidden_market
-----------------------------------------------------------
--修改时间：20241118
--修改人：Fanxujia
--修改内容：
--使用手工文件排除掉每个渠道的不需要显示的市场
-----------------------------------------------------------
with hidden_market as (
select distinct
case when upper(data_source) = 'CHPA' then 'IQVIA-CHPA(Monthly)'
when upper(data_source) = '科室市场' then 'XH Data(Quarterly)'
when upper(data_source) = 'EC' then 'EC(Monthly)'
when upper(data_source) = 'RETAIL' then 'Retail(Quarterly)'
when upper(data_source) = 'CHC' then 'CHC(Quarterly)'
when upper(data_source) = 'AIA' then 'AIA(Monthly)'
when upper(data_source) = 'THC' then 'THC(Quarterly)'
when upper(data_source) = 'COUNTY' then 'IQVIA-COUNTY(Quarterly)'
else data_source
end as data_source,
upper(hidden_market) as hidden_market
from dwd.dwd_gnd_ims_hidden_market
)
MERGE into dm.DM_TD_EXT_CHPA_ORG as t1
USING hidden_market t2
on t1.data_source = t2.data_source
and upper(t1.MARKET) = upper(t2.hidden_market)
WHEN MATCHED THEN
DELETE

220
ORG/DM_TD_EXT_COUNTY_ORG.sql Normal file
View File

@@ -0,0 +1,220 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_COUNTY_ORG (
-- REGION_CODE STRING,
-- REGION_NAME_EN STRING,
-- RSM_NAME STRING,
-- RC_CODE STRING,
-- RC_NAME_EN STRING,
-- RC_NAME STRING,
-- REGIONCENTER STRING,
-- AUDIT_COD STRING,
-- MARKET STRING,
-- BU_CATEGORY STRING,
-- BU_NAME_EN STRING,
-- SUB_BU_NAME STRING,
-- SUB_BU_CODE_AUTH STRING,
-- BU_CODE_AUTH STRING,
-- DATA_SOURCE STRING,
-- REGION_RATIO FLOAT,
-- TEAM STRING,
-- MR_TRTY_CODE STRING,
-- NSD_KCODE STRING,
-- NSD_NAME STRING,
-- RSD_KCODE STRING,
-- RSD_NAME STRING,
-- RSM_KCODE STRING,
-- DSM_KCODE STRING,
-- DSM_NAME STRING,
-- MR_KCODE STRING,
-- MR_NAME STRING)
-- USING delta
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_county_org';
-- COMMAND ----------
-- 20260130 由于DM.dm_td_org数据需要转换，新增这个CELL。
create or replace temporary view dm_td_org_temp
as
SELECT org_key,mr_trty_tag,district_code,rsm_name,rc_name,p2.region_name,p2.rsd_kcode,p2.rsd_name
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_code = 'PC12' then 'PC09' else p2.bu_code end bu_code
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_report = 'EC' THEN 'Retail' ELSE p2.bu_report end bu_report
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_code = 'NA27' then 'NA06' else p2.sub_bu_code end sub_bu_code
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_name = 'EC' then 'Retail' else p2.sub_bu_name end sub_bu_name
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_name_en = 'EC' then 'Retail' else p2.sub_bu_name_en end sub_bu_name_en
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name = 'EC' then 'Retail' else p2.bu_name end bu_name
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name_en = 'EC' then 'Retail' else p2.bu_name_en end bu_name_en
,CASE WHEN p2.yyyymm >= '202601' and p2.nsd_kcode = 'ND_EC0001_V' then 'KSGX559' else p2.nsd_kcode end nsd_kcode
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name = 'EC'and p2.nsd_name = 'Vacant' then '李白杨' else p2.nsd_name end nsd_name
,CASE WHEN p2.yyyymm >= '202601' and p2.nsd_trty_code = 'ND_EC0001' then 'ND_RT0001' else p2.nsd_trty_code end nsd_trty_code
,CASE WHEN p2.yyyymm >= '202601' and p2.bsd_kcode = 'VP_EC0001_V' then 'VP_RT0001_V' else p2.bsd_kcode end bsd_kcode
,CASE WHEN p2.yyyymm >= '202601' and p2.bsd_trty_code = 'VP_EC0001' then 'VP_RT0001' else p2.bsd_trty_code end bsd_trty_code
,mr_kcode,dsm_trty_code,rsm_trty_code,dsm_kcode,rsd_trty_code
,district_name,dsm_name,mr_name,yyyymm,mr_trty_code,REGION_CODE,rc_code,bu_category,rsm_kcode
FROM DM.dm_td_org P2
-- COMMAND ----------
----------------------------------------------------county------------------------------------------------------------------------------
-----------------------除了Eagle、CHC、Retail、EC的人都能看
insert overwrite table DM.DM_TD_EXT_COUNTY_ORG
(REGION_CODE,region_name_en,rc_code,rc_name_en,RegionCenter,AUDIT_COD,MARKET,bu_category,bu_name_en,sub_bu_name,sub_bu_code_auth,bu_code_auth,DATA_SOURCE,REGION_RATIO,RSM_Name,RC_Name,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
)
SELECT DISTINCT '' REGION_CODE,'' region_name_en,'' rc_code,'' rc_name_en,'' RegionCenter,A.AUDIT_COD,B.MARKET,C.bu_category,C.bu_name_en,C.sub_bu_name,C.sub_bu_code sub_bu_code_auth,concat(c.bu_code,'IQVIA-COUNTY(Quarterly)') bu_code_auth,A.DATA_SOURCE,1 REGION_RATIO,'','',
'' as TEAM,
'' as MR_TRTY_CODE,
'' as NSD_KCODE,
'' as NSD_NAME,
'' as RSD_KCODE,
'' as RSD_NAME,
'' as RSM_KCODE,
'' as DSM_KCODE,
'' as DSM_NAME,
'' as MR_KCODE,
'' as MR_NAME
FROM DM.dm_tf_external_sales A
INNER JOIN DM.dm_td_external_market_pack_mapping B ON A.PACK_COD = B.PACK_COD
AND A.DATA_SOURCE = B.DATA_SOURCE
INNER JOIN
(
SELECT DISTINCT '' bu_code,'' sub_bu_code,'' sub_bu_name,'' bu_category,'' bu_name_en -----------目前权限控制到BU，因此需要BU有值。
FROM dm_td_org_temp C
WHERE C.yyyymm = (select key_value from dm.dm_td_date_config where key_code='pfmc_cvh')
AND C.BU_NAME_EN NOT IN ('Eagle','CHC','EC','Retail')
) C
WHERE A.DATA_SOURCE = 'IQVIA-COUNTY(Quarterly)'
-- COMMAND ----------
-- DBTITLE 1,add
--修改时间：20240808
--修改人：FanXuJia
--修改背景：
--事实表与维度表来自于不同手工表，目前存在一部分audit_cod在事实表有销量，但是在维度表没有这个audit_cod，导致销量缺失。
--因此需要从事实表补全这部分audit_cod
with sales as (
select distinct PACK_COD,AUDIT_COD,DATA_SOURCE from dm.dm_tf_external_sales
union
select distinct PACK_COD,AUDIT_COD,DATA_SOURCE from dm.dm_tf_external_sales_thc
)
,market_pack as (
select distinct PACK_COD,MARKET,DATA_SOURCE from dm.dm_td_external_market_pack_mapping
)
,org as (
select distinct AUDIT_COD,MARKET,DATA_SOURCE from dm.DM_TD_EXT_COUNTY_ORG
)
,tmp_org as (
select distinct a.AUDIT_COD,b.MARKET,a.DATA_SOURCE
from sales a
left join market_pack b on a.PACK_COD = b.PACK_COD and a.DATA_SOURCE = b.DATA_SOURCE
left join org c on a.DATA_SOURCE = c.DATA_SOURCE and a.AUDIT_COD = c.AUDIT_COD and b.MARKET = c.MARKET
where C.AUDIT_COD IS NULL
)
insert into table dm.DM_TD_EXT_COUNTY_ORG
(
REGION_CODE,
region_name_en,
RSM_Name,
rc_code,
rc_name_en,
RC_Name,
RegionCenter,
AUDIT_COD,
MARKET,
bu_category,
bu_name_en,
sub_bu_name,
sub_bu_code_auth,
bu_code_auth,
DATA_SOURCE,
REGION_RATIO,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
)
select
'ROC' as REGION_CODE,
'ROC' as region_name_en,
'Vacant' as RSM_Name,
'ROC' as rc_code,
'ROC' as rc_name_en,
'Vacant' as RC_Name,
null as RegionCenter,
AUDIT_COD,
MARKET,
null as bu_category,
null as bu_name_en,
null as sub_bu_name,
null as sub_bu_code_auth,
DATA_SOURCE as bu_code_auth,
DATA_SOURCE,
1 as REGION_RATIO,
'ROC' as TEAM,
'Vacant' as MR_TRTY_CODE,
'Vacant' as NSD_KCODE,
'Vacant' as NSD_NAME,
'Vacant' as RSD_KCODE,
'Vacant' as RSD_NAME,
'Vacant' as RSM_KCODE,
'Vacant' as DSM_KCODE,
'Vacant' as DSM_NAME,
'Vacant' as MR_KCODE,
'Vacant' as MR_NAME
from tmp_org
where DATA_SOURCE = 'IQVIA-COUNTY(Quarterly)'
-- COMMAND ----------
-- DBTITLE 1,DM_TD_EXT_CHPA_ORG
-----------------------------------------------------------
--修改时间：20241118
--修改人：Fanxujia
--修改内容：
--使用手工文件排除掉每个渠道的不需要显示的市场
-----------------------------------------------------------
with hidden_market as (
select distinct
case when upper(data_source) = 'CHPA' then 'IQVIA-CHPA(Monthly)'
when upper(data_source) = '科室市场' then 'XH Data(Quarterly)'
when upper(data_source) = 'EC' then 'EC(Monthly)'
when upper(data_source) = 'RETAIL' then 'Retail(Quarterly)'
when upper(data_source) = 'CHC' then 'CHC(Quarterly)'
when upper(data_source) = 'AIA' then 'AIA(Monthly)'
when upper(data_source) = 'THC' then 'THC(Quarterly)'
when upper(data_source) = 'COUNTY' then 'IQVIA-COUNTY(Quarterly)'
else data_source
end as data_source,
upper(hidden_market) as hidden_market
from dwd.dwd_gnd_ims_hidden_market
union all
select
'AIA(Monthly)' as data_source,
'AIA ALL Market'
)
MERGE into dm.DM_TD_EXT_COUNTY_ORG as t1
USING hidden_market t2
on t1.data_source = t2.data_source
and upper(t1.MARKET) = upper(t2.hidden_market)
WHEN MATCHED THEN
DELETE

219
ORG/DM_TD_EXT_EC_ORG.sql Normal file
View File

@@ -0,0 +1,219 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_EC_ORG (
-- REGION_CODE STRING,
-- REGION_NAME_EN STRING,
-- RSM_NAME STRING,
-- RC_CODE STRING,
-- RC_NAME_EN STRING,
-- RC_NAME STRING,
-- REGIONCENTER STRING,
-- AUDIT_COD STRING,
-- MARKET STRING,
-- BU_CATEGORY STRING,
-- BU_NAME_EN STRING,
-- SUB_BU_NAME STRING,
-- SUB_BU_CODE_AUTH STRING,
-- BU_CODE_AUTH STRING,
-- DATA_SOURCE STRING,
-- REGION_RATIO FLOAT,
-- TEAM STRING,
-- MR_TRTY_CODE STRING,
-- NSD_KCODE STRING,
-- NSD_NAME STRING,
-- RSD_KCODE STRING,
-- RSD_NAME STRING,
-- RSM_KCODE STRING,
-- DSM_KCODE STRING,
-- DSM_NAME STRING,
-- MR_KCODE STRING,
-- MR_NAME STRING)
-- USING delta
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_ec_org';
-- COMMAND ----------
-- 20260130 由于DM.dm_td_org数据需要转换，新增这个CELL。
create or replace temporary view dm_td_org_temp
as
SELECT org_key,mr_trty_tag,district_code,rsm_name,rc_name,p2.region_name,p2.rsd_kcode,p2.rsd_name
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_code = 'PC12' then 'PC09' else p2.bu_code end bu_code
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_report = 'EC' THEN 'Retail' ELSE p2.bu_report end bu_report
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_code = 'NA27' then 'NA06' else p2.sub_bu_code end sub_bu_code
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_name = 'EC' then 'Retail' else p2.sub_bu_name end sub_bu_name
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_name_en = 'EC' then 'Retail' else p2.sub_bu_name_en end sub_bu_name_en
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name = 'EC' then 'Retail' else p2.bu_name end bu_name
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name_en = 'EC' then 'Retail' else p2.bu_name_en end bu_name_en
,CASE WHEN p2.yyyymm >= '202601' and p2.nsd_kcode = 'ND_EC0001_V' then 'KSGX559' else p2.nsd_kcode end nsd_kcode
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name = 'EC'and p2.nsd_name = 'Vacant' then '李白杨' else p2.nsd_name end nsd_name
,CASE WHEN p2.yyyymm >= '202601' and p2.nsd_trty_code = 'ND_EC0001' then 'ND_RT0001' else p2.nsd_trty_code end nsd_trty_code
,CASE WHEN p2.yyyymm >= '202601' and p2.bsd_kcode = 'VP_EC0001_V' then 'VP_RT0001_V' else p2.bsd_kcode end bsd_kcode
,CASE WHEN p2.yyyymm >= '202601' and p2.bsd_trty_code = 'VP_EC0001' then 'VP_RT0001' else p2.bsd_trty_code end bsd_trty_code
,mr_kcode,dsm_trty_code,rsm_trty_code,dsm_kcode,rsd_trty_code
,district_name,dsm_name,mr_name,yyyymm,mr_trty_code,REGION_CODE,rc_code,bu_category,rsm_kcode
FROM DM.dm_td_org P2
-- COMMAND ----------
----------------------------------------------------------EC----------------------------------------------------------
insert overwrite dm.DM_TD_EXT_EC_ORG(
REGION_CODE,region_name_en,rc_code,rc_name_en,RegionCenter,AUDIT_COD,MARKET,bu_category,bu_name_en,sub_bu_name,sub_bu_code_auth,bu_code_auth,DATA_SOURCE,REGION_RATIO,RSM_Name,RC_Name,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
)
SELECT DISTINCT '' REGION_CODE,'' region_name_en,'' rc_code,'' rc_name_en,'' RegionCenter,A.AUDIT_COD,B.MARKET,C.bu_category,C.bu_name_en,C.sub_bu_name,C.sub_bu_code sub_bu_code_auth,concat(c.bu_code,'EC(Monthly)') bu_code_auth,A.DATA_SOURCE,1 REGION_RATIO,'','',
'' as TEAM,
'' as MR_TRTY_CODE,
'' as NSD_KCODE,
'' as NSD_NAME,
'' as RSD_KCODE,
'' as RSD_NAME,
'' as RSM_KCODE,
'' as DSM_KCODE,
'' as DSM_NAME,
'' as MR_KCODE,
'' as MR_NAME
FROM DM.dm_tf_external_sales A
INNER JOIN DM.dm_td_external_market_pack_mapping B ON A.PACK_COD = B.PACK_COD
AND A.DATA_SOURCE = B.DATA_SOURCE
INNER JOIN
(
SELECT DISTINCT '' bu_code,'' sub_bu_code,'' sub_bu_name,'' bu_category,'' bu_name_en
FROM dm_td_org_temp C
WHERE C.yyyymm = (select key_value from dm.dm_td_date_config where key_code='pfmc_cvh')
AND C.BU_NAME_EN = 'Eagle'
) C
WHERE A.DATA_SOURCE = 'EC(Monthly)'
-- COMMAND ----------
-- DBTITLE 1,add
--修改时间：20240808
--修改人：FanXuJia
--修改背景：
--事实表与维度表来自于不同手工表，目前存在一部分audit_cod在事实表有销量，但是在维度表没有这个audit_cod，导致销量缺失。
--因此需要从事实表补全这部分audit_cod
with sales as (
select distinct PACK_COD,AUDIT_COD,DATA_SOURCE from dm.dm_tf_external_sales
union
select distinct PACK_COD,AUDIT_COD,DATA_SOURCE from dm.dm_tf_external_sales_thc
)
,market_pack as (
select distinct PACK_COD,MARKET,DATA_SOURCE from dm.dm_td_external_market_pack_mapping
)
,org as (
select distinct AUDIT_COD,MARKET,DATA_SOURCE from dm.DM_TD_EXT_EC_ORG
)
,tmp_org as (
select distinct a.AUDIT_COD,b.MARKET,a.DATA_SOURCE
from sales a
left join market_pack b on a.PACK_COD = b.PACK_COD and a.DATA_SOURCE = b.DATA_SOURCE
left join org c on a.DATA_SOURCE = c.DATA_SOURCE and a.AUDIT_COD = c.AUDIT_COD and b.MARKET = c.MARKET
where C.AUDIT_COD IS NULL
)
insert into table dm.DM_TD_EXT_EC_ORG
(
REGION_CODE,
region_name_en,
RSM_Name,
rc_code,
rc_name_en,
RC_Name,
RegionCenter,
AUDIT_COD,
MARKET,
bu_category,
bu_name_en,
sub_bu_name,
sub_bu_code_auth,
bu_code_auth,
DATA_SOURCE,
REGION_RATIO,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
)
select
'ROC' as REGION_CODE,
'ROC' as region_name_en,
'Vacant' as RSM_Name,
'ROC' as rc_code,
'ROC' as rc_name_en,
'Vacant' as RC_Name,
null as RegionCenter,
AUDIT_COD,
MARKET,
null as bu_category,
null as bu_name_en,
null as sub_bu_name,
null as sub_bu_code_auth,
DATA_SOURCE as bu_code_auth,
DATA_SOURCE,
1 as REGION_RATIO,
'ROC' as TEAM,
'Vacant' as MR_TRTY_CODE,
'Vacant' as NSD_KCODE,
'Vacant' as NSD_NAME,
'Vacant' as RSD_KCODE,
'Vacant' as RSD_NAME,
'Vacant' as RSM_KCODE,
'Vacant' as DSM_KCODE,
'Vacant' as DSM_NAME,
'Vacant' as MR_KCODE,
'Vacant' as MR_NAME
from tmp_org
where DATA_SOURCE = 'EC(Monthly)'
-- COMMAND ----------
-- DBTITLE 1,hidden_market
-----------------------------------------------------------
--修改时间：20241118
--修改人：Fanxujia
--修改内容：
--使用手工文件排除掉每个渠道的不需要显示的市场
-----------------------------------------------------------
with hidden_market as (
select distinct
case when upper(data_source) = 'CHPA' then 'IQVIA-CHPA(Monthly)'
when upper(data_source) = '科室市场' then 'XH Data(Quarterly)'
when upper(data_source) = 'EC' then 'EC(Monthly)'
when upper(data_source) = 'RETAIL' then 'Retail(Quarterly)'
when upper(data_source) = 'CHC' then 'CHC(Quarterly)'
when upper(data_source) = 'AIA' then 'AIA(Monthly)'
when upper(data_source) = 'THC' then 'THC(Quarterly)'
when upper(data_source) = 'COUNTY' then 'IQVIA-COUNTY(Quarterly)'
else data_source
end as data_source,
upper(hidden_market) as hidden_market
from dwd.dwd_gnd_ims_hidden_market
union all
select
'AIA(Monthly)' as data_source,
'AIA ALL Market'
)
MERGE into dm.DM_TD_EXT_EC_ORG as t1
USING hidden_market t2
on t1.data_source = t2.data_source
and upper(t1.MARKET) = upper(t2.hidden_market)
WHEN MATCHED THEN
DELETE

438
ORG/DM_TD_EXT_RETAIL_ORG.sql Normal file
View File

@@ -0,0 +1,438 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_RETAIL_ORG (
-- REGION_CODE STRING,
-- REGION_NAME_EN STRING,
-- RSM_NAME STRING,
-- RC_CODE STRING,
-- RC_NAME_EN STRING,
-- RC_NAME STRING,
-- REGIONCENTER STRING,
-- AUDIT_COD STRING,
-- MARKET STRING,
-- BU_CATEGORY STRING,
-- BU_NAME_EN STRING,
-- SUB_BU_NAME STRING,
-- SUB_BU_CODE_AUTH STRING,
-- BU_CODE_AUTH STRING,
-- DATA_SOURCE STRING,
-- REGION_RATIO FLOAT,
-- TEAM STRING,
-- MR_TRTY_CODE STRING,
-- NSD_KCODE STRING,
-- NSD_NAME STRING,
-- RSD_KCODE STRING,
-- RSD_NAME STRING,
-- RSM_KCODE STRING,
-- DSM_KCODE STRING,
-- DSM_NAME STRING,
-- MR_KCODE STRING,
-- MR_NAME STRING)
-- USING delta
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_retail_org';
-- COMMAND ----------
-- MAGIC %run ../../../Common/config
-- COMMAND ----------
-- MAGIC %python
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dim_product_wide`').createOrReplaceTempView('cdw_dwd_dim_product_wide')
-- COMMAND ----------
-- 20260130 由于DM.dm_td_org数据需要转换，新增这个CELL。
create or replace temporary view dm_td_org_temp
as
SELECT org_key,mr_trty_tag,district_code,rsm_name,rc_name,p2.region_name,p2.rsd_kcode,p2.rsd_name
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_code = 'PC12' then 'PC09' else p2.bu_code end bu_code
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_report = 'EC' THEN 'Retail' ELSE p2.bu_report end bu_report
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_code = 'NA27' then 'NA06' else p2.sub_bu_code end sub_bu_code
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_name = 'EC' then 'Retail' else p2.sub_bu_name end sub_bu_name
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_name_en = 'EC' then 'Retail' else p2.sub_bu_name_en end sub_bu_name_en
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name = 'EC' then 'Retail' else p2.bu_name end bu_name
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name_en = 'EC' then 'Retail' else p2.bu_name_en end bu_name_en
,CASE WHEN p2.yyyymm >= '202601' and p2.nsd_kcode = 'ND_EC0001_V' then 'KSGX559' else p2.nsd_kcode end nsd_kcode
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name = 'EC'and p2.nsd_name = 'Vacant' then '李白杨' else p2.nsd_name end nsd_name
,CASE WHEN p2.yyyymm >= '202601' and p2.nsd_trty_code = 'ND_EC0001' then 'ND_RT0001' else p2.nsd_trty_code end nsd_trty_code
,CASE WHEN p2.yyyymm >= '202601' and p2.bsd_kcode = 'VP_EC0001_V' then 'VP_RT0001_V' else p2.bsd_kcode end bsd_kcode
,CASE WHEN p2.yyyymm >= '202601' and p2.bsd_trty_code = 'VP_EC0001' then 'VP_RT0001' else p2.bsd_trty_code end bsd_trty_code
,mr_kcode,dsm_trty_code,rsm_trty_code,dsm_kcode,rsd_trty_code
,district_name,dsm_name,mr_name,yyyymm,mr_trty_code,REGION_CODE,rc_code,bu_category,rsm_kcode
FROM DM.dm_td_org P2
WHERE rc_name NOT LIKE '%B2C%'
-- COMMAND ----------
----------------------------------------------------Retail-----------------------------------------------------------------------
insert overwrite DM.DM_TD_EXT_RETAIL_ORG
(REGION_CODE,region_name_en,rc_code,rc_name_en,RegionCenter,AUDIT_COD,MARKET,bu_category,bu_name_en,sub_bu_name,sub_bu_code_auth,bu_code_auth,DATA_SOURCE,REGION_RATIO,RSM_Name,RC_Name,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
)
with
market_ta (
select
MARKET,case when TA = 'ONCO' then 'OBU' else 'Retail' end as subbu
from dm.dm_td_external_market_ta
where DATA_SOURCE = 'Retail(Quarterly)'
group by 1,2
)
,pack_market as (
select distinct a.pack_cod,a.market,b.subbu
from dm.dm_td_external_market_pack_mapping a
left join market_ta b on a.MARKET = b.market
where data_source = 'Retail(Quarterly)'
)
,external_geo as (
select distinct province_c,audit_cod,REGION_CENTER
from dm.dm_td_external_geo_temp
where DATA_SOURCE = 'Retail(Quarterly)'
)
,YM_CVH as (
select key_value from dm.dm_td_date_config
where key_code = 'pfmc_cvh'
)
,td_org as (
select distinct
REGION_CODE,
region_name,
rc_code,
rc_name,
bu_category,
bu_name_en,
sub_bu_name,
sub_bu_code,
bu_code,
RSM_Name,
rsd_name,
mr_trty_code,
rsm_trty_code
from dm_td_org_temp t1
inner join YM_CVH t2
on t1.yyyymm = t2.key_value
)
,territory as (
select distinct
trty_code,
inst_code,
family_code,
bu
from dm.dm_td_sd_territory_indication t1
inner join YM_CVH t2
on t1.yyyymm = t2.key_value
where bu in ('Retail') --20260320 chenwu Retail数据源的架构，去掉OBU，只保留Retail
and t1.key_hp<>3 --chenwu 20250904 排除双考影响
)
,province_market as (
select t1.trty_code,
t2.province_name,
t5.market
from territory t1
left join dwd.dwd_td_institution t2
on t1.inst_code = t2.inst_code
left join cdw_dwd_dim_product_wide t3
on t1.family_code = t3.prd_family_cd
left join dwd.dwd_td_imscode_azcode t4
on t3.prd_brand_cd = t4.brandcode
left join pack_market t5
on t4.ims_pack_cod = t5.pack_cod
where t1.bu=t5.subbu
)
SELECT
DISTINCT t1.REGION_CODE REGION_CODE,
t1.region_name,
t1.rc_code,
t1.rc_name,
t3.REGION_CENTER RegionCenter,
t3.AUDIT_COD,
t2.MARKET,
t1.bu_category,
t1.bu_name_en,
t1.sub_bu_name,
t1.sub_bu_code sub_bu_code_auth,
concat(t1.bu_code, 'Retail(Quarterly)') bu_code_auth,
'Retail(Quarterly)' as DATA_SOURCE,
1 REGION_RATIO,
t1.RSM_Name,
t1.rsd_name,
'' as TEAM,
'' as MR_TRTY_CODE,
'' as NSD_KCODE,
'' as NSD_NAME,
'' as RSD_KCODE,
'' as RSD_NAME,
'' as RSM_KCODE,
'' as DSM_KCODE,
'' as DSM_NAME,
'' as MR_KCODE,
'' as MR_NAME
from td_org t1
inner join province_market t2
on t1.mr_trty_code = t2.trty_code
inner join external_geo t3 on t2.province_name = t3.province_c
where t2.market is not null
and t3.audit_cod is not null
union all
SELECT DISTINCT
'ROC' as REGION_CODE,
'ROC' as region_name_en,
'ROC' as rc_code,
'ROC' as rc_name_en,
'Other Low Tiers' as RegionCenter,
'ROC' AUDIT_COD,
t2.MARKET,
'All Channel' as bu_category,
CASE WHEN T2.subbu = 'OBU' THEN 'OBU' ELSE 'Retail' END as bu_name_en,
CASE WHEN T2.subbu = 'OBU' THEN 'OBU' ELSE 'Retail' END as sub_bu_name,
CASE WHEN T2.subbu = 'OBU' THEN 'OBU' ELSE 'Retail' END as sub_bu_code_auth,
'Retail(Quarterly)' as bu_code_auth,
'Retail(Quarterly)' as DATA_SOURCE,
1 as REGION_RATIO,
'Vacant' as RSM_Name,
'Vacant' as rsd_name,
'' as TEAM,
'' as MR_TRTY_CODE,
'' as NSD_KCODE,
'' as NSD_NAME,
'' as RSD_KCODE,
'' as RSD_NAME,
'' as RSM_KCODE,
'' as DSM_KCODE,
'' as DSM_NAME,
'' as MR_KCODE,
'' as MR_NAME
FROM pack_market t2
--特殊处理部分
union all
SELECT
DISTINCT t1.REGION_CODE REGION_CODE,
t1.region_name,
t1.rc_code,
t1.rc_name,
t3.REGION_CENTER RegionCenter,
t3.AUDIT_COD,
"Atorvastatin Market" as MARKET,
t1.bu_category,
t1.bu_name_en,
t1.sub_bu_name,
t1.sub_bu_code sub_bu_code_auth,
concat(t1.bu_code, 'Retail(Quarterly)') bu_code_auth,
'Retail(Quarterly)' as DATA_SOURCE,
1 REGION_RATIO,
t1.RSM_Name,
t1.rsd_name,
'' as TEAM,
'' as MR_TRTY_CODE,
'' as NSD_KCODE,
'' as NSD_NAME,
'' as RSD_KCODE,
'' as RSD_NAME,
'' as RSM_KCODE,
'' as DSM_KCODE,
'' as DSM_NAME,
'' as MR_KCODE,
'' as MR_NAME
from td_org t1
inner join province_market t2
on t1.mr_trty_code = t2.trty_code
inner join external_geo t3 on t2.province_name = t3.province_c
where t2.market is not null
and t3.audit_cod is not null
and t2.market = 'Rosuvastatin Market'
-- COMMAND ----------
-- DBTITLE 1,DTP
insert into DM.DM_TD_EXT_RETAIL_ORG
(REGION_CODE,region_name_en,rc_code,rc_name_en,RegionCenter,AUDIT_COD,MARKET,bu_category,bu_name_en,sub_bu_name,sub_bu_code_auth,bu_code_auth,DATA_SOURCE,REGION_RATIO,RSM_Name,RC_Name,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
)
WITH
-- ① 市场与子BU映射
market_ta AS (
select distinct MARKET,bu as subbu from dwd.dwd_gnd_dtp_tblmarket
-- WHERE UPPER(TA) <> 'ALL'
),
-- ② 产品包与市场映射
pack_2_market AS (
SELECT DISTINCT
a.PACK_CODE,
a.market,
b.subbu
FROM
dm.dm_td_ext_dtp_market_pack_mapping a
LEFT JOIN market_ta b ON a.MARKET = b.market
),
-- FAMILY与market映射
family_2_markt as (select distinct
t3.prd_family_cd,
-- t4.ims_pack_cod,
t5.market
from
cdw_dwd_dim_product_wide t3
inner JOIN dwd.dwd_td_imscode_azcode t4
ON t3.prd_brand_cd = t4.brandcode
inner join pack_2_market t5
on t4.ims_pack_cod = t5.pack_code
where
t3.prd_family_cd is not null
and t4.ims_pack_cod is not null
),
-- ③ 外部地理信息
external_geo AS (
SELECT DISTINCT
province_c,
audit_cod,
REGION_CENTER
FROM dm.dm_td_external_geo_temp
WHERE DATA_SOURCE = 'DTP(Quarterly)'
),
-- ④ 当前考核月份
YM_CVH AS (
SELECT key_value
FROM dm.dm_td_date_config
WHERE key_code = 'pfmc_cvh'
),
-- ⑤ 组织架构（取当前考核月份快照）
td_org AS (
SELECT DISTINCT
REGION_CODE,
region_name,
rc_code,
rc_name,
bu_category,
bu_name_en,
sub_bu_name,
sub_bu_code,
bu_code,
RSM_Name,
rsd_name,
mr_trty_code,
rsm_trty_code
FROM dm_td_org_temp t1
INNER JOIN YM_CVH t2 ON t1.yyyymm = t2.key_value
),
-- ⑥ 辖区-指征明细（仅保留 OBU/RIN，排除双考）
territory AS (
SELECT DISTINCT
trty_code,
inst_code,
family_code,
bu
FROM dm.dm_td_sd_territory_indication t1
INNER JOIN YM_CVH t2 ON t1.yyyymm = t2.key_value
WHERE t1.bu IN ('OBU', 'RIN') -- 20260320 chenwu DTP数据源只保留 OBU 和 RIN
AND t1.key_hp <> 3 -- 20250904 chenwu 排除双考影响
),
-- ⑦ 辖区-省份-市场关联
province_market AS (
SELECT
t1.trty_code,
t2.province_name,
t3.market
FROM territory t1
LEFT JOIN dwd.dwd_td_institution t2 ON t1.inst_code = t2.inst_code
INNER JOIN family_2_markt t3 ON t1.family_code = t3.prd_family_cd
)
-- ════════════════════════════════════════════
-- 主查询 Part 1：正常地理覆盖行
-- ════════════════════════════════════════════
SELECT DISTINCT
t1.REGION_CODE,
t1.region_name,
t1.rc_code,
t1.rc_name AS rc_name_en,
t3.REGION_CENTER AS RegionCenter,
t3.AUDIT_COD,
t2.MARKET,
t1.bu_category,
t1.bu_name_en,
t1.sub_bu_name,
t1.sub_bu_code AS sub_bu_code_auth,
CONCAT(t1.bu_code, 'DTP(Quarterly)') AS bu_code_auth,
'DTP(Quarterly)' AS DATA_SOURCE,
1 AS REGION_RATIO,
t1.RSM_Name,
t1.rsd_name,
'' AS TEAM,
'' AS MR_TRTY_CODE,
'' AS NSD_KCODE,
'' AS NSD_NAME,
'' AS RSD_KCODE,
'' AS RC_Name,
'' AS RSM_KCODE,
'' AS DSM_KCODE,
'' AS DSM_NAME,
'' AS MR_KCODE,
'' AS MR_NAME
FROM td_org t1
INNER JOIN province_market t2 ON t1.mr_trty_code = t2.trty_code
INNER JOIN external_geo t3 ON t2.province_name = t3.province_c
WHERE t2.market IS NOT NULL
AND t3.audit_cod IS NOT NULL
UNION ALL
-- ════════════════════════════════════════════
-- 主查询 Part 2：ROC 虚拟兜底行
-- ════════════════════════════════════════════
SELECT DISTINCT
'ROC' AS REGION_CODE,
'ROC' AS region_name,
'ROC' AS rc_code,
'ROC' AS rc_name_en,
'Other Low Tiers' AS RegionCenter,
'ROC' AS AUDIT_COD,
t2.MARKET,
'All Channel' AS bu_category,
nvl(subbu,'ONCO') AS bu_name_en,
nvl(subbu,'ONCO') AS sub_bu_name,
nvl(subbu,'ONCO') AS sub_bu_code_auth,
'DTP(Quarterly)' AS bu_code_auth,
'DTP(Quarterly)' AS DATA_SOURCE,
1 AS REGION_RATIO,
'Vacant' AS RSM_Name,
'Vacant' AS RC_Name,
'' AS TEAM,
'' AS MR_TRTY_CODE,
'' AS NSD_KCODE,
'' AS NSD_NAME,
'' AS RSD_KCODE,
'' AS RSD_NAME,
'' AS RSM_KCODE,
'' AS DSM_KCODE,
'' AS DSM_NAME,
'' AS MR_KCODE,
'' AS MR_NAME
FROM pack_2_market t2

340
ORG/DM_TD_EXT_THC_ORG.sql Normal file
View File

@@ -0,0 +1,340 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_THC_ORG (
-- REGION_CODE STRING,
-- REGION_NAME_EN STRING,
-- RSM_NAME STRING,
-- RC_CODE STRING,
-- RC_NAME_EN STRING,
-- RC_NAME STRING,
-- REGIONCENTER STRING,
-- AUDIT_COD STRING,
-- MARKET STRING,
-- BU_CATEGORY STRING,
-- BU_NAME_EN STRING,
-- SUB_BU_NAME STRING,
-- SUB_BU_CODE_AUTH STRING,
-- BU_CODE_AUTH STRING,
-- DATA_SOURCE STRING,
-- REGION_RATIO FLOAT,
-- TEAM STRING,
-- MR_TRTY_CODE STRING,
-- NSD_KCODE STRING,
-- NSD_NAME STRING,
-- RSD_KCODE STRING,
-- RSD_NAME STRING,
-- RSM_KCODE STRING,
-- DSM_KCODE STRING,
-- DSM_NAME STRING,
-- MR_KCODE STRING,
-- MR_NAME STRING)
-- USING delta
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_thc_org';
-- COMMAND ----------
-- MAGIC %run ../../../Common/config
-- COMMAND ----------
-- MAGIC %python
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dim_product_wide`').createOrReplaceTempView('cdw_dwd_dim_product_wide')
-- COMMAND ----------
-- 20260130 由于DM.dm_td_org数据需要转换，新增这个CELL。
create or replace temporary view dm_td_org_temp
as
SELECT org_key,mr_trty_tag,district_code,rsm_name,rc_name,p2.region_name,p2.rsd_kcode,p2.rsd_name
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_code = 'PC12' then 'PC09' else p2.bu_code end bu_code
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_report = 'EC' THEN 'Retail' ELSE p2.bu_report end bu_report
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_code = 'NA27' then 'NA06' else p2.sub_bu_code end sub_bu_code
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_name = 'EC' then 'Retail' else p2.sub_bu_name end sub_bu_name
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_name_en = 'EC' then 'Retail' else p2.sub_bu_name_en end sub_bu_name_en
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name = 'EC' then 'Retail' else p2.bu_name end bu_name
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name_en = 'EC' then 'Retail' else p2.bu_name_en end bu_name_en
,CASE WHEN p2.yyyymm >= '202601' and p2.nsd_kcode = 'ND_EC0001_V' then 'KSGX559' else p2.nsd_kcode end nsd_kcode
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name = 'EC'and p2.nsd_name = 'Vacant' then '李白杨' else p2.nsd_name end nsd_name
,CASE WHEN p2.yyyymm >= '202601' and p2.nsd_trty_code = 'ND_EC0001' then 'ND_RT0001' else p2.nsd_trty_code end nsd_trty_code
,CASE WHEN p2.yyyymm >= '202601' and p2.bsd_kcode = 'VP_EC0001_V' then 'VP_RT0001_V' else p2.bsd_kcode end bsd_kcode
,CASE WHEN p2.yyyymm >= '202601' and p2.bsd_trty_code = 'VP_EC0001' then 'VP_RT0001' else p2.bsd_trty_code end bsd_trty_code
,mr_kcode,dsm_trty_code,rsm_trty_code,dsm_kcode,rsd_trty_code
,district_name,dsm_name,mr_name,yyyymm,mr_trty_code,REGION_CODE,rc_code,bu_category,rsm_kcode
FROM DM.dm_td_org P2
-- COMMAND ----------
------------------------------------------------------------------------------------------------------
--修改时间：20241113
--修改人：Fanxujia
--修改内容：
--THC增加省份数据，因此增加架构数据。只需要Eagle的架构
------------------------------------------------------------------------------------------------------
----------------------------------------------------THC------------------------------------------------------------------------------
--------只有eagle的人能看THC
insert overwrite dm.DM_TD_EXT_THC_ORG
(REGION_CODE,region_name_en,rc_code,rc_name_en,RegionCenter,AUDIT_COD,MARKET,bu_category,bu_name_en,sub_bu_name,sub_bu_code_auth,bu_code_auth,DATA_SOURCE,REGION_RATIO,RSM_Name,RC_Name,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
)
with pack_market as (
select distinct pack_cod,market
from dm.dm_td_external_market_pack_mapping
where data_source = 'THC(Quarterly)'
)
,external_geo as (
select distinct province_c,audit_cod,REGION_CENTER
from dm.dm_td_external_geo_temp
where DATA_SOURCE = 'THC(Quarterly)'
)
,YM_CVH as (
select key_value from dm.dm_td_date_config
where key_code = 'pfmc_cvh'
)
,td_org as (
select distinct
REGION_CODE,
region_name,
rc_code,
rc_name,
bu_category,
bu_name_en,
sub_bu_name,
sub_bu_code,
bu_code,
RSM_Name,
rsd_name,
mr_trty_code,
rsm_trty_code
from dm_td_org_temp t1
inner join YM_CVH t2
on t1.yyyymm = t2.key_value
)
,territory as (
select distinct
trty_code,
inst_code,
family_code
from dm.dm_td_sd_territory_indication t1
inner join YM_CVH t2
on t1.yyyymm = t2.key_value
where bu = 'Eagle'
and t1.key_hp<>3 --chenwu 20250904 排除双考影响
)
,province_market as (
select t1.trty_code,
t2.province_name,
t5.market
from territory t1
left join dwd.dwd_td_institution t2
on t1.inst_code = t2.inst_code
left join cdw_dwd_dim_product_wide t3
on t1.family_code = t3.prd_family_cd
left join dwd.dwd_td_imscode_azcode t4
on t3.prd_brand_cd = t4.brandcode
left join pack_market t5
on t4.ims_pack_cod = t5.pack_cod
)
,national_sales as (
select distinct
PACK_COD,
AUDIT_COD
from DM.dm_tf_external_sales_thc
where DATA_SOURCE = 'THC(Quarterly)'
)
SELECT
DISTINCT t1.REGION_CODE,
t1.region_name,
t1.rc_code,
t1.rc_name,
t3.REGION_CENTER RegionCenter,
t3.AUDIT_COD,
t2.MARKET,
t1.bu_category,
t1.bu_name_en,
t1.sub_bu_name,
t1.sub_bu_code sub_bu_code_auth,
concat(t1.bu_code, 'THC(Quarterly)') bu_code_auth,
'THC(Quarterly)' as DATA_SOURCE,
1 REGION_RATIO,
t1.RSM_Name,
t1.rsd_name,
'' as TEAM,
'' as MR_TRTY_CODE,
'' as NSD_KCODE,
'' as NSD_NAME,
'' as RSD_KCODE,
'' as RSD_NAME,
'' as RSM_KCODE,
'' as DSM_KCODE,
'' as DSM_NAME,
'' as MR_KCODE,
'' as MR_NAME
from td_org t1
inner join province_market t2
on t1.mr_trty_code = t2.trty_code
inner join external_geo t3 on t2.province_name = t3.province_c
where t2.market is not null
and t3.audit_cod is not null
union all
SELECT DISTINCT
'' as REGION_CODE,
'' as region_name_en,
'' as rc_code,
'' as rc_name_en,
'' as RegionCenter,
t1.AUDIT_COD,
t2.MARKET,
'' as bu_category,
'' as bu_name_en,
'' as sub_bu_name,
'' as sub_bu_code_auth,
'THC(Quarterly)' as bu_code_auth,
'THC(Quarterly)' as DATA_SOURCE,
1 as REGION_RATIO,
'' as RSM_Name,
'' as rsd_name,
'' as TEAM,
'' as MR_TRTY_CODE,
'' as NSD_KCODE,
'' as NSD_NAME,
'' as RSD_KCODE,
'' as RSD_NAME,
'' as RSM_KCODE,
'' as DSM_KCODE,
'' as DSM_NAME,
'' as MR_KCODE,
'' as MR_NAME
FROM national_sales t1
INNER JOIN pack_market t2
ON t1.PACK_COD = t2.PACK_COD
-- COMMAND ----------
-- DBTITLE 1,add
--修改时间：20240808
--修改人：FanXuJia
--修改背景：
--事实表与维度表来自于不同手工表，目前存在一部分audit_cod在事实表有销量，但是在维度表没有这个audit_cod，导致销量缺失。
--因此需要从事实表补全这部分audit_cod
with sales as (
select distinct PACK_COD,AUDIT_COD,DATA_SOURCE from dm.dm_tf_external_sales
union
select distinct PACK_COD,AUDIT_COD,DATA_SOURCE from dm.dm_tf_external_sales_thc
)
,market_pack as (
select distinct PACK_COD,MARKET,DATA_SOURCE from dm.dm_td_external_market_pack_mapping
)
,org as (
select distinct AUDIT_COD,MARKET,DATA_SOURCE from dm.DM_TD_EXT_THC_ORG
)
,tmp_org as (
select distinct a.AUDIT_COD,b.MARKET,a.DATA_SOURCE
from sales a
left join market_pack b on a.PACK_COD = b.PACK_COD and a.DATA_SOURCE = b.DATA_SOURCE
left join org c on a.DATA_SOURCE = c.DATA_SOURCE and a.AUDIT_COD = c.AUDIT_COD and b.MARKET = c.MARKET
where C.AUDIT_COD IS NULL
)
insert into table dm.DM_TD_EXT_THC_ORG
(
REGION_CODE,
region_name_en,
RSM_Name,
rc_code,
rc_name_en,
RC_Name,
RegionCenter,
AUDIT_COD,
MARKET,
bu_category,
bu_name_en,
sub_bu_name,
sub_bu_code_auth,
bu_code_auth,
DATA_SOURCE,
REGION_RATIO,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
)
select
'ROC' as REGION_CODE,
'ROC' as region_name_en,
'Vacant' as RSM_Name,
'ROC' as rc_code,
'ROC' as rc_name_en,
'Vacant' as RC_Name,
null as RegionCenter,
AUDIT_COD,
MARKET,
null as bu_category,
null as bu_name_en,
null as sub_bu_name,
null as sub_bu_code_auth,
DATA_SOURCE as bu_code_auth,
DATA_SOURCE,
1 as REGION_RATIO,
'ROC' as TEAM,
'Vacant' as MR_TRTY_CODE,
'Vacant' as NSD_KCODE,
'Vacant' as NSD_NAME,
'Vacant' as RSD_KCODE,
'Vacant' as RSD_NAME,
'Vacant' as RSM_KCODE,
'Vacant' as DSM_KCODE,
'Vacant' as DSM_NAME,
'Vacant' as MR_KCODE,
'Vacant' as MR_NAME
from tmp_org
where DATA_SOURCE = 'THC(Quarterly)'
-- COMMAND ----------
-- DBTITLE 1,hidden_market
-----------------------------------------------------------
--修改时间：20241118
--修改人：Fanxujia
--修改内容：
--使用手工文件排除掉每个渠道的不需要显示的市场
-----------------------------------------------------------
with hidden_market as (
select distinct
case when upper(data_source) = 'CHPA' then 'IQVIA-CHPA(Monthly)'
when upper(data_source) = '科室市场' then 'XH Data(Quarterly)'
when upper(data_source) = 'EC' then 'EC(Monthly)'
when upper(data_source) = 'RETAIL' then 'Retail(Quarterly)'
when upper(data_source) = 'CHC' then 'CHC(Quarterly)'
when upper(data_source) = 'AIA' then 'AIA(Monthly)'
when upper(data_source) = 'THC' then 'THC(Quarterly)'
when upper(data_source) = 'COUNTY' then 'IQVIA-COUNTY(Quarterly)'
else data_source
end as data_source,
upper(hidden_market) as hidden_market
from dwd.dwd_gnd_ims_hidden_market
)
MERGE into dm.DM_TD_EXT_THC_ORG as t1
USING hidden_market t2
on t1.data_source = t2.data_source
and upper(t1.MARKET) = upper(t2.hidden_market)
WHEN MATCHED THEN
DELETE

279
ORG/DM_TD_EXT_XIEHE_ORG.sql Normal file
View File

@@ -0,0 +1,279 @@
-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TD_EXT_XIEHE_ORG (
-- REGION_CODE STRING,
-- REGION_NAME_EN STRING,
-- RSM_NAME STRING,
-- RC_CODE STRING,
-- RC_NAME_EN STRING,
-- RC_NAME STRING,
-- REGIONCENTER STRING,
-- AUDIT_COD STRING,
-- MARKET STRING,
-- BU_CATEGORY STRING,
-- BU_NAME_EN STRING,
-- SUB_BU_NAME STRING,
-- SUB_BU_CODE_AUTH STRING,
-- BU_CODE_AUTH STRING,
-- DATA_SOURCE STRING,
-- REGION_RATIO FLOAT,
-- TEAM STRING,
-- MR_TRTY_CODE STRING,
-- NSD_KCODE STRING,
-- NSD_NAME STRING,
-- RSD_KCODE STRING,
-- RSD_NAME STRING,
-- RSM_KCODE STRING,
-- DSM_KCODE STRING,
-- DSM_NAME STRING,
-- MR_KCODE STRING,
-- MR_NAME STRING)
-- USING delta
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_ext_xiehe_org';
-- COMMAND ----------
-- MAGIC %run ../../../Common/config
-- COMMAND ----------
-- MAGIC %python
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dim_product_wide`').createOrReplaceTempView('cdw_dwd_dim_product_wide')
-- COMMAND ----------
-- 20260130 由于DM.dm_td_org数据需要转换，新增这个CELL。
create or replace temporary view dm_td_org_temp
as
SELECT org_key,mr_trty_tag,district_code,rsm_name,rc_name,p2.region_name,p2.rsd_kcode,p2.rsd_name
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_code = 'PC12' then 'PC09' else p2.bu_code end bu_code
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_report = 'EC' THEN 'Retail' ELSE p2.bu_report end bu_report
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_code = 'NA27' then 'NA06' else p2.sub_bu_code end sub_bu_code
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_name = 'EC' then 'Retail' else p2.sub_bu_name end sub_bu_name
,CASE WHEN p2.yyyymm >= '202601' and p2.sub_bu_name_en = 'EC' then 'Retail' else p2.sub_bu_name_en end sub_bu_name_en
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name = 'EC' then 'Retail' else p2.bu_name end bu_name
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name_en = 'EC' then 'Retail' else p2.bu_name_en end bu_name_en
,CASE WHEN p2.yyyymm >= '202601' and p2.nsd_kcode = 'ND_EC0001_V' then 'KSGX559' else p2.nsd_kcode end nsd_kcode
,CASE WHEN p2.yyyymm >= '202601' and p2.bu_name = 'EC'and p2.nsd_name = 'Vacant' then '李白杨' else p2.nsd_name end nsd_name
,CASE WHEN p2.yyyymm >= '202601' and p2.nsd_trty_code = 'ND_EC0001' then 'ND_RT0001' else p2.nsd_trty_code end nsd_trty_code
,CASE WHEN p2.yyyymm >= '202601' and p2.bsd_kcode = 'VP_EC0001_V' then 'VP_RT0001_V' else p2.bsd_kcode end bsd_kcode
,CASE WHEN p2.yyyymm >= '202601' and p2.bsd_trty_code = 'VP_EC0001' then 'VP_RT0001' else p2.bsd_trty_code end bsd_trty_code
,mr_kcode,dsm_trty_code,rsm_trty_code,dsm_kcode,rsd_trty_code
,district_name,dsm_name,mr_name,yyyymm,mr_trty_code,REGION_CODE,rc_code,bu_category,rsm_kcode
FROM DM.dm_td_org P2
-- COMMAND ----------
---------------------------------------------------XIEHE------------------------------------------------------------------------------
--XIEHE参考CHPA
------所有BU能看XIEHE
insert overwrite dm.dm_td_ext_xiehe_org (
REGION_CODE,
region_name_en,
rc_code,
rc_name_en,
RegionCenter,
AUDIT_COD,
MARKET,
bu_category,
bu_name_en,
sub_bu_name,
sub_bu_code_auth,
bu_code_auth,
DATA_SOURCE,
REGION_RATIO,
RSM_Name,
RC_Name,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
)
SELECT
DISTINCT t1.REGION_CODE,
t1.region_name,
t1.rc_code,
t1.rc_name,
t4.REGION_CENTER RegionCenter,
t4.AUDIT_COD,
t3.MARKET,
t1.bu_category,
t1.bu_name_en,
t1.sub_bu_name,
t1.sub_bu_code sub_bu_code_auth,
concat(t1.bu_code, 'XH Data(Quarterly)') bu_code_auth,
'XH Data(Quarterly)' DATA_SOURCE,
1 REGION_RATIO,
t1.RSM_Name,
t1.rsd_name,
'' as TEAM,
'' as MR_TRTY_CODE,
'' as NSD_KCODE,
'' as NSD_NAME,
'' as RSD_KCODE,
'' as RSD_NAME,
'' as RSM_KCODE,
'' as DSM_KCODE,
'' as DSM_NAME,
'' as MR_KCODE,
'' as MR_NAME
from
dm_td_org_temp t1
left join(
select
distinct A.yyyymm,
A.trty_code,
B.city_name,
D.ims_prod_cod,
D.ims_pack_cod
from
dm.dm_td_sd_territory_indication A
left join dwd.dwd_td_institution B on A.inst_code = B.inst_code
left join cdw_dwd_dim_product_wide C on C.prd_family_cd = A.family_code
left join dwd.dwd_td_imscode_azcode D on C.prd_brand_cd = D.brandcode
where A.bu not in ('CUBE','KA')
and A.key_hp<>3 --chenwu 20250904 排除双考影响
) t2 on t2.trty_code = t1.mr_trty_code
and t2.yyyymm = t1.yyyymm
left join dws.dws_ims_td_market t3 on t2.ims_pack_cod = t3.pack_cod
left join DM.DM_TD_EXTERNAL_GEO_TEMP t4 on t2.city_name = concat(t4.city_c, '市')
AND t4.DATA_SOURCE = 'XH Data(Quarterly)'
--使用和performance cvh 的org架构
where
t1.yyyymm in(
select
key_value
from
dm.dm_td_date_config
where
key_code = 'pfmc_cvh'
)
and t3.market is not null
and t4.audit_cod is not null
-- COMMAND ----------
-- DBTITLE 1,add
--修改时间：20240808
--修改人：FanXuJia
--修改背景：
--事实表与维度表来自于不同手工表，目前存在一部分audit_cod在事实表有销量，但是在维度表没有这个audit_cod，导致销量缺失。
--因此需要从事实表补全这部分audit_cod
with sales as (
select distinct PACK_COD,AUDIT_COD,DATA_SOURCE from dm.dm_tf_external_sales
union
select distinct PACK_COD,AUDIT_COD,DATA_SOURCE from dm.dm_tf_external_sales_thc
)
,market_pack as (
select distinct PACK_COD,MARKET,DATA_SOURCE from dm.dm_td_external_market_pack_mapping
)
,org as (
select distinct AUDIT_COD,MARKET,DATA_SOURCE from dm.DM_TD_EXT_XIEHE_ORG
)
,tmp_org as (
select distinct a.AUDIT_COD,b.MARKET,a.DATA_SOURCE
from sales a
left join market_pack b on a.PACK_COD = b.PACK_COD and a.DATA_SOURCE = b.DATA_SOURCE
left join org c on a.DATA_SOURCE = c.DATA_SOURCE and a.AUDIT_COD = c.AUDIT_COD and b.MARKET = c.MARKET
where C.AUDIT_COD IS NULL
)
insert into table dm.dm_td_ext_xiehe_org
(
REGION_CODE,
region_name_en,
RSM_Name,
rc_code,
rc_name_en,
RC_Name,
RegionCenter,
AUDIT_COD,
MARKET,
bu_category,
bu_name_en,
sub_bu_name,
sub_bu_code_auth,
bu_code_auth,
DATA_SOURCE,
REGION_RATIO,
TEAM,
MR_TRTY_CODE,
NSD_KCODE,
NSD_NAME,
RSD_KCODE,
RSD_NAME,
RSM_KCODE,
DSM_KCODE,
DSM_NAME,
MR_KCODE,
MR_NAME
)
select
'ROC' as REGION_CODE,
'ROC' as region_name_en,
'Vacant' as RSM_Name,
'ROC' as rc_code,
'ROC' as rc_name_en,
'Vacant' as RC_Name,
null as RegionCenter,
AUDIT_COD,
MARKET,
null as bu_category,
null as bu_name_en,
null as sub_bu_name,
null as sub_bu_code_auth,
DATA_SOURCE as bu_code_auth,
DATA_SOURCE,
1 as REGION_RATIO,
'ROC' as TEAM,
'Vacant' as MR_TRTY_CODE,
'Vacant' as NSD_KCODE,
'Vacant' as NSD_NAME,
'Vacant' as RSD_KCODE,
'Vacant' as RSD_NAME,
'Vacant' as RSM_KCODE,
'Vacant' as DSM_KCODE,
'Vacant' as DSM_NAME,
'Vacant' as MR_KCODE,
'Vacant' as MR_NAME
from tmp_org
where DATA_SOURCE = 'XH Data(Quarterly)'
-- COMMAND ----------
-- DBTITLE 1,hidden_market
-----------------------------------------------------------
--修改时间：20241118
--修改人：Fanxujia
--修改内容：
--使用手工文件排除掉每个渠道的不需要显示的市场
-----------------------------------------------------------
with hidden_market as (
select distinct
case when upper(data_source) = 'CHPA' then 'IQVIA-CHPA(Monthly)'
when upper(data_source) = '科室市场' then 'XH Data(Quarterly)'
when upper(data_source) = 'EC' then 'EC(Monthly)'
when upper(data_source) = 'RETAIL' then 'Retail(Quarterly)'
when upper(data_source) = 'CHC' then 'CHC(Quarterly)'
when upper(data_source) = 'AIA' then 'AIA(Monthly)'
when upper(data_source) = 'THC' then 'THC(Quarterly)'
when upper(data_source) = 'COUNTY' then 'IQVIA-COUNTY(Quarterly)'
else data_source
end as data_source,
upper(hidden_market) as hidden_market
from dwd.dwd_gnd_ims_hidden_market
)
MERGE into dm.dm_td_ext_xiehe_org as t1
USING hidden_market t2
on t1.data_source = t2.data_source
and upper(t1.MARKET) = upper(t2.hidden_market)
WHEN MATCHED THEN
DELETE

11
OTHERS/01 dm_td_report_url.sql Normal file
View File

@@ -0,0 +1,11 @@
-- Databricks notebook source
----------------------------------------------------------
--修改时间：20241213
--修改人：Fanxujia
--修改内容：
--新增配置表，里面是MA报告以及AIA报告的url链接，用于PBI上做跳转的链接
----------------------------------------------------------
insert overwrite table dm.dm_td_report_url
(ReportName,URL)
select reportname,url
from dwd.dwd_gnd_ims_ma_aia_redirection

110
OTHERS/02 dws_ext_email_warning.sql Normal file
View File

@@ -0,0 +1,110 @@
-- Databricks notebook source
-- DBTITLE 1,DDL语句定义表
-- CREATE OR REPLACE TABLE dwd.dwd_ext_email_warning (
-- data_source STRING COMMENT '数据源名称',
-- max_data STRING COMMENT '数据源最新的日期，YYYYMM格式',
-- record_date STRING COMMENT '记录数据的日期',
-- etl_timestamp TIMESTAMP COMMENT 'ETL运行时间')
-- USING delta
-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DWD/dwd_ext_email_warning'
-- ;
-- CREATE OR REPLACE TABLE dws.dws_ext_email_warning (
-- data_source STRING COMMENT '数据源名称',
-- max_data STRING COMMENT '数据源最新的日期，YYYYMM格式',
-- if_update INT COMMENT '0代表没有更新，1代表更新过数据源',
-- etl_timestamp TIMESTAMP)
-- USING delta
-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DWS/dws_ext_email_warning'
-- COMMAND ----------
-- MAGIC %md
-- MAGIC ### dwd层
-- COMMAND ----------
-- DBTITLE 1,删除今天的数据并插入最新数据
delete from dwd.dwd_ext_email_warning
where record_date = date_format( from_utc_timestamp( current_timestamp,'UTC+8'),'yyyy-MM-dd');--删除今天的输入记录
insert into dwd.dwd_ext_email_warning
select
data_source
,max(yyyymm) max_data --数据源最大日期
,date_format( from_utc_timestamp( current_timestamp,'UTC+8'),'yyyy-MM-dd') record_date
,from_utc_timestamp( current_timestamp,'UTC+8') etl_timestamp
from dm.dm_tf_ext_unionall_sales
group by DATA_SOURCE
-- COMMAND ----------
-- MAGIC %md
-- MAGIC ### dws层
-- COMMAND ----------
-- DBTITLE 1,计算出各数据是否更新了数据
create or replace temporary view temp_datasource_update_info as
/*
1 获取今天日期的各数据源的记录情况
2 获取今天之前的上一个周期记录情况
3 今天日期的数据源最新数据月份 与 上一个周期的数据源最新数据月份 是否相等
*/
with
today_record_date (
select
data_source
,max_data
,record_date
from dwd.dwd_ext_email_warning
where record_date = date_format( from_utc_timestamp( current_timestamp,'UTC+8'),'yyyy-MM-dd') --取今天的数据
)
,previous_record_date (
select
data_source
,max_data
,record_date
from
(
select
data_source
,max_data
,record_date
,row_number() over( partition by data_source order by record_date desc ) num
from dwd.dwd_ext_email_warning
where record_date <> date_format( from_utc_timestamp( current_timestamp,'UTC+8'),'yyyy-MM-dd')
)
where num = 1
)
select
t1.data_source
,t1.max_data
,case when t1.max_data = t2.max_data then 0 else 1 end as if_update
,from_utc_timestamp( current_timestamp,'UTC+8') etl_timestamp
from today_record_date t1
inner join previous_record_date t2 on t1.data_source = t2.data_source
-- COMMAND ----------
-- DBTITLE 1,写入dws层
INSERT OVERWRITE dws.dws_ext_email_warning
select
data_source
,max_data
,if_update
,etl_timestamp
from temp_datasource_update_info
-- COMMAND ----------
-- MAGIC %md
-- MAGIC ### ads层
-- COMMAND ----------
-- select
-- data_source
-- ,max_data
-- ,if_update
-- from dws.dws_ext_email_warning

42
OTHERS/external_triggered_email.py Normal file
View File

@@ -0,0 +1,42 @@
# Databricks notebook source
import base64
import time
import urllib.parse
import requests
from Crypto.Cipher import AES
from Crypto.Util.Padding import pad
# === 配置区 ===
# Spring Boot 服务端的密钥（Base64编码）
SECRET_KEY_BASE64 = "qkLkLHPnqNfJ5d3X6DgLpw=="
# 接口地址
# https://mdcuat.astrazeneca.cn/earth/app/mail/task/ma/check
# https://mdc.astrazeneca.cn/earth/app/mail/task/ma/check
API_URL = "https://mdc.astrazeneca.cn/earth/app/mail/task/ma/check"
# === AES 加密函数 ===
def generate_aes_token(secret_key_base64: str) -> str:
# Hutool 的 SecureUtil.aes() 默认是 AES/ECB/PKCS5Padding
secret_key = base64.b64decode(secret_key_base64)
cipher = AES.new(secret_key, AES.MODE_ECB)
# 加密当前时间戳（毫秒）
timestamp = str(int(time.time() * 1000))
encrypted = cipher.encrypt(pad(timestamp.encode("utf-8"), AES.block_size))
# 输出 Base64 编码的 token
return base64.b64encode(encrypted).decode("utf-8")
# === 调用接口 ===
def call_check_api():
token = generate_aes_token(SECRET_KEY_BASE64)
encoded_token = urllib.parse.quote(token)
url = f"{API_URL}?token={encoded_token}"
print(f"请求URL: {url}")
response = requests.get(url)
print("响应状态码:", response.status_code)
print("响应内容:", response.text)
if __name__ == "__main__":
call_check_api()

617
Retail/01 load_tmp_data.py Normal file
View File

@@ -0,0 +1,617 @@
# Databricks notebook source
### constant
LEVEL_TA_CV = 'CV'
LEVLE_TA_RE = 'RE'
LEVEL_TA_RE2 = 'RE2'
LEVEL_TA_GI = 'GI'
LEVEL_TA_DM = 'DM'
LEVEL_TA_RD = 'RD'
LEVEL_MARKET_HTN = '高血压用药'
LEVEL_MARKET_STATIN_XZK = '他汀类+血脂康'
LEVEL_MARKET_BRILINTA = 'Brilinta Market'
LEVEL_MARKET_COPD = '小儿咳喘'
LEVEL_MARKET_AAGSA_PPI_ORAL = '慢性胃炎、胃溃疡'
LEVEL_MARKET_ATOMIZER = '慢性阻塞性肺疾病'
LEVEL_MARKET_NIAD = 'NIAD'
LEVEL_MARKET_RD = 'RD Market'
# COMMAND ----------
############################################################START##############################################################
### STEP-1: load rawdata to tmp table
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-1: load rawdata to tmp table
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load rawdata to tmp table
# MAGIC -- 1.1 load brand data to tmp.tmp_retail_brand_rawdata
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert overwrite table tmp.tmp_retail_brand_rawdata
# MAGIC
# MAGIC select
# MAGIC type,
# MAGIC ta,
# MAGIC market,
# MAGIC zk_brand_category,
# MAGIC zk_common_name,
# MAGIC zk_manu_des,
# MAGIC rc_name_en,
# MAGIC province_city,
# MAGIC quarter,
# MAGIC ytd,
# MAGIC '',
# MAGIC sales_value * 1000000,
# MAGIC sales_volume * 1000000
# MAGIC from dwd.dwd_gnd_ext_zk_brand
# MAGIC where ranked_by = 'value'
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load rawdata to tmp table
# MAGIC -- 1.2 load brand data to tmp.tmp_retail_pack_rawdata
# MAGIC -- 1.2.1 load from dwd.dwd_gnd_ext_retail_htn (pack-CV-高血压-化学药-全国.xlsx)
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert overwrite table tmp.tmp_retail_pack_rawdata
# MAGIC
# MAGIC select
# MAGIC case when length(zk_product_id) < 7 then right(concat('0000000',zk_product_id),7) else zk_product_id end as product_id,
# MAGIC zk_product_id,
# MAGIC zk_region,
# MAGIC zk_rx_otc,
# MAGIC zk_medicine_type,
# MAGIC zk_medicine_tier1,
# MAGIC zk_medicine_tier2,
# MAGIC zk_medicine_tier3,
# MAGIC zk_medicine_tier4,
# MAGIC zk_common_name,
# MAGIC zk_dosage_form,
# MAGIC zk_user_type,
# MAGIC zk_category_name,
# MAGIC zk_product_name,
# MAGIC zk_brand_name,
# MAGIC zk_manu_des,
# MAGIC zk_corp_des,
# MAGIC zk_pack_des,
# MAGIC month,
# MAGIC quarter,
# MAGIC replace(price,',','' ),
# MAGIC replace(sales_unit,',','' ),
# MAGIC replace(sales_value,',','' ),
# MAGIC digital_spread_rate,
# MAGIC weighted_spread_rate,
# MAGIC counting_unit,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 'CV',
# MAGIC '高血压用药',
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 0,
# MAGIC null
# MAGIC from dwd.dwd_gnd_ext_retail_htn
# MAGIC --------------------------------- *hard_code* ---------------------------------------
# MAGIC -- 多达一、天一宁、others、氨氯地平阿托伐他汀钙 数据在 高血压和他汀血脂康里面重复，
# MAGIC -- 只需要取一份，优先取有拆分比例的
# MAGIC -- 对应的product_id 分别是：'-356','5258049','5852881','9167744','9526959','9167556','9279325','8533952'
# MAGIC -- 或者使用条件：zk_common_name <> '氨氯地平阿托伐他汀钙'
# MAGIC where zk_common_name <> '氨氯地平阿托伐他汀钙'
# MAGIC --------------------------------- *hard_code* ---------------------------------------
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load rawdata to tmp table
# MAGIC -- 1.2 load brand data to tmp.tmp_retail_pack_rawdata
# MAGIC -- 1.2.2 load from dwd.dwd_gnd_ext_retail_statin_xzk (pack-CV-他汀类+血脂康-全国.xlsx)
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert into table tmp.tmp_retail_pack_rawdata
# MAGIC
# MAGIC select
# MAGIC case when length(zk_product_id) < 7 then right(concat('0000000',zk_product_id),7) else zk_product_id end as product_id,
# MAGIC zk_product_id,
# MAGIC zk_region,
# MAGIC zk_rx_otc,
# MAGIC zk_medicine_type,
# MAGIC zk_medicine_tier1,
# MAGIC zk_medicine_tier2,
# MAGIC zk_medicine_tier3,
# MAGIC zk_medicine_tier4,
# MAGIC zk_common_name,
# MAGIC zk_dosage_form,
# MAGIC zk_user_type,
# MAGIC zk_category_name,
# MAGIC zk_product_name,
# MAGIC zk_brand_name,
# MAGIC zk_manu_des,
# MAGIC zk_corp_des,
# MAGIC zk_pack_des,
# MAGIC month,
# MAGIC quarter,
# MAGIC replace(price,',','' ),
# MAGIC replace(sales_unit,',','' ),
# MAGIC replace(sales_value,',','' ),
# MAGIC digital_spread_rate,
# MAGIC weighted_spread_rate,
# MAGIC counting_unit,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 'CV',
# MAGIC '他汀类+血脂康',
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 0,
# MAGIC null
# MAGIC from dwd.dwd_gnd_ext_retail_statin_xzk
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load rawdata to tmp table
# MAGIC -- 1.2 load brand data to tmp.tmp_retail_pack_rawdata
# MAGIC -- 1.2.3 load from dwd.dwd_gnd_ext_retail_nataional_oap (pack-CV-抗血栓2通用名-全国.xlsx)
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert into table tmp.tmp_retail_pack_rawdata
# MAGIC
# MAGIC select
# MAGIC case when length(zk_product_id) < 7 then right(concat('0000000',zk_product_id),7) else zk_product_id end as product_id,
# MAGIC zk_product_id,
# MAGIC zk_region,
# MAGIC zk_rx_otc,
# MAGIC zk_medicine_type,
# MAGIC zk_medicine_tier1,
# MAGIC zk_medicine_tier2,
# MAGIC zk_medicine_tier3,
# MAGIC zk_medicine_tier4,
# MAGIC zk_common_name,
# MAGIC zk_dosage_form,
# MAGIC zk_user_type,
# MAGIC zk_category_name,
# MAGIC zk_product_name,
# MAGIC zk_brand_name,
# MAGIC zk_manu_des,
# MAGIC zk_corp_des,
# MAGIC zk_pack_des,
# MAGIC month,
# MAGIC quarter,
# MAGIC replace(price,',','' ),
# MAGIC replace(sales_unit,',','' ),
# MAGIC replace(sales_value,',','' ),
# MAGIC digital_spread_rate,
# MAGIC weighted_spread_rate,
# MAGIC counting_unit,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 'CV',
# MAGIC 'Brilinta Market',
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 0,
# MAGIC null
# MAGIC from dwd.dwd_gnd_ext_retail_nataional_oap
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load rawdata to tmp table
# MAGIC -- 1.2 load brand data to tmp.tmp_retail_pack_rawdata
# MAGIC -- 1.2.4 load from dwd.dwd_gnd_ext_retail_anti_asthma_copd (pack-RE-慢阻肺-全国.xlsx)
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert into table tmp.tmp_retail_pack_rawdata
# MAGIC
# MAGIC select
# MAGIC case when length(zk_product_id) < 7 then right(concat('0000000',zk_product_id),7) else zk_product_id end as product_id,
# MAGIC zk_product_id,
# MAGIC zk_region,
# MAGIC zk_rx_otc,
# MAGIC zk_medicine_type,
# MAGIC zk_medicine_tier1,
# MAGIC zk_medicine_tier2,
# MAGIC zk_medicine_tier3,
# MAGIC zk_medicine_tier4,
# MAGIC zk_common_name,
# MAGIC zk_dosage_form,
# MAGIC zk_user_type,
# MAGIC zk_category_name,
# MAGIC zk_product_name,
# MAGIC zk_brand_name,
# MAGIC zk_manu_des,
# MAGIC zk_corp_des,
# MAGIC zk_pack_des,
# MAGIC month,
# MAGIC quarter,
# MAGIC replace(price,',','' ),
# MAGIC replace(sales_unit,',','' ),
# MAGIC replace(sales_value,',','' ),
# MAGIC digital_spread_rate,
# MAGIC weighted_spread_rate,
# MAGIC counting_unit,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC --------------2024-----------
# MAGIC --'RE',
# MAGIC --'慢性阻塞性肺疾病',
# MAGIC --------------2025-----------
# MAGIC 'RE2',
# MAGIC '小儿咳喘',
# MAGIC -----------------------------
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 0,
# MAGIC null
# MAGIC from dwd.dwd_gnd_ext_retail_anti_asthma_copd
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load rawdata to tmp table
# MAGIC -- 1.2 load brand data to tmp.tmp_retail_pack_rawdata
# MAGIC -- 1.2.5 load from dwd.dwd_gnd_ext_retail_aagsa_ppi_oral (pack-GI-慢性胃炎胃溃疡-全国.xlsx)
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert into table tmp.tmp_retail_pack_rawdata
# MAGIC
# MAGIC select
# MAGIC case when length(zk_product_id) < 7 then right(concat('0000000',zk_product_id),7) else zk_product_id end as product_id,
# MAGIC zk_product_id,
# MAGIC zk_region,
# MAGIC zk_rx_otc,
# MAGIC zk_medicine_type,
# MAGIC zk_medicine_tier1,
# MAGIC zk_medicine_tier2,
# MAGIC zk_medicine_tier3,
# MAGIC zk_medicine_tier4,
# MAGIC zk_common_name,
# MAGIC zk_dosage_form,
# MAGIC zk_user_type,
# MAGIC zk_category_name,
# MAGIC zk_product_name,
# MAGIC zk_brand_name,
# MAGIC zk_manu_des,
# MAGIC zk_corp_des,
# MAGIC zk_pack_des,
# MAGIC month,
# MAGIC quarter,
# MAGIC replace(price,',','' ),
# MAGIC replace(sales_unit,',','' ),
# MAGIC replace(sales_value,',','' ),
# MAGIC digital_spread_rate,
# MAGIC weighted_spread_rate,
# MAGIC counting_unit,
# MAGIC null as pack_code,
# MAGIC null as molecule_code,
# MAGIC null as molecule_desc,
# MAGIC null as product_code,
# MAGIC null as product_desc,
# MAGIC 'GI' as level_ta,
# MAGIC '慢性胃炎、胃溃疡' as level_market,
# MAGIC null as level_molecule,
# MAGIC null as level_brand,
# MAGIC null as ratio_val,
# MAGIC null as ratio_vol,
# MAGIC 0 as data_flag,
# MAGIC null as brand_flag
# MAGIC from dwd.dwd_gnd_ext_retail_aagsa_ppi_oral
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load rawdata to tmp table
# MAGIC -- 1.2 load brand data to tmp.tmp_retail_pack_rawdata
# MAGIC -- 1.2.6 load from dwd.dwd_gnd_ext_retail_atomizer (pack-雾化器-全国&县域数据.xlsx)
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert into table tmp.tmp_retail_pack_rawdata
# MAGIC
# MAGIC select
# MAGIC case when length(zk_product_id) < 7 then right(concat('0000000',zk_product_id),7) else zk_product_id end as product_id,
# MAGIC zk_product_id,
# MAGIC zk_region,
# MAGIC zk_rx_otc,
# MAGIC zk_medicine_type,
# MAGIC zk_medicine_tier1,
# MAGIC zk_medicine_tier2,
# MAGIC zk_medicine_tier3,
# MAGIC zk_medicine_tier4,
# MAGIC zk_common_name,
# MAGIC zk_dosage_form,
# MAGIC zk_user_type,
# MAGIC zk_category_name,
# MAGIC zk_product_name,
# MAGIC zk_brand_name,
# MAGIC zk_manu_des,
# MAGIC zk_corp_des,
# MAGIC zk_pack_des,
# MAGIC month,
# MAGIC quarter,
# MAGIC replace(price,',','' ),
# MAGIC replace(sales_unit,',','' ),
# MAGIC replace(sales_value,',','' ),
# MAGIC digital_spread_rate,
# MAGIC weighted_spread_rate,
# MAGIC counting_unit,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 'RE',
# MAGIC '慢性阻塞性肺疾病',
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 0,
# MAGIC null
# MAGIC from dwd.dwd_gnd_ext_retail_atomizer
# MAGIC ------------------------------ *hard_code* ------------------------------------------
# MAGIC -- 24年关于 RE-慢性阻塞性肺疾病的PACK数据，额外买了“雾化器”的24省数据
# MAGIC -- 所以这里只取24省明细数据，全国的数已经在“pack-GI-慢性胃炎胃溃疡-全国.xlsx”中取过了
# MAGIC -- 25年就直接取全部数据
# MAGIC -- where zk_region <> '全国'
# MAGIC ------------------------------ *hard_code* ------------------------------------------
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load rawdata to tmp table
# MAGIC -- 1.2 load brand data to tmp.tmp_retail_pack_rawdata
# MAGIC -- 1.2.7 load from dwd.dwd_gnd_ext_retail_nataional_niad (pack-DM-口服降糖化学药.xlsx)
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert into table tmp.tmp_retail_pack_rawdata
# MAGIC
# MAGIC select
# MAGIC case when length(zk_product_id) < 7 then right(concat('0000000',zk_product_id),7) else zk_product_id end as product_id,
# MAGIC zk_product_id,
# MAGIC zk_region,
# MAGIC zk_rx_otc,
# MAGIC zk_medicine_type,
# MAGIC zk_medicine_tier1,
# MAGIC zk_medicine_tier2,
# MAGIC zk_medicine_tier3,
# MAGIC zk_medicine_tier4,
# MAGIC zk_common_name,
# MAGIC zk_dosage_form,
# MAGIC zk_user_type,
# MAGIC zk_category_name,
# MAGIC zk_product_name,
# MAGIC zk_brand_name,
# MAGIC zk_manu_des,
# MAGIC zk_corp_des,
# MAGIC zk_pack_des,
# MAGIC month,
# MAGIC quarter,
# MAGIC replace(price,',','' ),
# MAGIC replace(sales_unit,',','' ),
# MAGIC replace(sales_value,',','' ),
# MAGIC digital_spread_rate,
# MAGIC weighted_spread_rate,
# MAGIC counting_unit,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 'DM',
# MAGIC 'NIAD',
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 0,
# MAGIC null
# MAGIC from dwd.dwd_gnd_ext_retail_nataional_niad
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load rawdata to tmp table
# MAGIC -- 1.2 load brand data to tmp.tmp_retail_pack_rawdata
# MAGIC -- 1.2.8 load from dwd.dwd_gnd_ext_retail_nataional_rd (pack-RD-肾科-全国.xlsx)
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert into table tmp.tmp_retail_pack_rawdata
# MAGIC
# MAGIC ------------------------------ *hard_code* ------------------------------------------
# MAGIC -- 剔除利尿剂数据
# MAGIC with tmp_exclusion as (
# MAGIC select distinct product_id
# MAGIC from dwd.dwd_gnd_ext_retail_pack_property
# MAGIC where lower(atc2_cod) = 'c03'
# MAGIC )
# MAGIC ------------------------------ *hard_code* ------------------------------------------
# MAGIC
# MAGIC select
# MAGIC case when length(zk_product_id) < 7 then right(concat('0000000',zk_product_id),7) else zk_product_id end as product_id,
# MAGIC zk_product_id,
# MAGIC zk_region,
# MAGIC zk_rx_otc,
# MAGIC zk_medicine_type,
# MAGIC zk_medicine_tier1,
# MAGIC zk_medicine_tier2,
# MAGIC zk_medicine_tier3,
# MAGIC zk_medicine_tier4,
# MAGIC zk_common_name,
# MAGIC zk_dosage_form,
# MAGIC zk_user_type,
# MAGIC zk_category_name,
# MAGIC zk_product_name,
# MAGIC zk_brand_name,
# MAGIC zk_manu_des,
# MAGIC zk_corp_des,
# MAGIC zk_pack_des,
# MAGIC month,
# MAGIC quarter,
# MAGIC replace(price,',','' ),
# MAGIC replace(sales_unit,',','' ),
# MAGIC replace(sales_value,',','' ),
# MAGIC digital_spread_rate,
# MAGIC weighted_spread_rate,
# MAGIC counting_unit,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 'RD',
# MAGIC 'RD Market',
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 0,
# MAGIC null
# MAGIC from dwd.dwd_gnd_ext_retail_nataional_rd a
# MAGIC where not exists (
# MAGIC select * from tmp_exclusion b
# MAGIC where case when length(zk_product_id) < 7 then right(concat('0000000',zk_product_id),7) else zk_product_id end = b.product_id
# MAGIC )
# MAGIC
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load rawdata to tmp table
# MAGIC -- 1.2 load brand data to tmp.tmp_retail_pack_rawdata
# MAGIC -- 1.2.9 load from dwd.dwd_gnd_ext_retail_metoprolol_tartrat (pack-CV-酒石酸美托洛尔.xlsx)
# MAGIC -- 2024年采买了酒石酸美托洛尔的PACK数据，2025年未采买
# MAGIC -------------------------------------------------------------------------------------
# MAGIC --insert into table tmp.tmp_retail_pack_rawdata
# MAGIC
# MAGIC select
# MAGIC case when length(zk_product_id) < 7 then right(concat('0000000',zk_product_id),7) else zk_product_id end as product_id,
# MAGIC zk_product_id,
# MAGIC zk_region,
# MAGIC zk_rx_otc,
# MAGIC zk_medicine_type,
# MAGIC zk_medicine_tier1,
# MAGIC zk_medicine_tier2,
# MAGIC zk_medicine_tier3,
# MAGIC zk_medicine_tier4,
# MAGIC zk_common_name,
# MAGIC zk_dosage_form,
# MAGIC zk_user_type,
# MAGIC zk_category_name,
# MAGIC zk_product_name,
# MAGIC zk_brand_name,
# MAGIC zk_manu_des,
# MAGIC zk_corp_des,
# MAGIC zk_pack_des,
# MAGIC month,
# MAGIC quarter,
# MAGIC replace(price,',','' ),
# MAGIC replace(sales_unit,',','' ),
# MAGIC replace(sales_value,',','' ),
# MAGIC digital_spread_rate,
# MAGIC weighted_spread_rate,
# MAGIC counting_unit,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 'CV',
# MAGIC '高血压用药',
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 0,
# MAGIC null
# MAGIC from dwd.dwd_gnd_ext_retail_metoprolol_tartrat a
# MAGIC ------------------------------ *hard_code* ------------------------------------------
# MAGIC -- 24年关于 CV-高血压的PACK数据，额外买了“酒石酸美托洛尔”的24省数据
# MAGIC -- 所以这里只取24省明细数据，全国的数已经在“pack-CV-高血压-化学药-全国.xlsx”中取过了
# MAGIC where zk_region <> '全国'
# MAGIC ------------------------------ *hard_code* ------------------------------------------
# COMMAND ----------
# %sql
# 20260302 chenwu取消此段运行，小儿咳喘模版暂不使用。Fasenra Market数据由DTP RAW DATA上传
# -------------------------------------------------------------------------------------
# -- STEP-1: load rawdata to tmp table
# -- 1.2 load brand data to tmp.tmp_retail_pack_rawdata
# -- 1.2.10 load from dwd.dwd_gnd_ext_retail_asthma (pack-手动补充)
# -- 手动补充的小儿咳喘的数据，这些数据无需拆分
# -------------------------------------------------------------------------------------
# insert into table tmp.tmp_retail_pack_rawdata
# select
# case when length(zk_product_id) < 7 then right(concat('0000000',zk_product_id),7) else zk_product_id end as product_id,
# zk_product_id,
# zk_region,
# zk_rx_otc,
# zk_medicine_type,
# zk_medicine_tier1,
# zk_medicine_tier2,
# zk_medicine_tier3,
# zk_medicine_tier4,
# zk_common_name,
# zk_dosage_form,
# zk_user_type,
# zk_category_name,
# zk_product_name,
# zk_brand_name,
# zk_manu_des,
# zk_corp_des,
# zk_pack_des,
# month,
# quarter,
# price,
# sales_unit,
# sales_value,
# digital_spread_rate,
# weighted_spread_rate,
# counting_unit,
# null,
# null,
# null,
# null,
# null,
# 'RE_NO_SPLIT',
# '小儿咳喘',
# null,
# null,
# null,
# null,
# 0,
# null
# from dwd.dwd_gnd_ext_retail_asthma
# COMMAND ----------
############################################################END##############################################################

672
Retail/02 split_brand_data.py Normal file
View File

@@ -0,0 +1,672 @@
# Databricks notebook source
### constant
DATA_TYPE_MARKET = 'MARKET'
DATA_TYPE_MOLECULE = 'MOLECULE'
DATA_TYPE_BRAND = 'BRAND'
DATA_TYPE_MULTI_MARKET = 'MULTI_MARKET'
DATA_TYPE_MULTI_MOLECULE = 'MULTI_MOLECULE'
DATA_TYPE_MULTI_BRAND = 'MULTI_BRAND'
DATA_TYPE_DUPLICATE = 'DUPLICATE'
DATA_TYPE_HEDGE_TA = 'HEDGE_TA'
DATA_TYPE_HEDGE_MARKET = 'HEDGE_MARKET'
DATA_TYPE_HEDGE_MOLECULE = 'HEDGE_MOLECULE'
DATA_TYPE_HEDGE_BRAND = 'HEDGE_BRAND'
DATA_FLAG_RAW = 0 ### final pack_flag = 1
DATA_FLAG_SUB = 1 ### final pack_flag = 2
DATA_FLAG_ADD = 2 ### final pack_flag = 2
DATA_FLAG_RATIO = 3 ### final pack_flag = 2
DATA_FLAG_AVG = 4 ### final pack_flag = 2 ### The current version is not in use
DATA_FLAG_HEDGE = 5 ### final pack_flag = 2
BRAND_FLAG_NATIVE = 1 ### final brand_flag = 1
BRAND_FLAG_CALCULATE = 2 ### final brand_flag = 2
BRAND_FLAG_NONE = 3 ### final brand_flag = 2
DATA_OTHER_TA = 'OTHER_TA'
DATA_OTHER_MARKET = 'OTHER_MARKET'
DATA_OTHER_MOLECULE = 'OTHER_MOLECULE'
DATA_OTHER_BRAND = 'OTHER_BRAND'
DATA_AREA_TYPE_ROC = 'ROC'
DATA_LABEL_AREA_ALL = '全国'
# COMMAND ----------
############################################################START##############################################################
### STEP-1: load data to tmp table
### STEP-2: substract data from level market to brand
### STEP-3: accumulate data from level brand to market
### STEP-4: calculate brand ratio
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-1: load data to tmp table
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load data to tmp table
# MAGIC -- 1.1 load LEVEL_MARKET data to tmp.tmp_retail_level_market
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC insert overwrite table tmp.tmp_retail_level_market
# MAGIC
# MAGIC select
# MAGIC a.quarter,
# MAGIC a.province_city,
# MAGIC b.level_ta,
# MAGIC b.level_market,
# MAGIC a.sales_val,
# MAGIC a.sales_vol,
# MAGIC 0
# MAGIC from tmp.tmp_retail_brand_rawdata a
# MAGIC inner join dwd.dwd_gnd_retail_split_automatic b
# MAGIC on a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.brand_category = b.brand
# MAGIC and ifnull(a.common_name, '') = ifnull(b.common_name, '')
# MAGIC where b.level = 'MARKET'
# MAGIC
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load data to tmp table
# MAGIC -- 1.2 load LEVEL_MOLECULE data to tmp.tmp_retail_level_molecule
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert overwrite table tmp.tmp_retail_level_molecule
# MAGIC
# MAGIC select
# MAGIC a.quarter,
# MAGIC a.province_city,
# MAGIC b.level_ta,
# MAGIC b.level_market,
# MAGIC b.level_molecule,
# MAGIC a.sales_val,
# MAGIC a.sales_vol,
# MAGIC 0
# MAGIC from tmp.tmp_retail_brand_rawdata a
# MAGIC inner join dwd.dwd_gnd_retail_split_automatic b
# MAGIC on a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.brand_category = b.brand
# MAGIC and ifnull(a.common_name, '') = ifnull(b.common_name, '')
# MAGIC where b.level = 'MOLECULE'
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load data to tmp table
# MAGIC -- 1.3 load LEVEL_BRAND(BRAND & MULTI_BRAND) data to tmp.tmp_retail_level_brand
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert overwrite table tmp.tmp_retail_level_brand
# MAGIC
# MAGIC with exploded_data as (
# MAGIC select
# MAGIC ta,
# MAGIC market,
# MAGIC brand,
# MAGIC ifnull(common_name,'' ) as common_name,
# MAGIC level,
# MAGIC split(level_ta,',') array_ta,
# MAGIC split(level_market,',') array_market,
# MAGIC split(level_molecule,',') array_molecule,
# MAGIC split(level_brand,',') array_brand
# MAGIC from dwd.dwd_gnd_retail_split_automatic
# MAGIC where level like '%MULTI_BRAND%'
# MAGIC ), tmp_config as (
# MAGIC select
# MAGIC ta,
# MAGIC market,
# MAGIC brand,
# MAGIC common_name,
# MAGIC level,
# MAGIC array_ta[idx] as level_ta,
# MAGIC array_market[idx] as level_market,
# MAGIC array_molecule[idx] as level_molecule,
# MAGIC array_brand[idx] as level_brand
# MAGIC from exploded_data
# MAGIC lateral view posexplode(array_ta) AS idx, split_ta
# MAGIC )
# MAGIC ----------------brand data---------------------
# MAGIC select
# MAGIC a.quarter,
# MAGIC a.province_city,
# MAGIC b.level_ta,
# MAGIC b.level_market,
# MAGIC b.level_molecule,
# MAGIC b.level_brand,
# MAGIC a.sales_val,
# MAGIC a.sales_vol,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 0
# MAGIC from tmp.tmp_retail_brand_rawdata a
# MAGIC inner join dwd.dwd_gnd_retail_split_automatic b
# MAGIC on a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.brand_category = b.brand
# MAGIC and ifnull(a.common_name, '') = ifnull(b.common_name, '')
# MAGIC where b.level = 'BRAND'
# MAGIC
# MAGIC union all
# MAGIC ----------------multi brand data---------------------
# MAGIC select
# MAGIC a.quarter,
# MAGIC a.province_city,
# MAGIC b.level_ta,
# MAGIC b.level_market,
# MAGIC b.level_molecule,
# MAGIC b.level_brand,
# MAGIC a.sales_val,
# MAGIC a.sales_vol,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 0
# MAGIC from tmp.tmp_retail_brand_rawdata a
# MAGIC inner join tmp_config b
# MAGIC on a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.brand_category = b.brand
# MAGIC and ifnull(a.common_name, '') = ifnull(b.common_name, '')
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load data to tmp table
# MAGIC -- 1.4 append HEDGE_TA data to every level
# MAGIC -- 1.4.1 append HEDGE_MARKET data to LEVEL_MARKET
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC with tmp_hedge as (
# MAGIC select
# MAGIC a.quarter,
# MAGIC a.province_city,
# MAGIC 0 - sum(a.sales_val) as hedge_val,
# MAGIC 0 - sum(a.sales_vol) as hedge_vol
# MAGIC from tmp.tmp_retail_brand_rawdata a
# MAGIC inner join dwd.dwd_gnd_retail_split_automatic b
# MAGIC on a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.brand_category = b.brand
# MAGIC and ifnull(a.common_name, '') = ifnull(b.common_name, '')
# MAGIC where b.level like '%HEDGE_TA%'
# MAGIC group by a.quarter,a.province_city
# MAGIC )
# MAGIC
# MAGIC insert into tmp.tmp_retail_level_market
# MAGIC select
# MAGIC quarter,
# MAGIC province_city,
# MAGIC 'HEDGE_TA',
# MAGIC 'HEDGE_MARKET',
# MAGIC hedge_val,
# MAGIC hedge_vol,
# MAGIC 5
# MAGIC from tmp_hedge
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load data to tmp table
# MAGIC -- 1.4 append HEDGE_TA data to every level
# MAGIC -- 1.4.2 append HEDGE_MOLECULE data to LEVEL_MOLECULE
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC with tmp_hedge as (
# MAGIC select
# MAGIC a.quarter,
# MAGIC a.province_city,
# MAGIC 0 - sum(a.sales_val) as hedge_val,
# MAGIC 0 - sum(a.sales_vol) as hedge_vol
# MAGIC from tmp.tmp_retail_brand_rawdata a
# MAGIC inner join dwd.dwd_gnd_retail_split_automatic b
# MAGIC on a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.brand_category = b.brand
# MAGIC and ifnull(a.common_name, '') = ifnull(b.common_name, '')
# MAGIC where b.level like '%HEDGE_TA%'
# MAGIC group by a.quarter,a.province_city
# MAGIC )
# MAGIC insert into tmp.tmp_retail_level_molecule
# MAGIC select
# MAGIC quarter,
# MAGIC province_city,
# MAGIC 'HEDGE_TA',
# MAGIC 'HEDGE_MARKET',
# MAGIC 'HEDGE_MOLECULE',
# MAGIC hedge_val,
# MAGIC hedge_vol,
# MAGIC 5
# MAGIC from tmp_hedge
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load data to tmp table
# MAGIC -- 1.4 append HEDGE_TA data to every level
# MAGIC -- 1.4.3 append HEDGE_BRAND data to LEVEL_BRAND
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC with tmp_hedge as (
# MAGIC select
# MAGIC a.quarter,
# MAGIC a.province_city,
# MAGIC 0 - sum(a.sales_val) as hedge_val,
# MAGIC 0 - sum(a.sales_vol) as hedge_vol
# MAGIC from tmp.tmp_retail_brand_rawdata a
# MAGIC inner join dwd.dwd_gnd_retail_split_automatic b
# MAGIC on a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.brand_category = b.brand
# MAGIC and ifnull(a.common_name, '') = ifnull(b.common_name, '')
# MAGIC where b.level like '%HEDGE_TA%'
# MAGIC group by a.quarter,a.province_city
# MAGIC )
# MAGIC
# MAGIC insert into table tmp.tmp_retail_level_brand
# MAGIC select
# MAGIC quarter,
# MAGIC province_city,
# MAGIC 'HEDGE_TA',
# MAGIC 'HEDGE_MARKET',
# MAGIC 'HEDGE_MOLECULE',
# MAGIC 'HEDGE_BRAND',
# MAGIC hedge_val,
# MAGIC hedge_vol,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 5
# MAGIC from tmp_hedge
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load data to tmp table
# MAGIC -- 1.5 load data from pack, balance level_brand
# MAGIC -- add regional data that is included in the pack but not in the brand
# MAGIC -------------------------------------------------------------------------------------
# MAGIC with tmp_pack_to_brand as (
# MAGIC select distinct a.quarter,a.level_ta,a.level_market, c.cmps_des_c,c.prod_des_c,a.product_id
# MAGIC from tmp.tmp_retail_pack_rawdata a
# MAGIC inner join dwd.dwd_gnd_ext_retail_pack_property c
# MAGIC on a.product_id = c.product_id
# MAGIC where exists (
# MAGIC select distinct b.level_ta, b.level_market
# MAGIC from dwd.dwd_gnd_retail_split_automatic b
# MAGIC where b.level_ta is not null
# MAGIC and a.level_ta = b.level_ta
# MAGIC and a.level_market = b.level_market
# MAGIC ) and a.data_flag = 0
# MAGIC and a.zk_regin <> '全国'
# MAGIC order by a.quarter,a.level_ta,a.level_market, c.cmps_des_c,c.prod_des_c
# MAGIC ), tmp_pack_sum as (
# MAGIC select
# MAGIC a.quarter,
# MAGIC a.zk_regin,
# MAGIC a.level_ta,
# MAGIC a.level_market,
# MAGIC b.cmps_des_c as molecule_desc,
# MAGIC nvl(b.prod_des_c,concat(b.cmps_des_c,'_OTHER_PROD')) as product_desc,
# MAGIC sum(a.sales_value) as sales_val,
# MAGIC round(sum(a.sales_unit * (c.counting_unit/ coalesce(c.unit,1)) ) ,4) as sales_vol
# MAGIC from tmp.tmp_retail_pack_rawdata a
# MAGIC inner join tmp_pack_to_brand b
# MAGIC on a.quarter = b.quarter
# MAGIC and a.level_ta = b.level_ta
# MAGIC and a.level_market = b.level_market
# MAGIC and a.product_id = b.product_id
# MAGIC inner join dwd.dwd_gnd_ext_retail_pack_property c
# MAGIC on a.product_id = c.product_id
# MAGIC where a.data_flag = 0 and zk_regin <> 'ROC'
# MAGIC group by a.quarter, a.zk_regin,a.level_ta,a.level_market,b.cmps_des_c,b.prod_des_c
# MAGIC order by a.quarter,a.zk_regin,a.level_ta
# MAGIC ), tmp_brand as (
# MAGIC select * from tmp.tmp_retail_level_brand where data_flag = 0
# MAGIC ),tmp_final as (
# MAGIC select
# MAGIC a.*,
# MAGIC null as ratio_val,
# MAGIC null as ratio_vol,
# MAGIC null as total_val,
# MAGIC null as total_vol,
# MAGIC 0 as data_flag --从pack聚合出来的brand数据，也是真实的原生数据
# MAGIC from tmp_pack_sum a
# MAGIC where not exists (
# MAGIC select 1 from tmp_brand b
# MAGIC where a.quarter = b.quarter
# MAGIC and a.zk_regin = b.province_city
# MAGIC and a.level_ta = b.ta
# MAGIC and a.level_market = b.market
# MAGIC and a.molecule_desc = b.molecule
# MAGIC and a.product_desc = b.brand
# MAGIC )
# MAGIC )
# MAGIC
# MAGIC insert into table tmp.tmp_retail_level_brand
# MAGIC select * from tmp_final
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-2: substract data from level market to brand
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-2: substract data from level market to brand
# MAGIC -- 2.1 substract level market data and add 'OTHER_MOLECULE' data
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC with tmp_aggregate_market as (
# MAGIC select
# MAGIC quarter,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC sum(sales_val) as sales_val,
# MAGIC sum(sales_vol) as sales_vol
# MAGIC from tmp.tmp_retail_level_molecule
# MAGIC group by quarter, province_city , ta ,market
# MAGIC ),tmp_molecule as (
# MAGIC select
# MAGIC a.quarter,
# MAGIC a.province_city,
# MAGIC a.ta,
# MAGIC a.market,
# MAGIC a.sales_val - ifnull(b.sales_val, 0) as sales_val,
# MAGIC a.sales_vol - ifnull(b.sales_vol, 0) as sales_vol
# MAGIC from tmp.tmp_retail_level_market a
# MAGIC left join tmp_aggregate_market b
# MAGIC on a.quarter = b.quarter
# MAGIC and a.province_city = b.province_city
# MAGIC and a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC )
# MAGIC
# MAGIC -------append OTHER_MOLECULE to tmp.tmp_retail_level_molecule
# MAGIC insert into tmp.tmp_retail_level_molecule
# MAGIC select
# MAGIC quarter,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC 'OTHER_MOLECULE',
# MAGIC sales_val,
# MAGIC sales_vol,
# MAGIC 1
# MAGIC from tmp_molecule
# MAGIC where sales_val > 0 or sales_vol > 0
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-2: substract data from level market to brand
# MAGIC -- 2.2 substract level molecule data and add 'OTHER_BRAND' data
# MAGIC -------------------------------------------------------------------------------------
# MAGIC with tmp_aggregate_molecule as (
# MAGIC select
# MAGIC quarter,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC molecule,
# MAGIC sum(sales_val) as sales_val,
# MAGIC sum(sales_vol) as sales_vol
# MAGIC from tmp.tmp_retail_level_brand
# MAGIC group by quarter,province_city ,ta ,market,molecule
# MAGIC ),tmp_brand as (
# MAGIC select
# MAGIC a.quarter,
# MAGIC a.province_city,
# MAGIC a.ta,
# MAGIC a.market,
# MAGIC a.molecule,
# MAGIC a.sales_val - ifnull(b.sales_val, 0) as sales_val,
# MAGIC a.sales_vol - ifnull(b.sales_vol, 0) as sales_vol
# MAGIC from tmp.tmp_retail_level_molecule a
# MAGIC left join tmp_aggregate_molecule b
# MAGIC on a.quarter = b.quarter
# MAGIC and a.province_city = b.province_city
# MAGIC and a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.molecule = b.molecule
# MAGIC )
# MAGIC
# MAGIC -------append OTHER_BRAND to tmp.tmp_retail_level_brand
# MAGIC insert into tmp.tmp_retail_level_brand
# MAGIC select
# MAGIC quarter,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC molecule,
# MAGIC 'OTHER_BRAND',
# MAGIC sales_val,
# MAGIC sales_vol,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 1
# MAGIC from tmp_brand
# MAGIC where sales_val > 0 or sales_vol > 0
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-3: accumulate data from level brand to market
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-3: accumulate data from level brand to market
# MAGIC -- 3.1 accumulate level molecule data and balance 'OTHER_BRAND' & 'OTHER_MOLECULE' data
# MAGIC -- 3.1.1 accumulate level molecule data
# MAGIC -------------------------------------------------------------------------------------
# MAGIC with tmp_aggregate_molecule as (
# MAGIC select
# MAGIC quarter,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC molecule,
# MAGIC sum(sales_val) as sales_val,
# MAGIC sum(sales_vol) as sales_vol
# MAGIC from tmp.tmp_retail_level_brand
# MAGIC group by quarter,province_city ,ta ,market,molecule
# MAGIC ), tmp_append_molecule as (
# MAGIC select a.*
# MAGIC from tmp_aggregate_molecule a
# MAGIC where not exists (
# MAGIC select 1
# MAGIC from tmp.tmp_retail_level_molecule b
# MAGIC where a.quarter = b.quarter
# MAGIC and a.province_city = b.province_city
# MAGIC and a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.molecule = b.molecule
# MAGIC )
# MAGIC )
# MAGIC -------append level molecule data accumulate from level brand
# MAGIC insert into tmp.tmp_retail_level_molecule
# MAGIC select
# MAGIC quarter,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC molecule,
# MAGIC sales_val,
# MAGIC sales_vol,
# MAGIC 2
# MAGIC from tmp_append_molecule
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-3: accumulate data from level brand to market
# MAGIC -- 3.1 accumulate level molecule data and balance 'OTHER_BRAND'& 'OTHER_MOLECULE' data
# MAGIC -- 3.1.2 balance 'OTHER_MOLECULE' data
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC merge into tmp.tmp_retail_level_molecule a
# MAGIC using(
# MAGIC select * from tmp.tmp_retail_level_molecule
# MAGIC where data_flag = 2
# MAGIC ) as b
# MAGIC on a.quarter = b.quarter
# MAGIC and a.province_city = b.province_city
# MAGIC and a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.molecule = 'OTHER_MOLECULE'
# MAGIC and a.data_flag = 1
# MAGIC when matched then
# MAGIC update set
# MAGIC a.sales_val = a.sales_val - b.sales_val,
# MAGIC a.sales_vol = a.sales_vol - b.sales_vol
# MAGIC
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-3: accumulate data from level brand to market
# MAGIC -- 3.1 accumulate level molecule data and balance 'OTHER_BRAND'& 'OTHER_MOLECULE' data
# MAGIC -- 3.1.3 balance 'OTHER_BRAND' data
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC merge into tmp.tmp_retail_level_brand a
# MAGIC using(
# MAGIC select * from tmp.tmp_retail_level_molecule
# MAGIC where data_flag = 2
# MAGIC ) as b
# MAGIC on a.quarter = b.quarter
# MAGIC and a.province_city = b.province_city
# MAGIC and a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.molecule = 'OTHER_MOLECULE'
# MAGIC and a.brand = 'OTHER_BRAND'
# MAGIC and a.data_flag = 1
# MAGIC when matched then
# MAGIC update set
# MAGIC a.sales_val = a.sales_val - b.sales_val,
# MAGIC a.sales_vol = a.sales_vol - b.sales_vol
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-3: accumulate data from level brand to market
# MAGIC -- 3.2 accumulate level market data and balance 'OTHER_MOLECULE' data
# MAGIC -- 3.2.1 accumulate level market data
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC with tmp_aggregate_market as (
# MAGIC select
# MAGIC quarter,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC sum(sales_val) as sales_val,
# MAGIC sum(sales_vol) as sales_vol
# MAGIC from tmp.tmp_retail_level_molecule
# MAGIC group by quarter,province_city ,ta ,market
# MAGIC ), tmp_append_market as (
# MAGIC select a.*
# MAGIC from tmp_aggregate_market a
# MAGIC where not exists (
# MAGIC select 1
# MAGIC from tmp.tmp_retail_level_market b
# MAGIC where a.quarter = b.quarter
# MAGIC and a.province_city = b.province_city
# MAGIC and a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC )
# MAGIC )
# MAGIC -------append level market data accumulate from level molecule
# MAGIC insert into tmp.tmp_retail_level_market
# MAGIC select
# MAGIC quarter,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC sales_val,
# MAGIC sales_vol,
# MAGIC 2
# MAGIC from tmp_append_market
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-3: accumulate data from level brand to market
# MAGIC -- 3.2 accumulate level market data and balance 'OTHER_MOLECULE' data
# MAGIC -- 3.2.2 balance 'OTHER_MOLECULE' data
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC ---------not using in this case, beacuse level ta has no data------------------------
# MAGIC --merge into tmp.tmp_retail_level_market a
# MAGIC --using(
# MAGIC -- select * from tmp.tmp_retail_level_molecule
# MAGIC -- where data_flag = 2
# MAGIC --) as b
# MAGIC --on a.quarter = b.quarter
# MAGIC --and a.province_city = b.province_city
# MAGIC --and a.ta = b.ta
# MAGIC --and a.market = 'OTHER_MARKET'
# MAGIC --and a.data_flag = 1
# MAGIC --when matched then
# MAGIC -- update set
# MAGIC -- a.sales_val = a.sales_val - b.sales_val,
# MAGIC -- a.sales_vol = a.sales_vol - b.sales_vol
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-4: calculate brand ratio
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-4: calculate brand ratio
# MAGIC -- update total_val&total_vol and ratio_val&ratio_vol
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC merge into tmp.tmp_retail_level_brand a
# MAGIC using(
# MAGIC select * from tmp.tmp_retail_level_brand
# MAGIC where province_city = '全国'
# MAGIC ) as b
# MAGIC on a.quarter = b.quarter
# MAGIC and a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.molecule = b.molecule
# MAGIC and a.brand = b.brand
# MAGIC when matched then
# MAGIC update set
# MAGIC a.total_val = b.sales_val,
# MAGIC a.total_vol = b.sales_vol,
# MAGIC a.ratio_val = round(a.sales_val/b.sales_val,10),
# MAGIC a.ratio_vol = round(a.sales_vol/b.sales_vol,10)
# COMMAND ----------
############################################################END##############################################################

1200
Retail/03 split_pack_data.py Normal file
View File

File diff suppressed because it is too large Load Diff

209
Retail/04 map_to_dws_table.py Normal file
View File

@@ -0,0 +1,209 @@
# Databricks notebook source
############################################################START##############################################################
### STEP-1: insert splited pack data into tmp final table: tmp_retail_final_sales
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: insert splited pack data into tmp final table
# MAGIC -- insert into tmp_retail_final_sales
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC with tmp_pack as (
# MAGIC select
# MAGIC ------------------------------------------------------
# MAGIC -- 有月度数据使用月度数据，无月度数据用季度数据去转
# MAGIC nvl(
# MAGIC a.month,
# MAGIC CONCAT(
# MAGIC SUBSTRING(a.quarter, 1, 4), -- 提取年份（前4位）
# MAGIC CASE
# MAGIC WHEN SUBSTRING(a.quarter, 6, 1) = '1' THEN '03' -- Q1 → 03月
# MAGIC WHEN SUBSTRING(a.quarter, 6, 1) = '2' THEN '06' -- Q2 → 06月
# MAGIC WHEN SUBSTRING(a.quarter, 6, 1) = '3' THEN '09' -- Q3 → 09月
# MAGIC WHEN SUBSTRING(a.quarter, 6, 1) = '4' THEN '12' -- Q4 → 12月
# MAGIC END
# MAGIC )
# MAGIC ) as YYYYMM,
# MAGIC ------------------------------------------------------
# MAGIC a.pack_code as iqvia_pack_code,
# MAGIC a.product_id as zk_product_id,
# MAGIC case when a.product_desc <> 'others' then a.product_desc else null end as prod_des_c,
# MAGIC case when a.product_desc <> 'others' then a.product_desc else concat('Others_', a.molecule_desc) end as PROD_MAPPING,
# MAGIC a.zk_regin as province_city,
# MAGIC a.level_market as market,
# MAGIC a.sales_value,
# MAGIC a.sales_unit,
# MAGIC ------------------------------------------------------
# MAGIC -- counting_unit取值逻辑：
# MAGIC -- 不能直接取原始pack文件表中的值，改为取pack_property表中counting_unit / unit的值
# MAGIC a.sales_unit * (b.counting_unit/ coalesce(b.unit,1)) as counting_unit,
# MAGIC ------------------------------------------------------
# MAGIC case when data_flag = 0 then 1 else 2 end as pack_flag,
# MAGIC case when brand_flag = 1 then 1 else 2 end as brand_flag
# MAGIC from tmp.tmp_retail_pack_rawdata a
# MAGIC left join dwd.dwd_gnd_ext_retail_pack_property b
# MAGIC on a.product_id = b.product_id
# MAGIC ), tmp_has_roc as (
# MAGIC select
# MAGIC product_id,
# MAGIC quarter,
# MAGIC ------------------------------------------------------
# MAGIC -- 有月度数据使用月度数据，无月度数据用季度数据去转
# MAGIC nvl(
# MAGIC month,
# MAGIC CONCAT(
# MAGIC SUBSTRING(quarter, 1, 4), -- 提取年份（前4位）
# MAGIC CASE
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '1' THEN '03' -- Q1 → 03月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '2' THEN '06' -- Q2 → 06月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '3' THEN '09' -- Q3 → 09月
# MAGIC WHEN SUBSTRING(quarter, 6, 1) = '4' THEN '12' -- Q4 → 12月
# MAGIC END
# MAGIC )
# MAGIC ) as month,
# MAGIC ------------------------------------------------------
# MAGIC pack_code
# MAGIC from tmp.tmp_retail_pack_rawdata
# MAGIC where zk_regin = 'ROC'
# MAGIC ), tmp_pack_this_year_with_roc as (
# MAGIC select
# MAGIC *
# MAGIC from tmp_pack a
# MAGIC where exists(
# MAGIC select * from tmp_has_roc b
# MAGIC where a.YYYYMM = b.month
# MAGIC and a.iqvia_pack_code = b.pack_code
# MAGIC and a.zk_product_id = b.product_id
# MAGIC ) and a.province_city <> '全国'
# MAGIC ), tmp_pack_next_year_with_roc as (
# MAGIC select
# MAGIC cast(YYYYMM + 100 as int) as YYYYMM,
# MAGIC iqvia_pack_code,
# MAGIC zk_product_id,
# MAGIC prod_des_c,
# MAGIC PROD_MAPPING,
# MAGIC province_city,
# MAGIC market,
# MAGIC sales_value as sales_value_ly,
# MAGIC sales_unit as sales_unit_ly,
# MAGIC counting_unit as counting_unit_ly,
# MAGIC pack_flag,
# MAGIC brand_flag
# MAGIC from tmp_pack a
# MAGIC where YYYYMM + 100 <= (select max(YYYYMM) from tmp_pack)
# MAGIC and exists(
# MAGIC select * from tmp_has_roc b
# MAGIC where a.YYYYMM = b.month
# MAGIC and a.iqvia_pack_code = b.pack_code
# MAGIC and a.zk_product_id = b.product_id
# MAGIC ) and a.province_city <> '全国'
# MAGIC
# MAGIC ), tmp_pack_this_year_without_roc as (
# MAGIC select
# MAGIC *
# MAGIC from tmp_pack a
# MAGIC where not exists(
# MAGIC select * from tmp_has_roc b
# MAGIC where a.YYYYMM = b.month
# MAGIC and a.iqvia_pack_code = b.pack_code
# MAGIC and a.zk_product_id = b.product_id
# MAGIC )
# MAGIC ), tmp_pack_next_year_without_roc as (
# MAGIC select
# MAGIC cast(YYYYMM + 100 as int) as YYYYMM,
# MAGIC iqvia_pack_code,
# MAGIC zk_product_id,
# MAGIC prod_des_c,
# MAGIC PROD_MAPPING,
# MAGIC province_city,
# MAGIC market,
# MAGIC sales_value as sales_value_ly,
# MAGIC sales_unit as sales_unit_ly,
# MAGIC counting_unit as counting_unit_ly,
# MAGIC pack_flag,
# MAGIC brand_flag
# MAGIC from tmp_pack a
# MAGIC where YYYYMM + 100 <= (select max(YYYYMM) from tmp_pack)
# MAGIC and not exists(
# MAGIC select * from tmp_has_roc b
# MAGIC where a.YYYYMM = b.month
# MAGIC and a.iqvia_pack_code = b.pack_code
# MAGIC and a.zk_product_id = b.product_id
# MAGIC )
# MAGIC ), tmp_final_sales as (
# MAGIC select
# MAGIC ifnull(a.yyyymm, b.yyyymm) as yyyymm,
# MAGIC ifnull(a.iqvia_pack_code, b.iqvia_pack_code) as iqvia_pack_code,
# MAGIC ifnull(a.zk_product_id, b.zk_product_id) as zk_product_id,
# MAGIC ifnull(a.prod_des_c, b.prod_des_c) as prod_des_c,
# MAGIC ifnull(a.PROD_MAPPING, b.PROD_MAPPING) as PROD_MAPPING,
# MAGIC ifnull(a.province_city, b.province_city) as province_city,
# MAGIC ifnull(a.market, b.market) as market,
# MAGIC ifnull(a.sales_value, 0) as sales_value,
# MAGIC ifnull(a.sales_unit, 0) as sales_unit,
# MAGIC ifnull(a.counting_unit, 0) as counting_unit,
# MAGIC ifnull(a.pack_flag, b.pack_flag) as pack_flag,
# MAGIC ifnull(a.brand_flag,b.brand_flag ) as brand_flag,
# MAGIC ifnull(b.sales_value_ly, 0) as sales_value_ly,
# MAGIC ifnull(b.sales_unit_ly, 0) as sales_unit_ly,
# MAGIC ifnull(b.counting_unit_ly, 0) as counting_unit_ly
# MAGIC from tmp_pack_this_year_with_roc a
# MAGIC full outer join tmp_pack_next_year_with_roc b
# MAGIC on a.YYYYMM = b.YYYYMM
# MAGIC and a.iqvia_pack_code = b.iqvia_pack_code
# MAGIC and a.zk_product_id = b.zk_product_id
# MAGIC and a.province_city = b.province_city
# MAGIC
# MAGIC union all
# MAGIC
# MAGIC select
# MAGIC ifnull(c.yyyymm, d.yyyymm) as yyyymm,
# MAGIC ifnull(c.iqvia_pack_code, d.iqvia_pack_code) as iqvia_pack_code,
# MAGIC ifnull(c.zk_product_id, d.zk_product_id) as zk_product_id,
# MAGIC ifnull(c.prod_des_c, d.prod_des_c) as prod_des_c,
# MAGIC ifnull(c.PROD_MAPPING, d.PROD_MAPPING) as PROD_MAPPING,
# MAGIC 'ROC' as province_city,
# MAGIC ifnull(c.market, d.market) as market,
# MAGIC ifnull(c.sales_value, 0) as sales_value,
# MAGIC ifnull(c.sales_unit, 0) as sales_unit,
# MAGIC ifnull(c.counting_unit, 0) as counting_unit,
# MAGIC --ifnull(c.pack_flag, d.pack_flag) as pack_flag,
# MAGIC 2 as pack_flag, -- 此类没有拆分比例，且pack只有全国的数，pack_flag固定为2
# MAGIC ifnull(c.brand_flag,d.brand_flag ) as brand_flag,
# MAGIC ifnull(d.sales_value_ly, 0) as sales_value_ly,
# MAGIC ifnull(d.sales_unit_ly, 0) as sales_unit_ly,
# MAGIC ifnull(d.counting_unit_ly, 0) as counting_unit_ly
# MAGIC from tmp_pack_this_year_without_roc c
# MAGIC full outer join tmp_pack_next_year_without_roc d
# MAGIC on c.YYYYMM = d.YYYYMM
# MAGIC and c.iqvia_pack_code = d.iqvia_pack_code
# MAGIC and c.zk_product_id = d.zk_product_id
# MAGIC and c.province_city = d.province_city
# MAGIC )
# MAGIC
# MAGIC insert overwrite table tmp.tmp_retail_final_sales
# MAGIC
# MAGIC select
# MAGIC yyyymm,
# MAGIC iqvia_pack_code,
# MAGIC zk_product_id,
# MAGIC prod_des_c,
# MAGIC PROD_MAPPING,
# MAGIC province_city,
# MAGIC market,
# MAGIC sales_value,
# MAGIC sales_value_ly,
# MAGIC sales_unit,
# MAGIC sales_unit_ly,
# MAGIC counting_unit,
# MAGIC counting_unit_ly,
# MAGIC pack_flag,
# MAGIC brand_flag
# MAGIC from tmp_final_sales
# MAGIC order by yyyymm
# COMMAND ----------
############################################################END################################################################

158
Retail/05 load_dtp_temp_data.py Normal file
View File

@@ -0,0 +1,158 @@
# Databricks notebook source
# MAGIC %sql
# MAGIC -- CREATE or REPLACE TABLE tmp.tmp_retail_dtp_pack_rawdata (
# MAGIC -- -- product_id STRING,
# MAGIC -- iqvia_pack_code string,
# MAGIC -- -- region_type STRING,
# MAGIC -- year STRING,
# MAGIC -- time STRING,
# MAGIC -- -- higher_level_region STRING,
# MAGIC -- region STRING,
# MAGIC -- -- prescription_nature STRING,
# MAGIC -- -- medicine_attribute STRING,
# MAGIC -- -- dosage_form STRING,
# MAGIC -- -- object STRING,
# MAGIC -- -- zk_classify1 STRING,
# MAGIC -- -- zk_classify2 STRING,
# MAGIC -- -- zk_classify3 STRING,
# MAGIC -- target_points STRING,
# MAGIC -- -- common_name STRING,
# MAGIC -- -- brand_name STRING,
# MAGIC -- -- product_name STRING,
# MAGIC -- -- category_name STRING,
# MAGIC -- -- pack_des STRING,
# MAGIC -- counting_unit DECIMAL(20,8),
# MAGIC -- -- factory STRING,
# MAGIC -- -- corp_des STRING,
# MAGIC -- average_price DECIMAL(20,8),
# MAGIC -- sales_amount DECIMAL(20,8),
# MAGIC -- sales_volume DECIMAL(20,8),
# MAGIC -- counting_units_obversion DECIMAL(20,8),
# MAGIC -- pack_code STRING,
# MAGIC -- molecule_code STRING,
# MAGIC -- molecule_desc STRING,
# MAGIC -- product_code STRING,
# MAGIC -- product_desc STRING,
# MAGIC -- level_ta STRING,
# MAGIC -- level_market STRING,
# MAGIC -- level_molecule STRING,
# MAGIC -- level_brand STRING,
# MAGIC -- ratio_val DECIMAL(20,10),
# MAGIC -- ratio_vol DECIMAL(20,10),
# MAGIC -- data_flag INT,
# MAGIC -- brand_flag INT)
# MAGIC -- USING delta
# MAGIC -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/TMP/tmp_retail_dtp_pack_rawdata'
# MAGIC -- ;
# COMMAND ----------
### constant
LEVEL_TA_ONC = 'ONC'
LEVEL_MARKET_EGFR_TKI = 'EGFR TKI'
# COMMAND ----------
############################################################START##############################################################
### STEP-2: load rawdata to tmp table
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-1: load rawdata to tmp table
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load rawdata to tmp table
# MAGIC -- 1.1 load brand data to tmp_retail_dtp_brand_rawdata
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert overwrite table tmp.tmp_retail_dtp_brand_rawdata
# MAGIC
# MAGIC select
# MAGIC type,
# MAGIC 'ONC' as ta,
# MAGIC market,
# MAGIC zk_brand_category,
# MAGIC zk_common_name,
# MAGIC zk_manu_des,
# MAGIC rc_name_en,
# MAGIC province_city,
# MAGIC yyyymm,
# MAGIC ytd,
# MAGIC sales_value,
# MAGIC sales_volume,
# MAGIC price,
# MAGIC pdot_counting_unit
# MAGIC from dwd.dwd_gnd_ext_dtp_zk_brand
# MAGIC where ranked_by = 'value'
# MAGIC
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load rawdata to tmp table
# MAGIC -- 1.2 load brand data to tmp_retail_dtp_pack_rawdata
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert overwrite table tmp.tmp_retail_dtp_pack_rawdata
# MAGIC
# MAGIC select
# MAGIC -- dtp_raw_data.product_id,
# MAGIC dtp_pack_data.iqvia_pack_code,
# MAGIC -- dtp_raw_data.region_type,
# MAGIC dtp_raw_data.year,
# MAGIC dtp_raw_data.time,
# MAGIC -- dtp_raw_data.higher_level_region,
# MAGIC dtp_raw_data.region,
# MAGIC -- dtp_raw_data.prescription_nature,
# MAGIC -- dtp_raw_data.medicine_attribute,
# MAGIC -- dtp_raw_data.dosage_form,
# MAGIC -- dtp_raw_data.object,
# MAGIC -- dtp_raw_data.zk_classify1,
# MAGIC -- dtp_raw_data.zk_classify2,
# MAGIC -- dtp_raw_data.zk_classify3,
# MAGIC dtp_raw_data.target_points,
# MAGIC -- dtp_raw_data.common_name,
# MAGIC -- dtp_raw_data.brand_name,
# MAGIC -- dtp_raw_data.product_name,
# MAGIC -- dtp_raw_data.category_name,
# MAGIC -------------------------* hard_code *-------------------------
# MAGIC -- 针对pack_des这个字段，pack原始表使用的是'*'，但手工表pack_properity使用的是'x'
# MAGIC -- 而pack_des字段与关联出pack_code有关，缺少此字段会导致关联失败，所以要手动消除差异
# MAGIC -- replace(dtp_raw_data.pack_des,'*','x'),
# MAGIC -------------------------* hard_code *-------------------------
# MAGIC max(dtp_raw_data.counting_unit) counting_unit,
# MAGIC -- dtp_raw_data.factory,
# MAGIC -- dtp_raw_data.corp_des,
# MAGIC max(dtp_raw_data.average_price) average_price,
# MAGIC sum(dtp_raw_data.sales_amount) sales_amount,
# MAGIC sum(dtp_raw_data.sales_volume) sales_volume,
# MAGIC max(dtp_raw_data.counting_units_obversion) counting_units_obversion,
# MAGIC null as pack_code,
# MAGIC null as molecule_code,
# MAGIC null as molecule_desc,
# MAGIC null as product_code,
# MAGIC null as product_desc,
# MAGIC 'ONC' as level_ta,
# MAGIC null as level_market, --- 此pack原始表文件中存在多个市场，后续通过关联更新
# MAGIC null as level_molecule,
# MAGIC null as level_brand,
# MAGIC null as ratio_val,
# MAGIC null as ratio_vol,
# MAGIC 0 as data_flag,
# MAGIC null as brand_flag
# MAGIC from dwd.dwd_gnd_ext_retail_dtp_datasource as dtp_raw_data
# MAGIC left join dwd.dwd_gnd_ext_dtp_pack_property as dtp_pack_data
# MAGIC on dtp_raw_data.product_id = dtp_pack_data.product_id
# MAGIC group by
# MAGIC dtp_pack_data.iqvia_pack_code,
# MAGIC dtp_raw_data.year,
# MAGIC dtp_raw_data.time,
# MAGIC dtp_raw_data.region,
# MAGIC dtp_raw_data.target_points
# COMMAND ----------
############################################################END##############################################################

421
Retail/06 split_dtp_brand_data.py Normal file
View File

@@ -0,0 +1,421 @@
# Databricks notebook source
### constant
DATA_TYPE_MARKET = 'MARKET'
DATA_TYPE_MOLECULE = 'MOLECULE'
DATA_TYPE_BRAND = 'BRAND'
DATA_TYPE_MULTI_MARKET = 'MULTI_MARKET'
DATA_TYPE_MULTI_MOLECULE = 'MULTI_MOLECULE'
DATA_TYPE_MULTI_BRAND = 'MULTI_BRAND'
DATA_TYPE_DUPLICATE = 'DUPLICATE'
DATA_TYPE_HEDGE_TA = 'HEDGE_TA'
DATA_TYPE_HEDGE_MARKET = 'HEDGE_MARKET'
DATA_TYPE_HEDGE_MOLECULE = 'HEDGE_MOLECULE'
DATA_TYPE_HEDGE_BRAND = 'HEDGE_BRAND'
DATA_FLAG_RAW = 0
DATA_FLAG_SUB = 1
DATA_FLAG_ADD = 2
DATA_FLAG_RATIO = 3
DATA_FLAG_AVG = 4
DATA_FLAG_HEDGE = 5
DATA_OTHER_TA = 'OTHER_TA'
DATA_OTHER_MARKET = 'OTHER_MARKET'
DATA_OTHER_MOLECULE = 'OTHER_MOLECULE'
DATA_OTHER_BRAND = 'OTHER_BRAND'
DATA_AREA_TYPE_ROC = 'ROC'
DATA_LABEL_AREA_ALL = '全国'
# COMMAND ----------
############################################################START##############################################################
### STEP-1: load data to tmp table
### STEP-2: substract data from level market to brand
### STEP-3: accumulate data from level brand to market
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-1: load data to tmp table
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load data to tmp table
# MAGIC -- 1.1 load LEVEL_MARKET data to tmp.tmp_retail_dtp_level_market
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert overwrite table tmp.tmp_retail_dtp_level_market
# MAGIC
# MAGIC select
# MAGIC a.yyyymm,
# MAGIC a.province_city,
# MAGIC b.level_ta,
# MAGIC b.level_market,
# MAGIC a.sales_val,
# MAGIC a.sales_vol,
# MAGIC 0
# MAGIC from tmp.tmp_retail_dtp_brand_rawdata a
# MAGIC inner join dwd.dwd_gnd_retail_split_automatic b
# MAGIC on a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.brand_category = b.brand
# MAGIC and ifnull(a.common_name, '') = ifnull(b.common_name, '')
# MAGIC where b.level = 'MARKET'
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load data to tmp table
# MAGIC -- 1.2 load LEVEL_MOLECULE data to tmp.tmp_retail_dtp_level_molecule
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert overwrite table tmp.tmp_retail_dtp_level_molecule
# MAGIC
# MAGIC select
# MAGIC a.yyyymm,
# MAGIC a.province_city,
# MAGIC b.level_ta,
# MAGIC b.level_market,
# MAGIC b.level_molecule,
# MAGIC a.sales_val,
# MAGIC a.sales_vol,
# MAGIC 0
# MAGIC from tmp.tmp_retail_dtp_brand_rawdata a
# MAGIC inner join dwd.dwd_gnd_retail_split_automatic b
# MAGIC on a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.brand_category = b.brand
# MAGIC and ifnull(a.common_name, '') = ifnull(b.common_name, '')
# MAGIC where b.level = 'MOLECULE'
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: load data to tmp table
# MAGIC -- 1.3 load LEVEL_BRAND data to tmp.tmp_retail_dtp_level_brand
# MAGIC -------------------------------------------------------------------------------------
# MAGIC insert overwrite table tmp.tmp_retail_dtp_level_brand
# MAGIC
# MAGIC select
# MAGIC a.yyyymm,
# MAGIC a.province_city,
# MAGIC b.level_ta,
# MAGIC b.level_market,
# MAGIC b.level_molecule,
# MAGIC b.level_brand,
# MAGIC a.sales_val,
# MAGIC a.sales_vol,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 0
# MAGIC from tmp.tmp_retail_dtp_brand_rawdata a
# MAGIC inner join dwd.dwd_gnd_retail_split_automatic b
# MAGIC on a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.brand_category = b.brand
# MAGIC and ifnull(a.common_name, '') = ifnull(b.common_name, '')
# MAGIC where b.level = 'BRAND'
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-2: substract data from level market to brand
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-2: substract data from level market to brand
# MAGIC -- 2.1 substract level market data and add 'OTHER_MOLECULE' data
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC with tmp_aggregate_market as (
# MAGIC select
# MAGIC yyyymm,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC sum(sales_val) as sales_val,
# MAGIC sum(sales_vol) as sales_vol
# MAGIC from tmp.tmp_retail_dtp_level_molecule
# MAGIC group by yyyymm, province_city , ta ,market
# MAGIC ),tmp_molecule as (
# MAGIC select
# MAGIC a.yyyymm,
# MAGIC a.province_city,
# MAGIC a.ta,
# MAGIC a.market,
# MAGIC a.sales_val - ifnull(b.sales_val, 0) as sales_val,
# MAGIC a.sales_vol - ifnull(b.sales_vol, 0) as sales_vol
# MAGIC from tmp.tmp_retail_dtp_level_market a
# MAGIC left join tmp_aggregate_market b
# MAGIC on a.yyyymm = b.yyyymm
# MAGIC and a.province_city = b.province_city
# MAGIC and a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC )
# MAGIC
# MAGIC -------append OTHER_MOLECULE to tmp.tmp_retail_dtp_level_molecule
# MAGIC insert into tmp.tmp_retail_dtp_level_molecule
# MAGIC select
# MAGIC yyyymm,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC 'OTHER_MOLECULE',
# MAGIC sales_val,
# MAGIC sales_vol,
# MAGIC 1
# MAGIC from tmp_molecule
# MAGIC where sales_val > 0 or sales_vol > 0
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-2: substract data from level market to brand
# MAGIC -- 2.2 substract level molecule data and add 'OTHER_BRAND' data
# MAGIC -------------------------------------------------------------------------------------
# MAGIC with tmp_aggregate_molecule as (
# MAGIC select
# MAGIC yyyymm,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC molecule,
# MAGIC sum(sales_val) as sales_val,
# MAGIC sum(sales_vol) as sales_vol
# MAGIC from tmp.tmp_retail_dtp_level_brand
# MAGIC group by yyyymm,province_city ,ta ,market,molecule
# MAGIC ),tmp_brand as (
# MAGIC select
# MAGIC a.yyyymm,
# MAGIC a.province_city,
# MAGIC a.ta,
# MAGIC a.market,
# MAGIC a.molecule,
# MAGIC a.sales_val - ifnull(b.sales_val, 0) as sales_val,
# MAGIC a.sales_vol - ifnull(b.sales_vol, 0) as sales_vol
# MAGIC from tmp.tmp_retail_dtp_level_molecule a
# MAGIC left join tmp_aggregate_molecule b
# MAGIC on a.yyyymm = b.yyyymm
# MAGIC and a.province_city = b.province_city
# MAGIC and a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.molecule = b.molecule
# MAGIC )
# MAGIC
# MAGIC -------append OTHER_BRAND to tmp.tmp_retail_dtp_level_brand
# MAGIC insert into tmp.tmp_retail_dtp_level_brand
# MAGIC select
# MAGIC yyyymm,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC molecule,
# MAGIC 'OTHER_BRAND',
# MAGIC sales_val,
# MAGIC sales_vol,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC null,
# MAGIC 1
# MAGIC from tmp_brand
# MAGIC where sales_val > 0 or sales_vol > 0
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-3: accumulate data from level brand to market
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-3: accumulate data from level brand to market
# MAGIC -- 3.1 accumulate level molecule data and balance 'OTHER_BRAND' & 'OTHER_MOLECULE' data
# MAGIC -- 3.1.1 accumulate level molecule data
# MAGIC -------------------------------------------------------------------------------------
# MAGIC with tmp_aggregate_molecule as (
# MAGIC select
# MAGIC yyyymm,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC molecule,
# MAGIC sum(sales_val) as sales_val,
# MAGIC sum(sales_vol) as sales_vol
# MAGIC from tmp.tmp_retail_dtp_level_brand
# MAGIC group by yyyymm,province_city ,ta ,market,molecule
# MAGIC ), tmp_append_molecule as (
# MAGIC select a.*
# MAGIC from tmp_aggregate_molecule a
# MAGIC where not exists (
# MAGIC select 1
# MAGIC from tmp.tmp_retail_dtp_level_molecule b
# MAGIC where a.yyyymm = b.yyyymm
# MAGIC and a.province_city = b.province_city
# MAGIC and a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.molecule = b.molecule
# MAGIC )
# MAGIC )
# MAGIC -------append level molecule data accumulate from level brand
# MAGIC insert into tmp.tmp_retail_dtp_level_molecule
# MAGIC select
# MAGIC yyyymm,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC molecule,
# MAGIC sales_val,
# MAGIC sales_vol,
# MAGIC 2
# MAGIC from tmp_append_molecule
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-3: accumulate data from level brand to market
# MAGIC -- 3.1 accumulate level molecule data and balance 'OTHER_BRAND'& 'OTHER_MOLECULE' data
# MAGIC -- 3.1.2 balance 'OTHER_MOLECULE' data
# MAGIC -------------------------------------------------------------------------------------
# MAGIC merge into tmp.tmp_retail_dtp_level_molecule a
# MAGIC using(
# MAGIC select
# MAGIC yyyymm,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC sum(sales_val) as sales_val,
# MAGIC sum(sales_vol) as sales_vol
# MAGIC from tmp.tmp_retail_dtp_level_molecule
# MAGIC where data_flag = 2
# MAGIC group by yyyymm,province_city,ta,market
# MAGIC ) as b
# MAGIC on a.yyyymm = b.yyyymm
# MAGIC and a.province_city = b.province_city
# MAGIC and a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.molecule = 'OTHER_MOLECULE'
# MAGIC and a.data_flag = 1
# MAGIC when matched then
# MAGIC update set
# MAGIC a.sales_val = a.sales_val - b.sales_val,
# MAGIC a.sales_vol = a.sales_vol - b.sales_vol
# MAGIC
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-3: accumulate data from level brand to market
# MAGIC -- 3.1 accumulate level molecule data and balance 'OTHER_BRAND'& 'OTHER_MOLECULE' data
# MAGIC -- 3.1.3 balance 'OTHER_BRAND' data
# MAGIC -------------------------------------------------------------------------------------
# MAGIC merge into tmp.tmp_retail_dtp_level_brand a
# MAGIC using(
# MAGIC select
# MAGIC yyyymm,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC sum(sales_val) as sales_val,
# MAGIC sum(sales_vol) as sales_vol
# MAGIC from tmp.tmp_retail_dtp_level_molecule
# MAGIC where data_flag = 2
# MAGIC group by yyyymm,province_city,ta,market
# MAGIC ) as b
# MAGIC on a.yyyymm = b.yyyymm
# MAGIC and a.province_city = b.province_city
# MAGIC and a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.molecule = 'OTHER_MOLECULE'
# MAGIC and a.brand = 'OTHER_BRAND'
# MAGIC and a.data_flag = 1
# MAGIC when matched then
# MAGIC update set
# MAGIC a.sales_val = a.sales_val - b.sales_val,
# MAGIC a.sales_vol = a.sales_vol - b.sales_vol
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-3: accumulate data from level brand to market
# MAGIC -- 3.2 accumulate level market data and balance 'OTHER_MOLECULE' data
# MAGIC -- 3.2.1 accumulate level market data
# MAGIC -------------------------------------------------------------------------------------
# MAGIC
# MAGIC with tmp_aggregate_market as (
# MAGIC select
# MAGIC yyyymm,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC sum(sales_val) as sales_val,
# MAGIC sum(sales_vol) as sales_vol
# MAGIC from tmp.tmp_retail_dtp_level_molecule
# MAGIC group by yyyymm,province_city ,ta ,market
# MAGIC ), tmp_append_market as (
# MAGIC select a.*
# MAGIC from tmp_aggregate_market a
# MAGIC where not exists (
# MAGIC select 1
# MAGIC from tmp.tmp_retail_dtp_level_market b
# MAGIC where a.yyyymm = b.yyyymm
# MAGIC and a.province_city = b.province_city
# MAGIC and a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC )
# MAGIC )
# MAGIC -------append level market data accumulate from level molecule
# MAGIC insert into tmp.tmp_retail_dtp_level_market
# MAGIC select
# MAGIC yyyymm,
# MAGIC province_city,
# MAGIC ta,
# MAGIC market,
# MAGIC sales_val,
# MAGIC sales_vol,
# MAGIC 2
# MAGIC from tmp_append_market
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-4: calculate brand ratio
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-4: calculate brand ratio
# MAGIC -- update total_val&total_vol and ratio_val&ratio_vol
# MAGIC -------------------------------------------------------------------------------------
# MAGIC merge into tmp.tmp_retail_dtp_level_brand a
# MAGIC using(
# MAGIC select * from tmp.tmp_retail_dtp_level_brand
# MAGIC where province_city = '全国'
# MAGIC ) as b
# MAGIC on a.yyyymm = b.yyyymm
# MAGIC and a.ta = b.ta
# MAGIC and a.market = b.market
# MAGIC and a.molecule = b.molecule
# MAGIC and a.brand = b.brand
# MAGIC when matched then
# MAGIC update set
# MAGIC a.total_val = b.sales_val,
# MAGIC a.total_vol = b.sales_vol,
# MAGIC a.ratio_val = round(a.sales_val/b.sales_val,10),
# MAGIC a.ratio_vol = round(a.sales_vol/b.sales_vol,10)
# COMMAND ----------
############################################################END################################################################

1107
Retail/07 split_dtp_pack_data.py Normal file
View File

File diff suppressed because it is too large Load Diff

510
Retail/08 map_to_dtp_dws_table.py Normal file
View File

@@ -0,0 +1,510 @@
# Databricks notebook source
# MAGIC %sql
# MAGIC -- CREATE or REPLACE TABLE tmp.tmp_retail_dtp_final_sales (
# MAGIC -- year STRING,
# MAGIC -- yq STRING,
# MAGIC -- yyyymm STRING,
# MAGIC -- iqvia_pack_code STRING,
# MAGIC -- geo_key STRING,
# MAGIC -- count_unit DOUBLE,
# MAGIC -- average_price DOUBLE,
# MAGIC -- sales_value DECIMAL(20,10),
# MAGIC -- sales_unit DECIMAL(20,10),
# MAGIC -- counting_units_obversion DECIMAL(20,10),
# MAGIC -- counting_unit DECIMAL(20,10),
# MAGIC -- sales_value_ly DECIMAL(20,10),
# MAGIC -- sales_unit_ly DECIMAL(20,10),
# MAGIC -- counting_unit_ly DECIMAL(20,10),
# MAGIC -- pack_flag INT,
# MAGIC -- brand_flag INT)
# MAGIC -- USING delta
# MAGIC -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/TMP/tmp_retail_dtp_final_sales';
# COMMAND ----------
############################################################START##############################################################
### STEP-1: insert splited pack data into tmp final table: tmp_retail_final_dtp_sales
### STEP-2: calculate OTHERS data
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-1: insert splited pack data
# COMMAND ----------
# MAGIC %sql
# MAGIC -------------------------------------------------------------------------------------
# MAGIC -- STEP-1: insert splited pack data into tmp final table
# MAGIC -- insert into tmp_retail_dtp_final_sales (dws.dws_retail_dtp_sales)
# MAGIC -------------------------------------------------------------------------------------
# MAGIC with tmp_has_roc as (
# MAGIC select
# MAGIC pack_code,
# MAGIC time
# MAGIC from tmp.tmp_retail_dtp_pack_rawdata
# MAGIC where region = 'ROC'
# MAGIC ), temp_dtp_pack_property as (
# MAGIC select iqvia_pack_code,max(counting_unit) as counting_unit,max(unit) as unit
# MAGIC from dwd.dwd_gnd_ext_dtp_pack_property
# MAGIC group by iqvia_pack_code
# MAGIC ), tmp_pack as (
# MAGIC select
# MAGIC a.year,
# MAGIC a.time,
# MAGIC a.pack_code,
# MAGIC a.region,
# MAGIC a.counting_unit,
# MAGIC a.average_price,
# MAGIC a.sales_amount,
# MAGIC a.sales_volume,
# MAGIC a.counting_units_obversion,
# MAGIC ------------------------------------------------------
# MAGIC -- counting_unit取值逻辑：
# MAGIC -- 不能直接取原始pack文件表中的值，改为取pack_property表中counting_unit / unit的值
# MAGIC a.sales_volume * (b.counting_unit/ coalesce(b.unit,1)) as counting_unit_property,
# MAGIC ------------------------------------------------------
# MAGIC a.data_flag,
# MAGIC a.brand_flag,
# MAGIC -- a.prescription_nature,
# MAGIC -- a.medicine_attribute,
# MAGIC -- a.dosage_form,
# MAGIC -- a.object,
# MAGIC -- a.zk_classify1,
# MAGIC -- a.zk_classify2,
# MAGIC -- a.zk_classify3,
# MAGIC a.target_points
# MAGIC -- a.common_name,
# MAGIC -- a.brand_name,
# MAGIC -- a.product_name,
# MAGIC -- a.pack_des,
# MAGIC -- a.factory,
# MAGIC -- a.corp_des
# MAGIC from tmp.tmp_retail_dtp_pack_rawdata a
# MAGIC left join temp_dtp_pack_property b
# MAGIC ----------------------------------------------------
# MAGIC -- format iqvia_pack_code from dwd_gnd_ext_dtp_pack_property
# MAGIC -- on a.pack_code =
# MAGIC -- case when length(trim(b.iqvia_pack_code)) < 12 and trim(b.iqvia_pack_code) REGEXP '^[0-9]'
# MAGIC -- then right(concat('000000000000',trim(b.iqvia_pack_code)),12)
# MAGIC -- else trim(b.iqvia_pack_code)
# MAGIC -- end
# MAGIC -- ----------------------------------------------------
# MAGIC -- and nvl(a.prescription_nature,'') = nvl(b.prescription_nature,'')
# MAGIC -- and nvl(a.medicine_attribute,'') = nvl(b.medicine_attribute,'')
# MAGIC -- and nvl(a.dosage_form,'') = nvl(b.dosage_form,'')
# MAGIC -- and nvl(a.object,'') = nvl(b.object,'')
# MAGIC -- and nvl(a.zk_classify1,'') = nvl(b.zk_classify1,'')
# MAGIC -- and nvl(a.zk_classify2,'') = nvl(b.zk_classify2,'')
# MAGIC -- and nvl(a.zk_classify3,'') = nvl(b.zk_classify3,'')
# MAGIC -- and nvl(a.target_points,'') = nvl(b.target_points,'')
# MAGIC -- and nvl(a.common_name,'') = nvl(b.common_name,'')
# MAGIC -- and nvl(a.brand_name,'') = nvl(b.brand_name,'')
# MAGIC -- and nvl(a.product_name,'') = nvl(b.product_name,'')
# MAGIC -- and nvl(a.pack_des,'') = nvl(b.zk_pack_des,'')
# MAGIC -- and nvl(a.factory,'') = nvl(b.factory,'')
# MAGIC -- and nvl(a.corp_des,'') = nvl(b.zk_corp_des,'')
# MAGIC -- 20260226 不再使用上面的join条件,改为product_id关联
# MAGIC on a.iqvia_pack_code = b.iqvia_pack_code
# MAGIC ),tmp_pack_this_year_with_roc (
# MAGIC select
# MAGIC a.year,
# MAGIC concat(a.year, 'Q', CEIL(CAST(RIGHT(a.time,2) AS INT)/3)) as yq,
# MAGIC a.time as yyyymm,
# MAGIC a.pack_code as iqvia_pack_code,
# MAGIC a.region as geo_key,
# MAGIC a.counting_unit as count_unit,
# MAGIC a.average_price,
# MAGIC a.sales_amount as sales_value,
# MAGIC a.sales_volume as sales_unit,
# MAGIC a.counting_units_obversion,
# MAGIC a.counting_unit_property as counting_unit,
# MAGIC case when a.data_flag = 0 then 1 else 2 end as pack_flag,
# MAGIC case when a.brand_flag = 1 then 1 else 2 end as brand_flag,
# MAGIC -- a.prescription_nature,
# MAGIC -- a.medicine_attribute,
# MAGIC -- a.dosage_form,
# MAGIC -- a.object,
# MAGIC -- a.zk_classify1,
# MAGIC -- a.zk_classify2,
# MAGIC -- a.zk_classify3,
# MAGIC a.target_points
# MAGIC -- a.common_name,
# MAGIC -- a.brand_name,
# MAGIC -- a.product_name,
# MAGIC -- a.pack_des,
# MAGIC -- a.factory,
# MAGIC -- a.corp_des
# MAGIC from tmp_pack a
# MAGIC where exists (
# MAGIC select * from tmp_has_roc c
# MAGIC where c.pack_code = a.pack_code
# MAGIC and c.time = a.time
# MAGIC ) and a.region <> '全国'
# MAGIC ), tmp_pack_next_year_with_roc (
# MAGIC select
# MAGIC cast(a.year + 1 as int) as year,
# MAGIC concat(cast(a.year + 1 as int) , 'Q', CEIL(CAST(RIGHT(a.time,2) AS INT)/3)) as yq,
# MAGIC cast(a.time + 100 as int) as yyyymm,
# MAGIC a.pack_code as iqvia_pack_code,
# MAGIC a.region as geo_key,
# MAGIC a.counting_unit as count_unit,
# MAGIC a.average_price,
# MAGIC a.sales_amount as sales_value_ly,
# MAGIC a.sales_volume as sales_unit_ly,
# MAGIC a.counting_unit_property as counting_unit_ly,
# MAGIC a.counting_units_obversion,
# MAGIC case when a.data_flag = 0 then 1 else 2 end as pack_flag,
# MAGIC case when a.brand_flag = 1 then 1 else 2 end as brand_flag,
# MAGIC -- a.prescription_nature,
# MAGIC -- a.medicine_attribute,
# MAGIC -- a.dosage_form,
# MAGIC -- a.object,
# MAGIC -- a.zk_classify1,
# MAGIC -- a.zk_classify2,
# MAGIC -- a.zk_classify3,
# MAGIC a.target_points
# MAGIC -- a.common_name,
# MAGIC -- a.brand_name,
# MAGIC -- a.product_name,
# MAGIC -- a.pack_des,
# MAGIC -- a.factory,
# MAGIC -- a.corp_des
# MAGIC from tmp_pack a
# MAGIC where a.time + 100 <= (select max(time) from tmp_pack)
# MAGIC and exists (
# MAGIC select * from tmp_has_roc c
# MAGIC where c.pack_code = a.pack_code
# MAGIC and c.time = a.time
# MAGIC ) and a.region <> '全国'
# MAGIC ), tmp_pack_with_roc as (
# MAGIC select
# MAGIC ifnull(a.year, b.year) as year,
# MAGIC ifnull(a.yq, b.yq) as yq ,
# MAGIC ifnull(a.yyyymm, b.yyyymm) as yyyymm,
# MAGIC ifnull(a.iqvia_pack_code, b.iqvia_pack_code) as iqvia_pack_code,
# MAGIC ifnull(a.geo_key, b.geo_key) as geo_key,
# MAGIC ifnull(a.count_unit, b.count_unit) as count_unit,
# MAGIC ifnull(a.average_price, b.average_price) as average_price,
# MAGIC ifnull(a.sales_value, 0) as sales_value,
# MAGIC ifnull(a.sales_unit, 0) as sales_unit,
# MAGIC ifnull(a.counting_unit, 0) as counting_unit,
# MAGIC ifnull(a.counting_units_obversion, b.counting_units_obversion) as counting_units_obversion,
# MAGIC ifnull(a.pack_flag, b.pack_flag) as pack_flag,
# MAGIC ifnull(a.brand_flag, b.brand_flag) as brand_flag,
# MAGIC ifnull(b.sales_value_ly, 0) as sales_value_ly,
# MAGIC ifnull(b.sales_unit_ly, 0) as sales_unit_ly,
# MAGIC ifnull(b.counting_unit_ly, 0) as counting_unit_ly
# MAGIC from tmp_pack_this_year_with_roc a
# MAGIC full outer join tmp_pack_next_year_with_roc b
# MAGIC on a.yyyymm = b.yyyymm
# MAGIC and a.iqvia_pack_code = b.iqvia_pack_code
# MAGIC and a.geo_key = b.geo_key
# MAGIC -- and nvl(a.prescription_nature,'') = nvl(b.prescription_nature,'')
# MAGIC -- and nvl(a.medicine_attribute,'') = nvl(b.medicine_attribute,'')
# MAGIC -- and nvl(a.dosage_form,'') = nvl(b.dosage_form,'')
# MAGIC -- and nvl(a.object,'') = nvl(b.object,'')
# MAGIC -- and nvl(a.zk_classify1,'') = nvl(b.zk_classify1,'')
# MAGIC -- and nvl(a.zk_classify2,'') = nvl(b.zk_classify2,'')
# MAGIC -- and nvl(a.zk_classify3,'') = nvl(b.zk_classify3,'')
# MAGIC and nvl(a.target_points,'') = nvl(b.target_points,'')
# MAGIC -- and nvl(a.common_name,'') = nvl(b.common_name,'')
# MAGIC -- and nvl(a.brand_name,'') = nvl(b.brand_name,'')
# MAGIC -- and nvl(a.product_name,'') = nvl(b.product_name,'')
# MAGIC -- and nvl(a.pack_des,'') = nvl(b.pack_des,'')
# MAGIC -- and nvl(a.factory,'') = nvl(b.factory,'')
# MAGIC -- and nvl(a.corp_des,'') = nvl(b.corp_des,'')
# MAGIC ), tmp_pack_this_year_without_roc (
# MAGIC select
# MAGIC left(a.time, 4) year,
# MAGIC concat(a.year, 'Q', CEIL(CAST(RIGHT(a.time,2) AS INT)/3)) as yq,
# MAGIC a.time as yyyymm,
# MAGIC a.pack_code as iqvia_pack_code,
# MAGIC 'ROC' as geo_key,
# MAGIC a.counting_unit as count_unit,
# MAGIC a.average_price,
# MAGIC a.sales_amount as sales_value,
# MAGIC a.sales_volume as sales_unit,
# MAGIC a.counting_units_obversion,
# MAGIC a.counting_unit_property as counting_unit,
# MAGIC case when data_flag = 0 then 1 else 2 end as pack_flag,
# MAGIC case when brand_flag = 1 then 1 else 2 end as brand_flag,
# MAGIC -- a.prescription_nature,
# MAGIC -- a.medicine_attribute,
# MAGIC -- a.dosage_form,
# MAGIC -- a.object,
# MAGIC -- a.zk_classify1,
# MAGIC -- a.zk_classify2,
# MAGIC -- a.zk_classify3,
# MAGIC a.target_points
# MAGIC -- a.common_name,
# MAGIC -- a.brand_name,
# MAGIC -- a.product_name,
# MAGIC -- a.pack_des,
# MAGIC -- a.factory,
# MAGIC -- a.corp_des
# MAGIC from tmp_pack a
# MAGIC where not exists (
# MAGIC select * from tmp_has_roc c
# MAGIC where c.pack_code = a.pack_code
# MAGIC and c.time = a.time
# MAGIC )
# MAGIC ), tmp_pack_next_year_without_roc as (
# MAGIC select
# MAGIC cast(a.year + 1 as int) as year,
# MAGIC concat(cast(a.year + 1 as int) , 'Q', CEIL(CAST(RIGHT(a.time,2) AS INT)/3)) as yq,
# MAGIC cast(a.time + 100 as int) as yyyymm,
# MAGIC a.pack_code as iqvia_pack_code,
# MAGIC 'ROC' as geo_key,
# MAGIC a.counting_unit as count_unit,
# MAGIC a.average_price,
# MAGIC a.sales_amount as sales_value_ly,
# MAGIC a.sales_volume as sales_unit_ly,
# MAGIC a.counting_unit_property as counting_unit_ly,
# MAGIC a.counting_units_obversion,
# MAGIC case when data_flag = 0 then 1 else 2 end as pack_flag,
# MAGIC case when brand_flag = 1 then 1 else 2 end as brand_flag,
# MAGIC -- a.prescription_nature,
# MAGIC -- a.medicine_attribute,
# MAGIC -- a.dosage_form,
# MAGIC -- a.object,
# MAGIC -- a.zk_classify1,
# MAGIC -- a.zk_classify2,
# MAGIC -- a.zk_classify3,
# MAGIC a.target_points
# MAGIC -- a.common_name,
# MAGIC -- a.brand_name,
# MAGIC -- a.product_name,
# MAGIC -- a.pack_des,
# MAGIC -- a.factory,
# MAGIC -- a.corp_des
# MAGIC from tmp_pack a
# MAGIC where a.time + 100 <= (select max(time) from tmp_pack)
# MAGIC and not exists (
# MAGIC select * from tmp_has_roc c
# MAGIC where c.pack_code = a.pack_code
# MAGIC and c.time = a.time
# MAGIC )
# MAGIC ),tmp_pack_without_roc as (
# MAGIC select
# MAGIC ifnull(a.year, b.year) as year,
# MAGIC ifnull(a.yq, b.yq) as yq ,
# MAGIC ifnull(a.yyyymm, b.yyyymm) as yyyymm,
# MAGIC ifnull(a.iqvia_pack_code, b.iqvia_pack_code) as iqvia_pack_code,
# MAGIC ifnull(a.geo_key, b.geo_key) as geo_key,
# MAGIC ifnull(a.count_unit, b.count_unit) as count_unit,
# MAGIC ifnull(a.average_price, b.average_price) as average_price,
# MAGIC ifnull(a.sales_value, 0) as sales_value,
# MAGIC ifnull(a.sales_unit, 0) as sales_unit,
# MAGIC ifnull(a.counting_unit, 0) as counting_unit,
# MAGIC ifnull(a.counting_units_obversion, b.counting_units_obversion) as counting_units_obversion,
# MAGIC ifnull(a.pack_flag, b.pack_flag) as pack_flag,
# MAGIC ifnull(a.brand_flag, b.brand_flag) as brand_flag,
# MAGIC ifnull(b.sales_value_ly, 0) as sales_value_ly,
# MAGIC ifnull(b.sales_unit_ly, 0) as sales_unit_ly,
# MAGIC ifnull(b.counting_unit_ly, 0) as counting_unit_ly
# MAGIC from tmp_pack_this_year_without_roc a
# MAGIC full outer join tmp_pack_next_year_without_roc b
# MAGIC on a.yyyymm = b.yyyymm
# MAGIC and a.iqvia_pack_code = b.iqvia_pack_code
# MAGIC and a.geo_key = b.geo_key
# MAGIC -- and nvl(a.prescription_nature,'') = nvl(b.prescription_nature,'')
# MAGIC -- and nvl(a.medicine_attribute,'') = nvl(b.medicine_attribute,'')
# MAGIC -- and nvl(a.dosage_form,'') = nvl(b.dosage_form,'')
# MAGIC -- and nvl(a.object,'') = nvl(b.object,'')
# MAGIC -- and nvl(a.zk_classify1,'') = nvl(b.zk_classify1,'')
# MAGIC -- and nvl(a.zk_classify2,'') = nvl(b.zk_classify2,'')
# MAGIC -- and nvl(a.zk_classify3,'') = nvl(b.zk_classify3,'')
# MAGIC and nvl(a.target_points,'') = nvl(b.target_points,'')
# MAGIC -- and nvl(a.common_name,'') = nvl(b.common_name,'')
# MAGIC -- and nvl(a.brand_name,'') = nvl(b.brand_name,'')
# MAGIC -- and nvl(a.product_name,'') = nvl(b.product_name,'')
# MAGIC -- and nvl(a.pack_des,'') = nvl(b.pack_des,'')
# MAGIC -- and nvl(a.factory,'') = nvl(b.factory,'')
# MAGIC -- and nvl(a.corp_des,'') = nvl(b.corp_des,'')
# MAGIC ), tmp_final_sales as (
# MAGIC select * from tmp_pack_with_roc
# MAGIC union all
# MAGIC select * from tmp_pack_without_roc
# MAGIC )
# MAGIC
# MAGIC insert overwrite table tmp.tmp_retail_dtp_final_sales
# MAGIC
# MAGIC select
# MAGIC year,
# MAGIC yq,
# MAGIC yyyymm,
# MAGIC iqvia_pack_code,
# MAGIC geo_key,
# MAGIC count_unit,
# MAGIC average_price,
# MAGIC sales_value,
# MAGIC sales_unit,
# MAGIC counting_units_obversion,
# MAGIC counting_unit,
# MAGIC sales_value_ly,
# MAGIC sales_unit_ly,
# MAGIC counting_unit_ly,
# MAGIC pack_flag,
# MAGIC brand_flag
# MAGIC from tmp_final_sales
# MAGIC order by yyyymm
# MAGIC
# COMMAND ----------
# MAGIC %md
# MAGIC ## STEP-2: calculate OTHERS data
# COMMAND ----------
# DBTITLE 1,不再计算
# %sql
# -------------------------------------------------------------------------------------
# -- STEP-2: calculate OTHERS data
# -- 2.1 calaulate DTP_AZ_OTHERS data
# -------------------------------------------------------------------------------------
# with tmp_az_total_now as (
# select
# cast(top_corp.sales_quarter as int ) yyyymm,
# top_corp.corp_name,
# top_corp.sales_amount *1000000.0 sales_amount,
# pack.CORP_COD
# from dwd.dwd_gnd_ext_retail_dtp_top_copd top_corp
# left join (
# select distinct ZK_Corp_C,CORP_COD
# from dwd.dwd_inc_gnd_retail_b2c_label_total
# ) pack on replace(top_corp.corp_name,'-','')=pack.ZK_Corp_C
# where pack.CORP_COD='A5Z'
# ), tmp_az_total_ly as (
# select
# cast(yyyymm + 100 as int) as yyyymm,
# corp_name,
# sales_amount as sales_amount_ly,
# CORP_COD
# from tmp_az_total_now
# ), tmp_az_total as (
# select
# a.*,
# ifnull(b.sales_amount_ly, 0) as sales_amount_ly
# from tmp_az_total_now a
# left join tmp_az_total_ly b
# on a.yyyymm = b.yyyymm
# and a.corp_name = b.corp_name
# and a.CORP_COD = b.CORP_COD
# ),tmp_az_pack_total as (
# select
# a.year,
# a.yq,
# a.yyyymm,
# sum(a.sales_value) as sales_value,
# sum(a.sales_unit) as sales_unit,
# sum(a.counting_unit) as counting_unit,
# sum(a.sales_value_ly) as sales_value_ly,
# sum(a.sales_unit_ly) as sales_unit_ly,
# sum(a.counting_unit_ly) as counting_unit_ly
# from tmp.tmp_retail_dtp_final_sales a
# where a.iqvia_pack_code in (
# select distinct iqvia_pack_code
# from tmp.tmp_zk_retail_dtp_market_corp
# where corp_cod = 'A5Z'
# )
# group by a.year,a.yq,a.yyyymm
# order by a.yyyymm
# )
# insert into table tmp.tmp_retail_dtp_final_sales
# select
# left(a.yyyymm, 4) as year,
# concat(left(a.yyyymm, 4), 'Q', CEIL(CAST(RIGHT(a.yyyymm,2) AS INT)/3)) as yq,
# a.yyyymm,
# 'DTP_AZ_OTHERS' as iqvia_pack_code,
# 'ROC' as geo_key,
# 0 as count_unit,
# 0 as average_price,
# a.sales_amount - nvl(b.sales_value, 0) as sales_value,
# 0 as sales_unit,
# 0 as counting_units_obversion,
# 0 as counting_unit,
# a.sales_amount_ly - nvl(b.sales_value_ly, 0) as sales_value_ly,
# 0 as sales_unit_ly,
# 0 as counting_unit_ly,
# 0 as pack_flag,
# 0 as brand_flag
# from tmp_az_total a
# left join tmp_az_pack_total b
# on a.yyyymm = b.yyyymm
# order by a.yyyymm
# COMMAND ----------
# DBTITLE 1,不再计算
# %sql
# -------------------------------------------------------------------------------------
# -- STEP-2: calculate OTHERS data
# -- 2.2 calaulate DTP_OTHERS data
# -------------------------------------------------------------------------------------
# with tmp_total_now as (
# SELECT
# dtp_name,
# sales_quarter as yyyymm,
# sales_amount * 1000000.0 as sales_amount
# FROM dwd.dwd_gnd_ext_retail_dtp_overall_market
# ), tmp_total_ly as (
# select
# dtp_name,
# cast(yyyymm + 100 as int) as yyyymm,
# sales_amount as sales_amount_ly
# from tmp_total_now
# ), tmp_total as (
# select
# a.*,
# ifnull(b.sales_amount_ly, 0) as sales_amount_ly
# from tmp_total_now a
# left join tmp_total_ly b
# on a.yyyymm = b.yyyymm
# ),tmp_pack_total as (
# select
# a.year,
# a.yq,
# a.yyyymm,
# sum(a.sales_value) as sales_value,
# sum(a.sales_unit) as sales_unit,
# sum(a.counting_unit) as counting_unit,
# sum(a.sales_value_ly) as sales_value_ly,
# sum(a.sales_unit_ly) as sales_unit_ly,
# sum(a.counting_unit_ly) as counting_unit_ly
# from tmp.tmp_retail_dtp_final_sales a
# group by a.year,a.yq,a.yyyymm
# order by a.yyyymm
# )
# insert into table tmp.tmp_retail_dtp_final_sales
# select
# left(a.yyyymm, 4) as year,
# concat(left(a.yyyymm, 4), 'Q', CEIL(CAST(RIGHT(a.yyyymm,2) AS INT)/3)) as yq,
# a.yyyymm,
# 'DTP_OTHERS' as iqvia_pack_code,
# 'ROC' as geo_key,
# 0 as count_unit,
# 0 as average_price,
# a.sales_amount - nvl(b.sales_value, 0) as sales_value,
# 0 as sales_unit,
# 0 as counting_units_obversion,
# 0 as counting_unit,
# a.sales_amount_ly - nvl(b.sales_value_ly, 0) as sales_value_ly,
# 0 as sales_unit_ly,
# 0 as counting_unit_ly,
# 0 as pack_flag,
# 0 as brand_flag
# from tmp_total a
# left join tmp_pack_total b
# on a.yyyymm = b.yyyymm
# order by a.yyyymm
# COMMAND ----------
############################################################END################################################################

177
Retail/09 dwd_inc_gnd_ext_retail_nataional.py Normal file
View File

@@ -0,0 +1,177 @@
# Databricks notebook source
#当更新pack 或品牌 事实数据时需要运行此代码，否则无需运行。
# COMMAND ----------
# MAGIC %sql
# MAGIC update dwd.dwd_gnd_ext_retail_corresponding_relationship set table_name ='dwd.dwd_gnd_ext_retail_nataional_oap' where file_name ='pack-CV-抗血栓2通用名-全国.xlsx';
# MAGIC update dwd.dwd_gnd_ext_retail_corresponding_relationship set table_name ='dwd.dwd_gnd_ext_retail_htn' where file_name ='pack-CV-高血压-化学药-全国.xlsx';
# MAGIC update dwd.dwd_gnd_ext_retail_corresponding_relationship set table_name ='dwd.dwd_gnd_ext_retail_atomizer' where file_name ='pack-雾化器-全国&县域数据.xlsx';
# MAGIC update dwd.dwd_gnd_ext_retail_corresponding_relationship set table_name ='dwd.dwd_gnd_ext_retail_anti_asthma_copd' where file_name ='pack-RE-慢阻肺-全国.xlsx';
# MAGIC update dwd.dwd_gnd_ext_retail_corresponding_relationship set table_name ='dwd.dwd_gnd_ext_zk_brand' where file_name ='Brand-品牌数据报表.xlsx';
# MAGIC update dwd.dwd_gnd_ext_retail_corresponding_relationship set table_name ='dwd.dwd_gnd_ext_retail_statin_xzk' where file_name ='pack-CV-他汀类+血脂康-全国.xlsx';
# MAGIC update dwd.dwd_gnd_ext_retail_corresponding_relationship set table_name ='dwd.dwd_gnd_ext_retail_nataional_rd' where file_name ='pack-RD-肾科-全国.xlsx';
# MAGIC update dwd.dwd_gnd_ext_retail_corresponding_relationship set table_name ='dwd.dwd_gnd_ext_retail_aagsa_ppi_oral' where file_name ='pack-GI-慢性胃炎胃溃疡-全国.xlsx';
# MAGIC update dwd.dwd_gnd_ext_retail_corresponding_relationship set table_name ='dwd.dwd_gnd_ext_retail_nataional_niad' where file_name ='pack-DM-口服降糖化学药.xlsx';
# MAGIC update dwd.dwd_gnd_ext_retail_corresponding_relationship set table_name ='dwd.dwd_gnd_ext_retail_metoprolol_tartrat' where file_name ='pack-CV-酒石酸美托洛尔.xlsx';
# MAGIC
# COMMAND ----------
# brand+ 省份数据自动接入
#获取配置表信息(表名、brand_flag
dfband = spark.sql("""
SELECT DISTINCT table_name tab ,file_name brand_flag FROM dwd.dwd_gnd_ext_retail_corresponding_relationship
where type_name ='BRAND'
""").collect()
def get_union_brand_data(df):
#数据为空
if df == None:
return None
#初始化结果集
union_query = None
for table in df:
# 选择当前表名
T = str(table.tab)
# 获取对应brand表维度对应得 market 名称
pack_flag = str(table.brand_flag)
sql = f"""
select
cast(left(quarter, 4)*100 + right(quarter,1)*3 as int ) AS YYYYMM
,cast(left(quarter, 4) as int ) AS year
,right(quarter, 2) AS quarter
,quarter AS yq
,type AS brand_cat_type
,case when ta = 'NIAD' then 'DM' else ta end AS TA
,market AS market
,zk_brand_category AS zk_brand_category
,zk_common_name AS zk_common_name
,zk_manu_des AS zk_manu_des
,rc_name_en AS rc_name_en
,province_city AS province_city
,ytd AS ytd
,cast(sales_value * 1000000 as decimal(30,10)) AS sales_val
,cast(sales_volume * 1000000 as decimal(30,10)) AS sales_vol
,cast(price as decimal(30,10)) as price
,cast(num_dist_rate as decimal(30,10)) as num_dist_rate
,cast(weig_dist_rate as decimal(30,10)) as weig_dist_rate
,cast(value_share as decimal(30,10)) as val_share
,cast(volume_share as decimal(30,10)) as vol_share
,replace(key_brand_ytd,'-','') as key_brand_ytd
,cast(replace(key_brand_rank_ytd,'-','0') as int) as key_brand_rank_ytd
,replace(top_brand_ytd,'-','') as top_brand_ytd
,cast(replace(top_brand_ms_ytd,'-','0') as decimal(30,10)) as top_brand_ms_ytd
,cast(replace(top_brand_inc_ms_ytd,'-','0') as decimal(30,10)) as top_brand_inc_ms_ytd
,cast(replace(top_brand_gr_ytd,'-','0') as decimal(30,10)) as top_brand_gr_ytd
,replace(key_brand_qtd,'-','') as key_brand_qtd
,cast(replace(key_brand_rank_qtd,'-','0') as int) as key_brand_rank_qtd
,replace(top_brand_qtd,'-','') as top_brand_qtd
,cast(replace(top_brand_ms_qtd,'-','0') as decimal(30,10)) as top_brand_ms_qtd
,cast(replace(top_brand_inc_ms_qtd,'-','0') as decimal(30,10)) as top_brand_inc_ms_qtd
,cast(replace(top_brand_gr_qtd,'-','0') as decimal(30,10)) as top_brand_gr_qtd
,ranked_by as ranked_by
,'{pack_flag}' as pack_flag
,from_utc_timestamp(current_timestamp(),'UTC+8') as etl_insert_dt
,from_utc_timestamp(current_timestamp(),'UTC+8') as etl_update_dt
from {T}
"""
# 读取数据
current_query = spark.sql(sql)
#union 数据
if union_query == None:
union_query = current_query
else:
union_query = union_query.union(current_query)
#返回数据集 / 写入表也行？？？
return union_query
brand_result = get_union_brand_data(dfband)
brand_result.write.mode("overwrite").saveAsTable("dwd.dwd_inc_gnd_ext_retail_nataional_brand_union_all")
# COMMAND ----------
# MAGIC %md
# MAGIC ###新逻辑
# MAGIC - 修改brand数据，先拆分成月维度的数据
# COMMAND ----------
# MAGIC %sql
# MAGIC /*
# MAGIC 修改时间：20250311
# MAGIC 修改人：chenwu
# MAGIC 修改内容：brand来数频率为 季度来数， 但是 pack 为 月度来数据，需要用季度的数据/3得到月度的
# MAGIC */
# MAGIC insert overwrite table dwd.dwd_inc_gnd_ext_retail_nataional_brand_union_all
# MAGIC with quarterly_table as (
# MAGIC select
# MAGIC *
# MAGIC from dwd.dwd_inc_gnd_ext_retail_nataional_brand_union_all
# MAGIC where market not in ('NIAD','Inhaled Extended Market','布地奈德雾化溶液')
# MAGIC -- 范围内只能是 季度来数据的，如果有月度来数据的需要排除掉
# MAGIC )
# MAGIC
# MAGIC ,month_table as (--转化成月度数据
# MAGIC SELECT
# MAGIC SUBSTR(q.yq, 1, 4)*100 + -- 提取年份
# MAGIC LPAD(m.month_num, 2, '0') -- 补零月份
# MAGIC AS YYYYMM -- 月份首日
# MAGIC ,`year`
# MAGIC ,`quarter`
# MAGIC ,yq
# MAGIC ,brand_cat_type
# MAGIC ,TA
# MAGIC ,market
# MAGIC ,zk_brand_category
# MAGIC ,zk_common_name
# MAGIC ,zk_manu_des
# MAGIC ,rc_name_en
# MAGIC ,province_city
# MAGIC ,ytd
# MAGIC ,sales_val /3 --除3
# MAGIC ,sales_vol /3 --除3
# MAGIC ,price
# MAGIC ,num_dist_rate
# MAGIC ,weig_dist_rate
# MAGIC ,val_share
# MAGIC ,vol_share
# MAGIC ,key_brand_ytd
# MAGIC ,key_brand_rank_ytd
# MAGIC ,top_brand_ytd
# MAGIC ,top_brand_ms_ytd
# MAGIC ,top_brand_inc_ms_ytd
# MAGIC ,top_brand_gr_ytd
# MAGIC ,key_brand_qtd
# MAGIC ,key_brand_rank_qtd
# MAGIC ,top_brand_qtd
# MAGIC ,top_brand_ms_qtd
# MAGIC ,top_brand_inc_ms_qtd
# MAGIC ,top_brand_gr_qtd
# MAGIC ,ranked_by
# MAGIC ,pack_flag
# MAGIC ,etl_insert_dt
# MAGIC ,etl_update_dt
# MAGIC FROM
# MAGIC quarterly_table q
# MAGIC LATERAL VIEW EXPLODE( -- 为每季度生成三个月
# MAGIC CASE
# MAGIC WHEN RIGHT(q.yq, 2) = 'Q1' THEN ARRAY(1, 2, 3)
# MAGIC WHEN RIGHT(q.yq, 2) = 'Q2' THEN ARRAY(4, 5, 6)
# MAGIC WHEN RIGHT(q.yq, 2) = 'Q3' THEN ARRAY(7, 8, 9)
# MAGIC WHEN RIGHT(q.yq, 2) = 'Q4' THEN ARRAY(10, 11, 12)
# MAGIC END
# MAGIC ) m AS month_num
# MAGIC )
# MAGIC
# MAGIC ,other_not_quarterly_table (
# MAGIC select
# MAGIC *
# MAGIC from dwd.dwd_inc_gnd_ext_retail_nataional_brand_union_all
# MAGIC where market in ('NIAD','Inhaled Extended Market','布地奈德雾化溶液')
# MAGIC -- 范围内只能是 月度来数据的
# MAGIC )
# MAGIC
# MAGIC select * from month_table
# MAGIC union all
# MAGIC select * from other_not_quarterly_table

4917
Retail/10 map_to_retail_dm_table.py Normal file
View File

File diff suppressed because it is too large Load Diff

1000
Retail/11 map_to_overview_dm_table.py Normal file
View File

File diff suppressed because it is too large Load Diff

81
Retail/12 dws_tf_external_retail_dtp_special.sql Normal file
View File

@@ -0,0 +1,81 @@
-- Databricks notebook source
insert overwrite table dws.dws_tf_external_retail_dtp_special
with dim_geo as (
SELECT
AUDIT_COD
,REGION_TYPE
,PROVINCE_C
FROM DM.DM_TD_EXTERNAL_GEO
WHERE DATA_SOURCE = 'DTP(Quarterly)'
GROUP BY 1,2,3
)
,all_data (--所有DTP BRAND 中的数据聚合
select
market
,province_city
,yyyymm
,sum(sales_value) as sales_value
,sum(sales_volume) as sales_volume
,sum(pdot_counting_unit) as pdot_counting_unit
from dwd.dwd_gnd_ext_dtp_zk_brand --数据来源表
where type = '品类'
and market = 'EGFR TKI'
and zk_brand_category ='EGFR TKI Market'
and ranked_by = 'value'
group by 1,2,3
)
,not_quanguo_data as (--计算非全国部分
select
market
,yyyymm
,sum(sales_value) as sales_value
,sum(sales_volume) as sales_volume
,sum(pdot_counting_unit) as pdot_counting_unit
from all_data
where province_city <> '全国'
group by 1,2
)
,roc_data as (--计算 roc部分的值 = 全国- 非全国合计
select
a.market
,'ROC' as province_city
,a.yyyymm
,a.sales_value-b.sales_value as sales_value
,a.sales_volume - b.sales_volume as sales_volume
,a.pdot_counting_unit - b.pdot_counting_unit as pdot_counting_unit
from all_data a
left join not_quanguo_data b on a.market = b.market and a.yyyymm = b.yyyymm
where a.province_city = '全国'
)
,all_data_with_roc as (--合并全部数据
select
market
,province_city
,yyyymm
,sales_value
,sales_volume
,pdot_counting_unit
from all_data where province_city <> '全国'
union all
select
market
,province_city
,yyyymm
,sales_value
,sales_volume
,pdot_counting_unit
from roc_data
)
select
a.market
,a.province_city
,concat(b.AUDIT_COD, 'DTP(Quarterly)', b.REGION_TYPE ) as audit_key --pbi dim_rc 关联键
,concat(b.AUDIT_COD, 'DTP(Quarterly)' ) audit_source --pbi dim_geo 关联键
,concat(left(a.yyyymm,4),'-',right(a.yyyymm,2),'-01') date_key --pbi dim_date 关联键
,a.yyyymm
,a.sales_value
,a.sales_volume
,a.pdot_counting_unit
from all_data_with_roc a
left join dim_geo b on a.province_city = b.PROVINCE_C

Some files were not shown because too many files have changed in this diff Show More