chenwu/MarketAnalysis-ETL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add EXTERNAL SQL scripts

Browse Source
This commit is contained in:
chenwu
2026-04-20 14:55:25 +08:00
parent c05ba7ec7e
commit b4979eed82
344 changed files with 61619 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

237
EXTERNAL/for_AIA_Dashboard/01 dm_td_aia_inst_mkt.sql Normal file
View File

@@ -0,0 +1,237 @@
-- Databricks notebook source
-------------------------------------------------------------------------------------------
--修改时间：20241021
--修改人：Fanxujia
--修改内容：
--提供给AIA Dashboard的机构维度表
--机构 + 市场粒度
-------------------------------------------------------------------------------------------
-- COMMAND ----------
insert overwrite table dm.dm_td_aia_inst_mkt --Pack_cod以及对应的market
WITH ALL_pack_mkt AS (
SELECT DISTINCT
PACK_CODE AS PACK_COD,
MARKET
FROM DM.DM_TD_EXT_AIA_MARKET_PACK_MAPPING
),
-- 销量表里所有机构 + 市场维度
ALL_INS_mkt AS (
SELECT DISTINCT
AUDIT_COD AS inst_code,
MARKET
FROM dm.dm_tf_ext_unionall_market_sales
WHERE DATA_SOURCE = 'AIA(Monthly)'
),
-- SalesDetail报告用的潜力事实表
-- 其中的潜力市场名字与外部数据报告里的市场的名字不同，不能直接关联
-- 关联思路：潜力市场 ---> 内部产品 ---> 外部产品 ---> 外部市场
potential_decile1 AS (
SELECT DISTINCT
p1.inst_code,
p1.mkt_code,
p1.mkt_name,
p1.potential_type,
p1.potential_type_name,
p1.is_defult_potential_type,
INT(p1.decile) AS decile,
p1.potential_value,
IFNULL(p1.default_decile, 10) AS default_decile
FROM dm.dm_tf_potential p1
WHERE yyyymm = (
SELECT KEY_VALUE
FROM dm.dm_td_date_config
WHERE key_code = 'pfmc_latest_mon'
)
),
-- 潜力市场与内部Brand的mapping
brand_market_mapping AS (
SELECT DISTINCT
brand_code,
market_code
FROM dwd.dwd_gnd_link_family_imsmkt
WHERE active_status = 1
),
-- 内外部产品编码映射表：获取内部Brand与外部pack的mapping
imscode_azcode AS (
SELECT DISTINCT
brandcode,
CASE
WHEN LENGTH(ims_pack_cod) < 7 AND ims_pack_cod REGEXP '^[0-9]'
THEN RIGHT(CONCAT('000000000000', ims_pack_cod), 12)
ELSE ims_pack_cod
END AS ims_pack_cod
FROM dwd.dwd_td_imscode_azcode
WHERE ims_pack_cod IS NOT NULL
),
-- 将潜力数据、Decile数据对应到外部的market
potential_decile2 AS (
SELECT DISTINCT
t1.inst_code,
t1.mkt_code AS potential_mkt_code,
t1.mkt_name AS potential_mkt_name,
t4.market,
t1.potential_type,
t1.potential_type_name,
t1.is_defult_potential_type,
t1.potential_value,
t1.decile,
CASE
WHEN t1.decile <= 3 THEN 'Decile 1-3'
WHEN t1.decile <= 5 THEN 'Decile 4-5'
WHEN t1.decile <= 8 THEN 'Decile 6-8'
WHEN t1.decile <= 10 THEN 'Decile 9-10'
END AS decile_group
FROM potential_decile1 t1
INNER JOIN brand_market_mapping t2
ON t1.mkt_code = t2.market_code
INNER JOIN imscode_azcode t3
ON t2.brand_code = t3.brandcode
INNER JOIN ALL_pack_mkt t4
ON t3.ims_pack_cod = t4.pack_cod
),
-- 外部市场与默认潜力市场的mapping关系（一对一的关系）
-- 增加对手工数据的处理
default_potential_market AS (
SELECT DISTINCT
REPLACE(REPLACE(REPLACE(UPPER(market), '（', '('), '）', ')'), ' (', '(') AS market,
REPLACE(
REPLACE(REPLACE(UPPER(Default_Potential_Market), '（', '('), '）', ')'),
' (',
'('
) AS Default_Potential_Market
FROM dwd.dwd_gnd_ext_default_potential_market_link
),
-- 潜力数据只到机构 + 市场，不到产品
potential_decile3 AS (
SELECT DISTINCT
t1.inst_code,
t1.market,
NVL(t2.potential_mkt_code, '') AS potential_mkt_code,
NVL(t2.potential_mkt_name, '') AS potential_mkt_name,
CASE
WHEN t2.potential_mkt_name IS NOT NULL AND t3.default_potential_market IS NULL
THEN '0'
WHEN t2.potential_mkt_name IS NULL
THEN ''
ELSE '1'
END AS is_defult_potential_mkt,
NVL(t2.potential_type, '') AS potential_type_code,
NVL(t2.potential_type_name, '') AS potential_type_name,
NVL(t2.is_defult_potential_type, '') AS is_defult_potential_type,
-- 潜力值字段类型decimal(25,8)，不转成空字符串了
t2.potential_value,
NVL(t2.decile, '') AS decile,
NVL(t2.decile_group, '') AS decile_group
FROM ALL_INS_mkt t1
LEFT JOIN potential_decile2 t2
ON t1.inst_code = t2.inst_code
AND t1.market = t2.market
LEFT JOIN default_potential_market t3
ON REPLACE(REPLACE(REPLACE(UPPER(t1.market), '（', '('), '）', ')'), ' (', '(') = t3.market
AND REPLACE(REPLACE(REPLACE(UPPER(t2.potential_mkt_name), '（', '('), '）', ')'), ' (', '(') = t3.Default_Potential_Market
)
SELECT DISTINCT
t1.inst_code,
t1.market,
t2.potential_mkt_code,
t2.potential_mkt_name,
t2.is_defult_potential_mkt,
t2.potential_type_code,
t2.potential_type_name,
t2.is_defult_potential_type,
t2.potential_value,
t2.decile,
t2.decile_group,
CASE
WHEN t2.decile = '' THEN 99
ELSE CAST(t2.decile AS INT)
END AS decile_order,
CASE
WHEN t1.inst_code IS NULL OR t1.inst_code = '' THEN 'ROC'
ELSE t1.inst_code
END AS audit_cod,
'AIA(Monthly)' AS DATA_SOURCE
FROM ALL_INS_mkt t1
LEFT JOIN potential_decile3 t2
ON t1.inst_code = t2.inst_code
AND t1.MARKET = t2.MARKET;
-- COMMAND ----------
--没有默认潜力市场的市场，增加一条假的默认潜力市场数据
with mkt1 as (
select distinct
market
from
dm.dm_td_aia_inst_mkt
),
mkt2 as (
select distinct
market
from
dm.dm_td_aia_inst_mkt
where
is_defult_potential_mkt = '1'
),
mkt as (
select
a.market
from
mkt1 a
left join mkt2 b
on a.market = b.market
where
b.market is null
),
all_inst_mkt as (
select distinct
t1.inst_code,
t1.audit_cod,
t1.market
from
dm.dm_td_aia_inst_mkt t1
inner join mkt t2
on t1.market = t2.market
) insert into dm.dm_td_aia_inst_mkt (
inst_code,
market,
potential_mkt_code,
potential_mkt_name,
is_defult_potential_mkt,
potential_type_code,
potential_type_name,
is_defult_potential_type,
potential_value,
decile,
decile_group,
decile_order,
audit_cod,
DATA_SOURCE
)
select distinct
inst_code,
market,
'' as potential_mkt_code,
'' as potential_mkt_name,
'1' as is_defult_potential_mkt,
'' as potential_type_code,
'' as potential_type_name,
'1' as is_defult_potential_type,
null as potential_value,
'' as decile,
'' as decile_group,
99 as decile_order,
audit_cod,
'AIA(Monthly)' as DATA_SOURCE
from
all_inst_mkt

BIN
EXTERNAL/for_AIA_Dashboard/01 dm_td_aia_inst_mkt.sql:Zone.Identifier Normal file
View File

Binary file not shown.

293
EXTERNAL/for_AIA_Dashboard/02 dm_td_aia_auth_sales.sql Normal file
View File

@@ -0,0 +1,293 @@
-- Databricks notebook source
-- MAGIC %run ../../../Common/config
-- COMMAND ----------
-- MAGIC %python
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dim_product_wide`').createOrReplaceTempView('cdw_dwd_dim_product_wide')
-- COMMAND ----------
-------------------------------------------------------------------------------------------
--修改时间：20250109
--修改人：Caoyining
--修改内容：
--提供给AIA Dashboard的权限表
--销售人员给自己岗位对应的数据，针对部分AIA Org N+1特殊岗位给予特定DSM角色RSM的岗位权限
-------------------------------------------------------------------------------------------
create or replace temporary view aia_auth_sales
as
--从销量表取到全部的机构、市场组合
with ALL_AUDIT_Market as (
select distinct t1.AUDIT_COD,NVL(t2.market,'ALL Market') market
from dm.dm_ext_aia_sales t1
left join DM.DM_TD_EXT_AIA_MARKET_PACK_MAPPING t2
on t1.pack_cod = t2.PACK_CODE
)
--CVH架构月
,CVH_YM as (
select key_value as yyyymm
from dm.dm_td_date_config
where key_code = 'pfmc_cvh'
)
--获取特定dsm的rsm岗位号
,org_dsmforrsm as (
select distinct rsm_trty_code
from dm.dm_td_org org
inner join CVH_YM ym
on org.yyyymm = ym.yyyymm
where dsm_trty_code in(
'DM_EG0072'
,'DM_EG0089'
,'DM_EG0101'
,'DM_EG0141'
,'DM_EG0066'
,'DM_EG0173'
,'DM_EG0090'
,'DM_EG0045'
,'DM_EG0011'
,'DM_EG0102')
)
,dsm_rsm_kcode as (
select distinct dsm_kcode,dsm_name,rsm_kcode
from dm.dm_td_org org
inner join CVH_YM ym
on org.yyyymm = ym.yyyymm
where dsm_trty_code in(
'DM_EG0072'
,'DM_EG0089'
,'DM_EG0101'
,'DM_EG0141'
,'DM_EG0066'
,'DM_EG0173'
,'DM_EG0090'
,'DM_EG0045'
,'DM_EG0011'
,'DM_EG0102')
)
--特定dsm岗位org表
,org_dsm as (
select distinct
t1.yyyymm,
t1.bu_code,
t1.nsd_kcode,
t1.nsd_name,
t1.rsd_kcode,
t1.rsd_name,
t1.rsm_kcode,
t1.rsm_name,
t1.dsm_kcode,
t1.dsm_name,
t1.mr_trty_code
from dm.dm_td_org t1
inner join CVH_YM t2
on t1.yyyymm = t2.yyyymm
inner join org_dsmforrsm t3
on t1.rsm_trty_code = t3.rsm_trty_code
)
,dsm_rsm_all as (
select
t2.yyyymm,
t2.bu_code,
t2.nsd_kcode,
t2.nsd_name,
t2.rsd_kcode,
t2.rsd_name,
t2.rsm_kcode,
t2.rsm_name,
t1.dsm_kcode,
t1.dsm_name,
t2.mr_trty_code
from dsm_rsm_kcode t1
left join org_dsm t2
on t1.rsm_kcode = t2.rsm_kcode
)
--剔除特定dsm岗位org表
,orgWithoutDsm as (
select distinct
t1.yyyymm,
t1.bu_code,
t1.nsd_kcode,
t1.nsd_name,
t1.rsd_kcode,
t1.rsd_name,
t1.rsm_kcode,
t1.rsm_name,
t1.dsm_kcode,
t1.dsm_name,
t1.mr_trty_code
from dm.dm_td_org t1
inner join CVH_YM t2
on t1.yyyymm = t2.yyyymm
where t1.dsm_trty_code not in (
'DM_EG0072'
,'DM_EG0089'
,'DM_EG0101'
,'DM_EG0141'
,'DM_EG0066'
,'DM_EG0173'
,'DM_EG0090'
,'DM_EG0045'
,'DM_EG0011'
,'DM_EG0102')
)
--territory表
,territory as (
select distinct
t1.yyyymm,
t1.trty_code,
t1.inst_code,
t1.family_code
from dm.dm_td_sd_territory_indication t1
inner join CVH_YM t2
on t1.yyyymm = t2.yyyymm
where t1.bu not in ('CUBE','KA')
)
--PACK与market的mapping关系表
,pack_market as (
select distinct pack_code as pack_cod,market
from DM.DM_TD_EXT_AIA_MARKET_PACK_MAPPING
)
--将内部的Family_code转为外部的pack_cod
,inst_market as (
select distinct
t1.yyyymm,
t1.trty_code,
t1.inst_code,
t5.market
from territory t1
left join dwd.dwd_td_institution t2
on t1.inst_code = t2.inst_code
left join cdw_dwd_dim_product_wide t3
on t1.family_code = t3.prd_family_cd
left join dwd.dwd_td_imscode_azcode t4
on t3.prd_brand_cd = t4.brandcode
left join pack_market t5
on t4.ims_pack_cod = t5.pack_cod
)
select distinct
t2.dsm_kcode as KCODE,
'dsm' as role,
t2.mr_trty_code,
concat(t0.AUDIT_COD,'AIA(Monthly)') as AUDIT_SOURCE,
concat(t0.MARKET,'AIA(Monthly)') as MKT_SOURCE,
concat(t2.bu_code,'AIA(Monthly)') as BU_CODE_AUTH,
concat(t3.pack_cod,'AIA(Monthly)') as PACK_SOURCE
from ALL_AUDIT_Market t0
inner join inst_market t1
on t0.AUDIT_COD = t1.inst_code
and t0.market = t1.market
inner join orgWithoutDsm t2
on t1.trty_code = t2.mr_trty_code
and t1.yyyymm = t2.yyyymm
inner join pack_market t3
on t1.market = t3.market
-- where t2.dsm_name <> 'Vacant'
union all
select distinct
t2.dsm_kcode as KCODE,
'dsm' as role,
t2.mr_trty_code,
concat(t0.AUDIT_COD,'AIA(Monthly)') as AUDIT_SOURCE,
concat(t0.MARKET,'AIA(Monthly)') as MKT_SOURCE,
concat(t2.bu_code,'AIA(Monthly)') as BU_CODE_AUTH,
concat(t3.pack_cod,'AIA(Monthly)') as PACK_SOURCE
from ALL_AUDIT_Market t0
inner join inst_market t1
on t0.AUDIT_COD = t1.inst_code
and t0.market = t1.market
inner join dsm_rsm_all t2
on t1.trty_code = t2.mr_trty_code
and t1.yyyymm = t2.yyyymm
inner join pack_market t3
on t1.market = t3.market
-- where t2.dsm_name <> 'Vacant'
union all
select distinct
t2.rsm_kcode as KCODE,
'rsm' as role,
t2.mr_trty_code,
concat(t0.AUDIT_COD,'AIA(Monthly)') as AUDIT_SOURCE,
concat(t0.MARKET,'AIA(Monthly)') as MKT_SOURCE,
concat(t2.bu_code,'AIA(Monthly)') as BU_CODE_AUTH,
concat(t3.pack_cod,'AIA(Monthly)') as PACK_SOURCE
from ALL_AUDIT_Market t0
inner join inst_market t1
on t0.AUDIT_COD = t1.inst_code
and t0.market = t1.market
inner join orgWithoutDsm t2
on t1.trty_code = t2.mr_trty_code
and t1.yyyymm = t2.yyyymm
inner join pack_market t3
on t1.market = t3.market
-- where t2.rsm_name <> 'Vacant'
union all
select distinct
t2.rsd_kcode as KCODE,
'rsd' as role,
t2.mr_trty_code,
concat(t0.AUDIT_COD,'AIA(Monthly)') as AUDIT_SOURCE,
concat(t0.MARKET,'AIA(Monthly)') as MKT_SOURCE,
concat(t2.bu_code,'AIA(Monthly)') as BU_CODE_AUTH,
concat(t3.pack_cod,'AIA(Monthly)') as PACK_SOURCE
from ALL_AUDIT_Market t0
inner join inst_market t1
on t0.AUDIT_COD = t1.inst_code
and t0.market = t1.market
inner join orgWithoutDsm t2
on t1.trty_code = t2.mr_trty_code
and t1.yyyymm = t2.yyyymm
inner join pack_market t3
on t1.market = t3.market
-- where t2.rsd_name <> 'Vacant'
union all
select distinct
t2.nsd_kcode as KCODE,
'nsd' as role,
t2.mr_trty_code,
concat(t0.AUDIT_COD,'AIA(Monthly)') as AUDIT_SOURCE,
concat(t0.MARKET,'AIA(Monthly)') as MKT_SOURCE,
concat(t2.bu_code,'AIA(Monthly)') as BU_CODE_AUTH,
concat(t3.pack_cod,'AIA(Monthly)') as PACK_SOURCE
from ALL_AUDIT_Market t0
inner join inst_market t1
on t0.AUDIT_COD = t1.inst_code
and t0.market = t1.market
inner join orgWithoutDsm t2
on t1.trty_code = t2.mr_trty_code
and t1.yyyymm = t2.yyyymm
inner join pack_market t3
on t1.market = t3.market
-- where t2.nsd_name <> 'Vacant'
-- COMMAND ----------
insert overwrite table dm.dm_auth_aia_sales_geo
select distinct KCODE,AUDIT_SOURCE
from aia_auth_sales
-- COMMAND ----------
insert overwrite table dm.dm_auth_aia_sales_mkt
select distinct KCODE,MKT_SOURCE
from aia_auth_sales
-- COMMAND ----------
insert overwrite table dm.dm_auth_aia_sales_org
select distinct
KCODE,
role,
mr_trty_code,
concat(mr_trty_code,'AIA(Monthly)') as trty_source
from aia_auth_sales
-- COMMAND ----------
insert overwrite table dm.dm_auth_aia_sales_pack
select distinct KCODE,PACK_SOURCE
from aia_auth_sales

BIN
EXTERNAL/for_AIA_Dashboard/02 dm_td_aia_auth_sales.sql:Zone.Identifier Normal file
View File

Binary file not shown.

23
EXTERNAL/for_AIA_Dashboard/03 dm_td_aia_original_col.sql Normal file
View File

@@ -0,0 +1,23 @@
-- Databricks notebook source
insert overwrite table dm.dm_td_aia_original_col
select distinct
case when length(trim(IQVIA_PACK_CODE)) < 12 and trim(IQVIA_PACK_CODE) REGEXP '^[0-9]' then right(concat('000000000000',trim(IQVIA_PACK_CODE)),12)
else trim(IQVIA_PACK_CODE) end as PACK_COD,
-- case when length(trim(IQVIA_PACK_CODE)) < 7 then right(concat('0000000',trim(IQVIA_PACK_CODE)),7)
-- else trim(IQVIA_PACK_CODE) end as PACK_COD,
'AIA(Monthly)' as DATA_SOURCE,
ATC1,
ATC2,
ATC3,
ATC4,
org_prd_cd,
org_mole_nm_c,
org_manu_prd_nm_c,
org_prd_nm_c,
org_pk,
org_prd_str,
org_pth,
min_pk_unit,
org_pk_unit,
org_manu_nm
from dwd.dwd_gnd_ext_aia_cpt_data;

BIN
EXTERNAL/for_AIA_Dashboard/03 dm_td_aia_original_col.sql:Zone.Identifier Normal file
View File

Binary file not shown.

161
EXTERNAL/for_AIA_Dashboard/04 dm_td_aia_nosales_inst.sql Normal file
View File

@@ -0,0 +1,161 @@
-- Databricks notebook source
---------------------------------------------------------------------------------------
--修改时间：20241113
--修改人：Fanxujia
--修改内容：
--无销量机构列表
--以CPA机构表为主表，根据事实表判断每个月销量是否为0
---------------------------------------------------------------------------------------
insert overwrite table dm.dm_td_aia_nosales_inst(
year,
cpa_ins_code,
az_ins_code,
az_ins_name,
January,
February,
March,
April,
May,
June,
July,
August,
September,
October,
November,
December,
etl_insert_dt,
etl_update_dt
)
--12个月
with MM as (
select explode(array('01','02','03','04','05','06','07','08','09','10','11','12')) as MM
)
--机构名称
,inst_name as (
select distinct
inst_code,
inst_name
from dm.dm_td_institution
)
--全部CPA机构,year
,ALL_INS_year as (
select distinct
t1.year,
t1.cpa_hospital_code,
t1.ins_cd_nl as inst_code,
t2.inst_name
from dwd.dwd_gnd_hospital_not_provided t1
left join inst_name t2
on t1.ins_cd_nl = t2.inst_code
)
--全部CPA机构,yyyymm
,ALL_INS_yyyymm as (
select distinct
concat(t1.year,t2.MM) as yyyymm,
t1.cpa_hospital_code,
t1.ins_cd_nl as inst_code,
t3.inst_name
from dwd.dwd_gnd_hospital_not_provided t1
cross join MM t2
left join inst_name t3
on t1.ins_cd_nl = t3.inst_code
)
--有销量的机构
,sales_ins as (
select
YYYYMM,
inst_code
from dm.dm_tf_external_sales
where data_source = 'AIA(Monthly)'
group by YYYYMM,inst_code
having sum(SALES_VALUE_CAL) <> 0
)
--销量最大月
,sales_maxym as (
select max(yyyymm) as max_ym
from dm.dm_tf_external_sales
where data_source = 'AIA(Monthly)'
)
--无销量的机构，by yyyymm
,no_sales_ins_yyyymm as (
select t1.YYYYMM,
t1.cpa_hospital_code,
t1.inst_code,
t1.inst_name,
left(t1.yyyymm,4) as year,
right(t1.yyyymm,2) as MM
from ALL_INS_yyyymm t1
left join sales_ins t2
on t1.inst_code = t2.inst_code
and t1.YYYYMM = t2.YYYYMM
cross join sales_maxym t3
where t2.inst_code is null
and t1.YYYYMM <= t3.max_ym
)
--无销量的机构，by year
,no_sales_ins_year1 as (
select distinct
year,
cpa_hospital_code,
inst_code,
inst_name,
case when MM = '01' then 'N' end as January,
case when MM = '02' then 'N' end as February,
case when MM = '03' then 'N' end as March,
case when MM = '04' then 'N' end as April,
case when MM = '05' then 'N' end as May,
case when MM = '06' then 'N' end as June,
case when MM = '07' then 'N' end as July,
case when MM = '08' then 'N' end as August,
case when MM = '09' then 'N' end as September,
case when MM = '10' then 'N' end as October,
case when MM = '11' then 'N' end as November,
case when MM = '12' then 'N' end as December
from no_sales_ins_yyyymm
)
,no_sales_ins_year2 as (
select
year,
cpa_hospital_code,
inst_code,
inst_name,
MAX(January) as January,
MAX(February) as February,
MAX(March) as March,
MAX(April) as April,
MAX(May) as May,
MAX(June) as June,
MAX(July) as July,
MAX(August) as August,
MAX(September) as September,
MAX(October) as October,
MAX(November) as November,
MAX(December) as December
from no_sales_ins_year1
group by year,cpa_hospital_code,inst_code,inst_name
)
select distinct
t1.year,
t1.cpa_hospital_code as cpa_ins_code,
t1.inst_code as az_ins_code,
t1.inst_name as az_ins_name,
NVL(t2.January,'') as January,
NVL(t2.February,'') as February,
NVL(t2.March,'') as March,
NVL(t2.April,'') as April,
NVL(t2.May,'') as May,
NVL(t2.June,'') as June,
NVL(t2.July,'') as July,
NVL(t2.August,'') as August,
NVL(t2.September,'') as September,
NVL(t2.October,'') as October,
NVL(t2.November,'') as November,
NVL(t2.December,'') as December,
from_utc_timestamp(current_timestamp,'UTC+8') as etl_insert_dt,
from_utc_timestamp(current_timestamp,'UTC+8') as etl_update_dt
from ALL_INS_year t1
left join no_sales_ins_year2 t2
on t1.year = t2.year
and t1.cpa_hospital_code = t2.cpa_hospital_code

BIN
EXTERNAL/for_AIA_Dashboard/04 dm_td_aia_nosales_inst.sql:Zone.Identifier Normal file
View File

Binary file not shown.

61
EXTERNAL/for_AIA_Dashboard/05 dm_td_aia_is_eagle_flag.sql Normal file
View File

@@ -0,0 +1,61 @@
-- Databricks notebook source
------------------------------------------------------------------------------------
--修改时间：20241128
--修改人：Fanxujia
--修改内容：
--新增is_eagle字段，参考Eagle BI报告
--粒度是机构+市场
------------------------------------------------------------------------------------
-- COMMAND ----------
-- MAGIC %run ../../../Common/config
-- COMMAND ----------
-- MAGIC %python
-- MAGIC spark.read.table(f'`{CDW_CATALOG}`.`dwd`.`dim_product_wide`').createOrReplaceTempView('cdw_dwd_dim_product_wide')
-- COMMAND ----------
insert overwrite table dm.dm_td_aia_is_eagle_flag
WITH cvh_org AS (
SELECT key_value
FROM dm.dm_td_date_config
WHERE key_code = 'pfmc_cvh'
),
pack_mkt AS (
SELECT DISTINCT pack_code as pack_cod, MARKET
FROM DM.DM_TD_EXT_AIA_MARKET_PACK_MAPPING
),
famil_mkt AS (
SELECT
DISTINCT
t1.prd_family_cd,
t3.MARKET
FROM cdw_dwd_dim_product_wide t1
INNER JOIN dwd.dwd_td_imscode_azcode t2
ON t1.prd_brand_cd = t2.brandcode
INNER JOIN pack_mkt t3
ON t2.ims_pack_cod = t3.pack_cod
),
base AS (
-- 谓词下推：先过滤主表，减少 JOIN 数据量
SELECT inst_code, project_name, family_code, yyyymm
FROM dm.dm_td_sd_territory_indication
WHERE project_name <> 'NA'
)
SELECT
base.inst_code,
famil_mkt.market,
base.project_name,
'AIA(Monthly)' DATA_SOURCE,
base.inst_code AUDIT_COD
FROM base
INNER JOIN cvh_org
ON base.yyyymm = cvh_org.key_value
LEFT SEMI JOIN dm.dm_ext_aia_sales t_aia -- 替代 DISTINCT + INNER JOIN
ON base.inst_code = t_aia.inst_code
INNER JOIN famil_mkt
ON base.family_code = famil_mkt.prd_family_cd

BIN
EXTERNAL/for_AIA_Dashboard/05 dm_td_aia_is_eagle_flag.sql:Zone.Identifier Normal file
View File

Binary file not shown.

133
EXTERNAL/for_AIA_Dashboard/06 dm_td_aia_rank.sql Normal file
View File

@@ -0,0 +1,133 @@
-- Databricks notebook source
------------------------------------------------------------------
--修改时间：20241209
--修改人：Fanxujia
--修改内容：
--新增产品排序表，供PBI使用
------------------------------------------------------------------
-- COMMAND ----------
--brand的排名
insert overwrite table dm.dm_td_aia_brand_rank
WITH max_month AS (
SELECT MAX(yyyymm) AS max_yyyymm
FROM (SELECT DISTINCT yyyymm FROM dm.dm_tf_ext_aia_sales) t
),
pack_prod AS (
-- 过滤前移，缩小维表体积再 BROADCAST
SELECT DISTINCT PACK_COD, PROD_DES_C
FROM dm.dm_td_external_packinfo
WHERE DATA_SOURCE = 'AIA(Monthly)'
AND PROD_DES_C <> 'OTHERS'
),
prod_sales AS (
SELECT /*+ BROADCAST(pack_prod) */
p.PROD_DES_C,
SUM(s.SALES_VALUE_CAL) AS SALES_VALUE_CAL
FROM dm.dm_tf_ext_aia_sales s
INNER JOIN pack_prod p ON s.PACK_CODE = p.PACK_COD
WHERE s.yyyymm BETWEEN CONCAT(LEFT((SELECT max_yyyymm FROM max_month), 4), '01')
AND (SELECT max_yyyymm FROM max_month)
GROUP BY p.PROD_DES_C
)
SELECT PROD_DES_C,
DENSE_RANK() OVER (ORDER BY SALES_VALUE_CAL DESC, PROD_DES_C) AS RANK
FROM prod_sales
UNION ALL SELECT 'Market', -1
UNION ALL SELECT 'AZ Total', 0
UNION ALL SELECT 'OTHERS', (SELECT COUNT(*) + 1 FROM prod_sales)
-- COMMAND ----------
--Class的排名
insert overwrite table dm.dm_td_aia_class_rank
WITH max_month AS (
SELECT MAX(yyyymm) AS max_yyyymm
FROM (SELECT DISTINCT yyyymm FROM dm.dm_tf_ext_aia_sales) t
),
pack_class AS (
SELECT DISTINCT PACK_CODE, upper(class) class
FROM dm.dm_td_ext_aia_market_pack_mapping
WHERE class <> '-'
),
class_sales AS (
SELECT /*+ BROADCAST(pack_class) */
c.class,
SUM(s.SALES_VALUE_CAL) AS SALES_VALUE_CAL
FROM dm.dm_tf_ext_aia_sales s
INNER JOIN pack_class c ON s.PACK_CODE = c.PACK_CODE
WHERE s.yyyymm BETWEEN CONCAT(LEFT((SELECT max_yyyymm FROM max_month), 4), '01')
AND (SELECT max_yyyymm FROM max_month)
GROUP BY c.class
)
SELECT class,
DENSE_RANK() OVER (ORDER BY SALES_VALUE_CAL DESC, class) AS RANK
FROM class_sales
UNION ALL SELECT 'Market', -1
UNION ALL SELECT 'AZ Total', 0
UNION ALL SELECT '-', (SELECT COUNT(*) + 1 FROM class_sales)
-- COMMAND ----------
INSERT OVERWRITE TABLE dm.dm_td_aia_key_competitor_rank
WITH max_month AS (
SELECT MAX(yyyymm) AS max_yyyymm
FROM (SELECT DISTINCT yyyymm FROM dm.dm_tf_ext_aia_sales) t
),
pack_key_competitor AS (
SELECT DISTINCT L.PACK_CODE, R.KEY_COMPETITOR_CN
FROM dm.dm_td_ext_aia_market_pack_mapping l
INNER JOIN (SELECT DISTINCT
PROD_DESC,max(PROD_DESC_C) KEY_COMPETITOR_CN
FROM dm.dm_td_ext_aia_pack_property
GROUP BY 1) r on UPPER(l.KEY_COMPETITOR) = UPPER(r.PROD_DESC)
WHERE KEY_COMPETITOR <> 'OTHERS'
),
key_competitor_sales AS (
SELECT /*+ BROADCAST(pack_key_competitor) */
k.KEY_COMPETITOR_CN,
SUM(s.SALES_VALUE_CAL) AS SALES_VALUE_CAL
FROM dm.dm_tf_ext_aia_sales s
INNER JOIN pack_key_competitor k ON s.PACK_CODE = k.PACK_CODE
WHERE s.yyyymm BETWEEN CONCAT(LEFT((SELECT max_yyyymm FROM max_month), 4), '01')
AND (SELECT max_yyyymm FROM max_month)
GROUP BY k.KEY_COMPETITOR_CN
)
SELECT KEY_COMPETITOR_CN,
DENSE_RANK() OVER (ORDER BY SALES_VALUE_CAL DESC, KEY_COMPETITOR_CN) AS RANK
FROM key_competitor_sales
UNION ALL SELECT 'Market', -1
UNION ALL SELECT 'AZ Total', 0
UNION ALL SELECT 'OTHERS', (SELECT COUNT(*) + 1 FROM key_competitor_sales)
-- COMMAND ----------
--molecule的排名
insert overwrite table dm.dm_td_aia_molecule_rank
WITH max_month AS (
SELECT MAX(yyyymm) AS max_yyyymm
FROM (SELECT DISTINCT yyyymm FROM dm.dm_tf_ext_aia_sales) t
),
pack_CMPS AS (
-- 过滤前移，缩小维表体积再 BROADCAST
SELECT DISTINCT PACK_CODE, CMPS_DESC_C
FROM DM.dm_td_ext_aia_pack_property
WHERE PROD_DESC_C <> 'OTHERS'
),
prod_sales AS (
SELECT /*+ BROADCAST(pack_prod) */
p.CMPS_DESC_C,
SUM(s.SALES_VALUE_CAL) AS SALES_VALUE_CAL
FROM dm.dm_tf_ext_aia_sales s
INNER JOIN pack_CMPS p ON s.PACK_CODE = p.PACK_CODE
WHERE s.yyyymm BETWEEN CONCAT(LEFT((SELECT max_yyyymm FROM max_month), 4), '01')
AND (SELECT max_yyyymm FROM max_month)
GROUP BY p.CMPS_DESC_C
)
SELECT CMPS_DESC_C,
DENSE_RANK() OVER (ORDER BY SALES_VALUE_CAL DESC, CMPS_DESC_C) AS RANK
FROM prod_sales
UNION ALL SELECT 'Market', -1
UNION ALL SELECT 'AZ Total', 0
UNION ALL SELECT 'OTHERS', (SELECT COUNT(*) + 1 FROM prod_sales)

BIN
EXTERNAL/for_AIA_Dashboard/06 dm_td_aia_rank.sql:Zone.Identifier Normal file
View File

Binary file not shown.

111
EXTERNAL/for_AIA_Dashboard/07 dm_ext_aia_data_remove_flag.sql Normal file
View File

@@ -0,0 +1,111 @@
-- Databricks notebook source
-- MAGIC %md
-- MAGIC - dwd_gnd_aia_mark_ins_data_by_brand --> brand 转换成 market
-- MAGIC - dwd_gnd_aia_mark_ins_data_by_mkt 与 brand 的进行拼接 得到 market ins data all market
-- MAGIC - 生成 dm.dm_ext_td_mark_ins_data_remove
-- COMMAND ----------
-- DBTITLE 1,创建brand的dwd临时视图
create or replace temporary view temp_by_brand as
select
azinscode as ins_code
,upper(azbrandnamee) as brand_name
from dwd.dwd_gnd_aia_mark_ins_data_by_brand
group by 1,2
-- COMMAND ----------
-- DBTITLE 1,创建mkt 的临时dwd视图
create or replace temporary view temp_by_mkt as
select
azinscode as azinscode
,market
from dwd.dwd_gnd_aia_mark_ins_data_by_mkt
group by 1,2
-- COMMAND ----------
-- DBTITLE 1,将brand转换成market
create or replace temporary view temp_aia_data_flag as
with pack_and_brand ( --brand 转 pack
select
upper(brandnamee) brand_name
,ims_pack_cod PACK_COD --取所有code带出market
from dwd.dwd_td_imscode_azcode
where ims_pack_cod is not null and startswith(ims_pack_cod,'0')
and upper(brandnamee) in (select brand_name from temp_by_brand group by 1 )--取出在配置表中的pack
group by 1,2
)
,pack_to_market (
select
a.PACK_CODE
,a.MARKET
from dm.dm_td_ext_aia_market_pack_mapping a
where --只取 AIA中的市场
lower(a.MARKET) not like '%all market%'
group by 1,2
)
select distinct
t1.brand_name
,t1.ins_code
,t3.market
from temp_by_brand t1
left join pack_and_brand t2 on t1.brand_name = t2.brand_name
inner join pack_to_market t3 on t2.pack_cod = t3.pack_code
where t1.ins_code in (select distinct audit_code from dm.dm_tf_ext_aia_sales ) --没有销量的机构不展示
union all
select
'remove_directly_by_mkt' as brand_name
,azinscode
,market
from temp_by_mkt--dwd.dwd_gnd_aia_mark_ins_data_by_mkt
where azinscode in (select distinct audit_code from dm.dm_tf_ext_aia_sales ) --没有销量的机构不展示
GROUP BY azinscode,market
-- COMMAND ----------
-- DBTITLE 1,写入dws层
insert overwrite dws.dws_ext_aia_data_remove_flag
select
ins_code
,brand_name
,market
,from_utc_timestamp(current_timestamp,'UTC+8') etl_insert_dt
from temp_aia_data_flag
-- COMMAND ----------
-- DBTITLE 1,写入dm层并去重
insert overwrite dm.dm_ext_aia_data_remove_flag
select
ins_code
,market
,from_utc_timestamp(current_timestamp,'UTC+8') etl_insert_dt
from dws.dws_ext_aia_data_remove_flag
group by
ins_code
,market
-- COMMAND ----------
-- DBTITLE 1,DDL 语句
-- CREATE OR REPLACE TABLE dm.dm_ext_aia_data_remove_flag (
-- ins_code STRING COMMENT '机构编码',
-- market STRING COMMENT '市场',
-- etl_insert_dt TIMESTAMP COMMENT 'ETL时间')
-- USING delta
-- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_ext_aia_data_remove_flag'
-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_ext_aia_data_remove_flag';
-- CREATE OR REPLACE TABLE dws.dws_ext_aia_data_remove_flag (
-- ins_code STRING COMMENT '机构编码',
-- brand_name STRING COMMENT '产品名称，用来区分配置表来源',
-- market STRING COMMENT '市场',
-- etl_insert_dt TIMESTAMP)
-- USING delta
-- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DWS/dws_ext_aia_data_remove_flag'
-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DWS/dws_ext_aia_data_remove_flag'

BIN
EXTERNAL/for_AIA_Dashboard/07 dm_ext_aia_data_remove_flag.sql:Zone.Identifier Normal file
View File

Binary file not shown.

56
EXTERNAL/for_AIA_Dashboard/07 dm_td_aia_remove_special_ins_bkp.py Normal file
View File

@@ -0,0 +1,56 @@
# Databricks notebook source
# MAGIC %sql
# MAGIC create or replace temporary view ins_mkt_distinct
# MAGIC as
# MAGIC select distinct
# MAGIC azinscode as ins_code
# MAGIC ,market as mkt
# MAGIC from dwd.dwd_gnd_aia_mark_ins_data_by_mkt
# COMMAND ----------
# MAGIC %sql
# MAGIC create or replace temporary view mkt_pack_mapping_distinct
# MAGIC as --pack code, prod desc, market 的关系
# MAGIC select distinct
# MAGIC PACK_COD
# MAGIC ,MARKET
# MAGIC ,PROD_DES
# MAGIC from dm.dm_td_external_market_pack_mapping
# MAGIC -- where DATA_SOURCE = 'AIA(Monthly)'
# COMMAND ----------
# MAGIC %sql
# MAGIC create or replace temporary view ins_pack_result as
# MAGIC
# MAGIC select distinct
# MAGIC a.ins_code
# MAGIC ,b.PACK_COD
# MAGIC from ins_mkt_distinct a
# MAGIC left join ( --pack 和 market 的mapping
# MAGIC select
# MAGIC PACK_COD
# MAGIC ,MARKET
# MAGIC from mkt_pack_mapping_distinct
# MAGIC ) b
# MAGIC on a.mkt = b.MARKET
# COMMAND ----------
# %sql
# CREATE TABLE dm.dm_td_aia_remove_special_ins (
# ins_code STRING,
# pack_cod STRING)
# USING delta
# -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_td_aia_remove_special_ins'
# LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_td_aia_remove_special_ins'
# COMMAND ----------
# MAGIC %sql
# MAGIC insert overwrite dm.dm_td_aia_remove_special_ins
# MAGIC select
# MAGIC ins_code
# MAGIC ,pack_cod
# MAGIC from ins_pack_result

BIN
EXTERNAL/for_AIA_Dashboard/07 dm_td_aia_remove_special_ins_bkp.py:Zone.Identifier Normal file
View File

Binary file not shown.

218
EXTERNAL/for_AIA_Dashboard/08 dm_ext_aia_data_quality_flag.sql Normal file
View File

@@ -0,0 +1,218 @@
-- Databricks notebook source
-- %python
-- import pandas as pd
-- # 读取上传的新增机构
-- df = pd.read_excel('AIA_DATA_FLAG_NEWLY_ADDED_INS.xlsx')
-- sdf = spark.createDataFrame(df)
-- sdf.createOrReplaceTempView('newly_added_ins')
-- df = pd.read_excel('AIA_DATA_FLAG_LIGHT.xlsx')
-- sdf = spark.createDataFrame(df)
-- sdf.createOrReplaceTempView('light_ins')
-- COMMAND ----------
-- DBTITLE 1,创建dwd的临时视图
create or replace temporary view newly_added_ins as
select
inscode
,is_newly_added
from dwd.dwd_gnd_aia_data_flag_newly_added_ins
-- COMMAND ----------
-- DBTITLE 1,创建dwd的临时视图
create or replace temporary view light_ins as
select
brand_name
,inst_code
,inst_name
,light
from dwd.dwd_gnd_aia_data_flag_light
-- COMMAND ----------
-- DBTITLE 1,处理新增机构的标签
-- 外部的机构编码 转换成 AZ 的机构编码
create or replace temporary view temp_extinscode_2_azinscode as
select
t1.inscode
,t1.IS_NEWLY_ADDED
,nvl( t2.ins_cd_nl , t1.inscode) az_inscode
from newly_added_ins t1
left join (
select
distinct ins_cd_nl,cpa_hospital_code
from dwd.dwd_gnd_hospital_not_provided
) t2
on t1.inscode = t2.cpa_hospital_code
where t2.ins_cd_nl is not null or len(t1.inscode)<9 --存在转换表中数据不全的情况，考虑过滤掉
-- COMMAND ----------
-- DBTITLE 1,处理AIA数据质量标签表
-- 格式化一下表，将红绿转换成数字供后续的排序做铺垫
create or replace temporary view light_ins_formatted as
select
upper(brand_name) brand_name
,inst_code
-- ,inst_name
,case when trim(light) = '绿灯' then 3
when trim(light) = '黄灯' then 2
when trim(light) = '红灯' then 1
end as light_num
from light_ins
group by 1,2,3
-- COMMAND ----------
-- DBTITLE 1,AIA数据质量标签临时表
create or replace temporary view temp_aia_data_flag as
with pack_and_brand ( --brand 转 pack
select
upper(brandnamee) brand_name
,ims_pack_cod PACK_COD --取所有code带出market
from dwd.dwd_td_imscode_azcode
where ims_pack_cod is not null and startswith(ims_pack_cod,'0')
and upper(brandnamee) in (select brand_name from light_ins_formatted group by 1 )--取出在配置表中的pack
group by 1,2
)
,pack_to_market (
select
a.PACK_CODE
,MARKET
from dm.dm_td_ext_aia_market_pack_mapping a
inner join pack_and_brand b on a.PACK_CODE = b.pack_cod --取出配置表中的brand对应的 market
where lower(a.MARKET) not like '%all market%'
group by 1,2
)
select
t1.brand_name
,t1.inst_code as ins_code
,t1.light_num
,t3.market
from light_ins_formatted t1
left join pack_and_brand t2 on t1.brand_name = t2.brand_name
inner join pack_to_market t3 on t2.pack_cod = t3.pack_code
where t1.inst_code in (select distinct AUDIT_CODE from dm.dm_tf_ext_aia_sales ) --没有销量的机构不展示
group by 1,2,3,4
--增加直接剔除表中的信息（需求：剔除数据异常机构的INS+MKT均为“红灯”，其余INS+MKT根据配置表进行判断）
union all
select
'remove_directly_by_mkt' as brand_name
,azinscode
,1 as light_num
,market
from dwd.dwd_gnd_aia_mark_ins_data_by_mkt
where azinscode in (select distinct AUDIT_CODE from dm.dm_tf_ext_aia_sales ) --没有销量的机构不展示
GROUP BY azinscode,market
-- COMMAND ----------
-- DBTITLE 1,dws 层结果
insert overwrite dws.dws_ext_aia_data_quality_flag
with market_ins_light_flag (
select
ins_code
,market
,brand_name
,min(light_num) light
from temp_aia_data_flag
group by 1,2,3
-- having count( distinct light_num) >1
)
select
t1.ins_code
,t1.market
,t1.light
,case when t2.az_inscode is not null then 1 else 0 end as is_newly_added
,'AIA(Monthly)' data_source
,from_utc_timestamp(current_timestamp,'UTC+8') etl_insert_dt
from market_ins_light_flag t1
left join temp_extinscode_2_azinscode t2 on t1.ins_code = t2.az_inscode -- 是否是新增机构
-- where t2.az_inscode is not null
group by 1,2,3,4,5
-- COMMAND ----------
-- DBTITLE 1,dm层
insert overwrite dm.dm_ext_aia_data_quality_flag
select
t1.ins_code
,t2.province_name
,t2.city_name
,t1.market
,t1.light
,t1.is_newly_added
,t1.data_source
,from_utc_timestamp(current_timestamp,'UTC+8') etl_insert_dt
from dws.dws_ext_aia_data_quality_flag t1
left join dm.dm_td_institution t2 on t1.ins_code = t2.inst_code --获取机构地理信息中的省份和城市
-- COMMAND ----------
-- MAGIC %md
-- MAGIC ### pbi 语句
-- COMMAND ----------
-- with mapping(
-- select 1 as o_value,'Y' as display_value
-- union all
-- select 0,''
-- )
-- select
-- ins_code
-- ,province_name
-- ,city_name
-- ,market
-- -- ,pack_cod
-- -- ,concat(pack_cod ,data_source) pack_key
-- ,b.display_value as is_newly_added
-- ,concat(ins_code ,data_source) audit_key
-- ,case
-- when min(light) = 1 then '红灯'
-- when min(light) = 2 then '黄灯'
-- else '绿灯'
-- end as light
-- from dm.dm_ext_aia_data_quality_flag a
-- left join mapping b on a.is_newly_added = b.o_value
-- group by 1,2,3,4,5,6
-- COMMAND ----------
-- MAGIC %md
-- MAGIC ### ddl 语句
-- COMMAND ----------
-- drop table dws.dws_ext_aia_data_quality_flag
-- COMMAND ----------
-- CREATE or REPLACE TABLE dws.dws_ext_aia_data_quality_flag (
-- ins_code STRING,
-- market STRING,
-- light INT COMMENT '0 red 1 yellow 2 green',
-- is_newly_added INT COMMENT '1 newly_added 0 old',
-- data_source STRING,
-- etl_insert_dt TIMESTAMP)
-- USING delta
-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DWS/dws_ext_aia_data_quality_flag'
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DWS/dws_ext_aia_data_quality_flag'
-- COMMAND ----------
-- CREATE or REPLACE TABLE dm.dm_ext_aia_data_quality_flag (
-- ins_code STRING,
-- province_name STRING,
-- city_name STRING,
-- market STRING,
-- light INT COMMENT '0 red 1 yellow 2 green',
-- is_newly_added INT COMMENT '1 newly_added 0 old',
-- data_source STRING,
-- etl_insert_dt TIMESTAMP)
-- USING delta
-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_ext_aia_data_quality_flag'
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_ext_aia_data_quality_flag'

BIN
EXTERNAL/for_AIA_Dashboard/08 dm_ext_aia_data_quality_flag.sql:Zone.Identifier Normal file
View File

Binary file not shown.