update20260427

2026-04-27 11:04:09 +08:00
commit 3e14e78af3
173 changed files with 62579 additions and 0 deletions
--- a/Merged_Data/Merged_Data_Config_table_bymonth.sql
+++ b/Merged_Data/Merged_Data_Config_table_bymonth.sql
@@ -0,0 +1,213 @@
+-- Databricks notebook source
+-- create or replace table dws.dws_td_ext_merge_data_config
+-- (
+-- market string,
+-- audit_cod string,
+-- province string,
+-- sourcename string,
+-- data_source string
+-- )
+-- USING delta
+-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dws_td_ext_merge_data_config';
+-- -- 上面是生产环境location，下面是测试环境location
+-- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dws_td_ext_merge_data_config';
+
+-- COMMAND ----------
+
+create or replace temporary view brand_province_config -- brand配置表
+as 
+select upper(`brand`) as brand, `province` as province, datasource as data_source from dwd.dwd_gnd_merge_data_brand_province
+
+-- COMMAND ----------
+
+create or replace temporary view market_province_config --market 配置表
+as 
+select  market, province, datasource as data_source from dwd.dwd_gnd_merge_data_brand_market
+
+-- COMMAND ----------
+
+-- -- insert overwrite dws.dws_td_ext_merge_data_config_special
+-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DWS/dws_external_merge_data_config_special' --prd
+-- -- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DWS/dws_external_merge_data_config_special' --test
+-- with pack_and_brand ( --brand 转 pack
+--     select 
+--       upper(brandnamee) brand_name
+--       ,ims_pack_cod PACK_COD --取所有code带出market
+--     from dwd.dwd_td_imscode_azcode
+--     where ims_pack_cod is not null and startswith(ims_pack_cod,'0')
+--     and upper(brandnamee) in (select brand from brand_province_config group by 1 )--取出在配置表中的pack
+--     group by 1,2
+--     )
+-- ,pack_to_market (
+--   select 
+--     a.PACK_CODE
+--     ,MARKET 
+--   from dm.dm_td_ext_unionall_market_pack_mapping a
+--   inner join pack_and_brand b on a.PACK_CODE = b.pack_cod --取出配置表中的brand对应的 market
+--   where a.DATASOURCE in ( 'AIA(Monthly)')--只取 AIA中的市场，chpa的市场太多
+--   and lower(a.MARKET) not like '%all market%'
+--       and market not in (
+--         'RE Extended Market'
+--       ,'RE Market'
+--       ,'RE Market (Global)'
+--       ,'Pulmicort VBP Benchmark Market'
+--       ,'Respules Market'
+--   )
+--   group by 1,2
+-- )
+-- -- ,market_to_pack ( -- 获取上述market中所有的pack
+-- --   select
+-- --     market
+-- --     ,pack_cod
+-- --   from dm.dm_td_external_market_pack_mapping a
+-- --   where DATA_SOURCE in  ( 'AIA(Monthly)','IQVIA-CHPA(Monthly)')--AIA和chpa的市场
+-- --   and market in (select market from pack_to_market group by 1)
+-- --   group by 1,2
+-- -- )
+-- ,province_to_audit (
+--     select
+--         PROVINCE_C
+--         ,AUDIT_COD
+--         ,DATA_SOURCE
+--         ,CASE WHEN DATA_SOURCE = 'AIA(Monthly)' THEN 'AIA'
+--             WHEN DATA_SOURCE = 'IQVIA-CHPA(Monthly)' THEN 'CHPA'
+--         ELSE NULL 
+--         END AS SOURCENAME
+--     from dm.dm_td_external_geo
+--     where DATA_SOURCE in ( 'AIA(Monthly)','IQVIA-CHPA(Monthly)')
+--     group by 1,2,3
+-- )
+-- ,not_selected_datasource (--因为pack2market2pack 会导致部分大市场包含所有的pack，故用配置表再删除一些不需要配置数据源的
+--   select 
+--     a.ims_pack_cod PACK_COD --取所有code
+--     ,b.province
+--   from dwd.dwd_td_imscode_azcode a
+--   inner join (
+--     select 
+--       UPPER(brand) as brand
+--       ,province
+--     from brand_province_config
+--     where data_source = '/' 
+--     group by 1,2
+--     ) b 
+--     on upper(a.brandnamee) = upper(b.brand )
+--   where a.ims_pack_cod is not null and startswith(a.ims_pack_cod,'0')
+--   group by 1,2
+--   union
+--   select 
+--     pack_cod,
+--     '天津'
+--   from market_to_pack
+--   where market= 'EGFR TKI Market'
+--   group by 1
+-- )
+-- select DATA_SOURCE,PACK_COD,AUDIT_COD
+-- from not_selected_datasource a 
+-- left join province_to_audit b on a.province = b.province_c
+-- group by 1,2,3
+
+-- COMMAND ----------
+
+INSERT OVERWRITE dws.dws_td_ext_merge_data_config
+-- create or replace table dws.dws_td_ext_merge_data_config
+-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DWS/dws_td_ext_merge_data_config' --test
+/*
+1通过brand找到pack，通过pack找到市场以市场下的所有pack
+2通过province找到对应的省份或者机构，从而获取到audit_code
+*/
+----pack
+with pack_and_brand ( --brand 转 pack
+    select 
+      upper(brandnamee) brand_name
+      ,ims_pack_cod PACK_COD --取所有code带出market
+    from dwd.dwd_td_imscode_azcode
+    where ims_pack_cod is not null and startswith(ims_pack_cod,'0')
+    and upper(brandnamee) in (select brand from brand_province_config group by 1 )--取出在配置表中的pack
+    group by 1,2
+    )
+ 
+,pack_to_market (
+  select 
+    a.PACK_CODE,
+    MARKET 
+  from dm.dm_td_ext_unionall_market_pack_mapping a
+  inner join pack_and_brand b on a.PACK_CODE = b.pack_cod --取出配置表中的brand对应的 market
+  where a.DATASOURCE in ( 'AIA(Monthly)')--只取 AIA中的市场，chpa的市场太多
+  and lower(a.MARKET) not like '%all market%'
+      and market not in (
+        'RE Extended Market'
+      ,'RE Market'
+      ,'RE Market (Global)'
+      ,'Pulmicort VBP Benchmark Market'
+      ,'Respules Market'
+  )
+  group by 1,2
+)
+,province_to_audit (
+    select
+        PROVINCE_C
+        ,AUDIT_COD
+        ,DATA_SOURCE
+        ,CASE WHEN DATA_SOURCE = 'AIA(Monthly)' THEN 'AIA'
+            WHEN DATA_SOURCE = 'IQVIA-CHPA(Monthly)' THEN 'CHPA'
+        ELSE NULL 
+        END AS SOURCENAME
+    from dm.dm_td_external_geo
+    where DATA_SOURCE in ( 'AIA(Monthly)','IQVIA-CHPA(Monthly)')
+    group by 1,2,3
+)
+
+,pack_result as (
+  select 
+    C.market
+    ,E.audit_cod
+    ,A.province
+    ,A.data_source as SOURCENAME
+    ,E.data_source
+  from brand_province_config A 
+  inner join pack_and_brand B on A.brand = B.brand_name
+  inner join pack_to_market C on B.pack_cod = C.pack_code
+  -- inner join market_to_pack D on C.market = D.market
+  inner join province_to_audit E on A.province = E.province_c AND A.data_source = E.SOURCENAME
+  where A.data_source <> '/' 
+  group by 1,2,3,4,5
+  )
+
+---- MARKET PART
+
+,market_result as (
+  select 
+    A.MARKET
+    ,C.AUDIT_COD
+    ,A.province
+    ,A.data_source as SOURCENAME
+    ,C.DATA_SOURCE
+  from market_province_config a
+  -- left join final_market_to_pack B on A.market = B.market
+  left join province_to_audit C on A.province = C.province_c AND A.data_source = C.SOURCENAME
+  where a.data_source <> '/' AND C.AUDIT_COD IS not NULL 
+  group by 1,2,3,4,5)
+
+
+----最终结果
+select 
+market,
+audit_cod,
+province,
+sourcename,
+data_source
+ FROM pack_result
+ where market NOT in (
+  select
+    market
+  from market_province_config
+  group by 1
+)
+union all
+select 
+market,
+audit_cod,
+province,
+sourcename,
+data_source
+ FROM market_result