MarketAnalysis-ETL/UNIONALL/DM_TF_EXT_UNIONALL_SALES_MAPPING.sql

-- Databricks notebook source
-- CREATE OR REPLACE TABLE DM.DM_TF_EXT_UNIONALL_SALES_MAPPING 
-- (
--   MARKET STRING,
--   KEY_COMPETITOR STRING,
--   CLASS STRING,
--   YYYYMM STRING,
--   PACK_COD STRING,
--   CORP_COD STRING,
--   AUDIT_COD STRING,
--   PLATFORM_TYPE STRING,
--   STORE_NAME STRING,
--   STORE_TYPE STRING,
--   REGION_TYPE STRING,
--   DATA_SOURCE STRING,
--   PACK_FLAG INT,
--   PROD_FLAG INT,
--   DTP_FLAG INT,
--   CMPS_FLAG INT,
--   NEW_CODE STRING COMMENT '主数据关联CODE',
--   INST_CODE STRING COMMENT '内部机构编码',
--   AIA_HP_FLAG INT,
--   DEPT_NAME STRING COMMENT '科室名称',
--   H_LEVEL INT,
--   REIMBURSE STRING COMMENT '报销情况',
--   REIMBURSE_TYPE STRING COMMENT '报销类型',
--   PRESCRIPTION_TYPE STRING COMMENT '处方来源',
--   VA decimal(35,10),
--   UT decimal(35,10),
--   CU decimal(35,10),
--   PT decimal(35,10),
--   SALES_UNIT_CAL decimal(35,10),
--   SALES_UNIT_CAL_LY decimal(35,10),
--   SALES_VALUE_CAL decimal(35,10),
--   SALES_VALUE_CAL_LY decimal(35,10),
--   CONUTING_UNIT decimal(35,10),
--   CONUTING_UNIT_LY decimal(35,10),
--   PDOT decimal(35,10),
--   PDOT_LY decimal(35,10),
--   PRESCRIPTION INT,
--   PRESCRIPTION_LY INT,
--   AIA_REMOVE INT,
--   MERGEDATA_FLAG STRING,
--   MERGEDATA_FLAG_MKT STRING,
--   MERGEDATA_FLAG_CHPA_DTP INT,
--   MERGEDATA_FLAG_DTP_NOROC INT,
--   MERGED_DATA_DTP_OTHERS INT,
--   TARGET_INS STRING
--   -- ,
--   -- ETL_INSERT_DT TIMESTAMP,
--   -- ETL_UPDATE_DT TIMESTAMP
--   )
-- USING delta
-- -- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales_mapping';
-- -- 上面是生产环境location，下面是测试环境location
-- LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/DM/dm_tf_ext_unionall_sales_mapping';

-- COMMAND ----------

CREATE OR REPLACE TEMPORARY VIEW ALL_EXTERNAL_SALES AS
--DTP
SELECT
  T0.MARKET,
  T0.KEY_COMPETITOR,
  T0.CLASS,
  T0.YYYYMM,
  T0.PACK_COD,
  T0.CORP_COD,
  T0.AUDIT_COD,
  T0.PLATFORM_TYPE,
  T0.STORE_NAME,
  T0.STORE_TYPE,
  T0.REGION_TYPE,
  T0.DATA_SOURCE,
  T0.PACK_FLAG,
  T0.PROD_FLAG,
  T0.DTP_FLAG,
  T0.CMPS_FLAG,
  T0.NEW_CODE,
  T0.INST_CODE,
  T0.AIA_HP_FLAG,
  T0.DEPT_NAME,
  T0.H_LEVEL,
  T0.REIMBURSE,
  T0.REIMBURSE_TYPE,
  T0.PRESCRIPTION_TYPE,
  T0.VA,
  T0.UT,
  T0.CU,
  T0.PT,
  T0.SALES_UNIT_CAL,
  T0.SALES_UNIT_CAL_LY,
  T0.SALES_VALUE_CAL,
  T0.SALES_VALUE_CAL_LY,
  T0.CONUTING_UNIT,
  T0.CONUTING_UNIT_LY,
  T0.PDOT,
  T0.PDOT_LY,
  T0.PRESCRIPTION,
  T0.PRESCRIPTION_LY,
  1 MERGED_DATA_DTP_OTHERS
FROM
  DM.DM_TF_EXT_UNIONALL_MARKET_SALES AS T0
WHERE
  T0.DATA_SOURCE = 'DTP(Quarterly)'

--Retail
UNION ALL
SELECT
  T0.MARKET,
  T0.KEY_COMPETITOR,
  T0.CLASS,
    CONCAT(
        SUBSTR(T0.YYYYMM, 1, 4),
        CASE 
            WHEN SUBSTR(T0.YYYYMM, 5, 2) IN ('01','02','03') THEN '03'
            WHEN SUBSTR(T0.YYYYMM, 5, 2) IN ('04','05','06') THEN '06'
            WHEN SUBSTR(T0.YYYYMM, 5, 2) IN ('07','08','09') THEN '09'
            ELSE '12'
        END
    ) AS YYYYMM,
  T0.PACK_COD,
  T0.CORP_COD,
  T0.AUDIT_COD,
  T0.PLATFORM_TYPE,
  T0.STORE_NAME,
  T0.STORE_TYPE,
  T0.REGION_TYPE,
  T0.DATA_SOURCE,
  T0.PACK_FLAG,
  T0.PROD_FLAG,
  T0.DTP_FLAG,
  T0.CMPS_FLAG,
  T0.NEW_CODE,
  T0.INST_CODE,
  T0.AIA_HP_FLAG,
  T0.DEPT_NAME,
  T0.H_LEVEL,
  T0.REIMBURSE,
  T0.REIMBURSE_TYPE,
  T0.PRESCRIPTION_TYPE,
  T0.VA,
  T0.UT,
  T0.CU,
  T0.PT,
  T0.SALES_UNIT_CAL,
  T0.SALES_UNIT_CAL_LY,
  T0.SALES_VALUE_CAL,
  T0.SALES_VALUE_CAL_LY,
  T0.CONUTING_UNIT,
  T0.CONUTING_UNIT_LY,
  T0.PDOT,
  T0.PDOT_LY,
  T0.PRESCRIPTION,
  T0.PRESCRIPTION_LY,
  1 MERGED_DATA_DTP_OTHERS
FROM
  DM.DM_TF_EXT_UNIONALL_MARKET_SALES AS T0
    LEFT JOIN (
      SELECT
        OTHERS_MONTH
      FROM
        tmp.tmp_dm_td_ext_retail_niad_month
    ) T1 --Retail取非niad先来数据部分
WHERE
  T0.DATA_SOURCE = 'Retail(Quarterly)'
  AND T0.YYYYMM<=T1.others_month
--EC
UNION ALL
SELECT
  T0.MARKET,
  T0.KEY_COMPETITOR,
  T0.CLASS,
    CONCAT(
        SUBSTR(T0.YYYYMM, 1, 4),
        CASE 
            WHEN SUBSTR(T0.YYYYMM, 5, 2) IN ('01','02','03') THEN '03'
            WHEN SUBSTR(T0.YYYYMM, 5, 2) IN ('04','05','06') THEN '06'
            WHEN SUBSTR(T0.YYYYMM, 5, 2) IN ('07','08','09') THEN '09'
            ELSE '12'
        END
    ) AS YYYYMM,
  T0.PACK_COD,
  T0.CORP_COD,
  T0.AUDIT_COD,
  T0.PLATFORM_TYPE,
  T0.STORE_NAME,
  T0.STORE_TYPE,
  T0.REGION_TYPE,
  T0.DATA_SOURCE,
  T0.PACK_FLAG,
  T0.PROD_FLAG,
  T0.DTP_FLAG,
  T0.CMPS_FLAG,
  T0.NEW_CODE,
  T0.INST_CODE,
  T0.AIA_HP_FLAG,
  T0.DEPT_NAME,
  T0.H_LEVEL,
  T0.REIMBURSE,
  T0.REIMBURSE_TYPE,
  T0.PRESCRIPTION_TYPE,
  T0.VA,
  T0.UT,
  T0.CU,
  T0.PT,
  T0.SALES_UNIT_CAL,
  T0.SALES_UNIT_CAL_LY,
  T0.SALES_VALUE_CAL,
  T0.SALES_VALUE_CAL_LY,
  T0.CONUTING_UNIT,
  T0.CONUTING_UNIT_LY,
  T0.PDOT,
  T0.PDOT_LY,
  T0.PRESCRIPTION,
  T0.PRESCRIPTION_LY,
  1 MERGED_DATA_DTP_OTHERS
FROM
  DM.DM_TF_EXT_UNIONALL_MARKET_SALES AS T0
    LEFT JOIN (
      SELECT
        OTHERS_MONTH
      FROM
        tmp.tmp_dm_td_ext_ec_niad_month
    ) T1 --EC取非niad先来数据部分
WHERE
  T0.DATA_SOURCE = 'EC(Monthly)'
  AND SUBSTR(T0.YYYYMM, 5, 2) IN ('03', '06', '09', '12')
--OTHERS DATA SOURCE
UNION ALL
SELECT
  T0.MARKET,
  T0.KEY_COMPETITOR,
  T0.CLASS,
  T0.YYYYMM,
  T0.PACK_COD,
  T0.CORP_COD,
  T0.AUDIT_COD,
  T0.PLATFORM_TYPE,
  T0.STORE_NAME,
  T0.STORE_TYPE,
  T0.REGION_TYPE,
  T0.DATA_SOURCE,
  T0.PACK_FLAG,
  T0.PROD_FLAG,
  T0.DTP_FLAG,
  T0.CMPS_FLAG,
  T0.NEW_CODE,
  T0.INST_CODE,
  T0.AIA_HP_FLAG,
  T0.DEPT_NAME,
  T0.H_LEVEL,
  T0.REIMBURSE,
  T0.REIMBURSE_TYPE,
  T0.PRESCRIPTION_TYPE,
  T0.VA,
  T0.UT,
  T0.CU,
  T0.PT,
  T0.SALES_UNIT_CAL,
  T0.SALES_UNIT_CAL_LY,
  T0.SALES_VALUE_CAL,
  T0.SALES_VALUE_CAL_LY,
  T0.CONUTING_UNIT,
  T0.CONUTING_UNIT_LY,
  T0.PDOT,
  T0.PDOT_LY,
  T0.PRESCRIPTION,
  T0.PRESCRIPTION_LY,
  1 MERGED_DATA_DTP_OTHERS
FROM
  DM.DM_TF_EXT_UNIONALL_MARKET_SALES AS T0
WHERE
  T0.DATA_SOURCE NOT IN ('Retail(Quarterly)', 'EC(Monthly)', 'DTP(Quarterly)')

-- COMMAND ----------

create or replace table tmp.tmp_tf_ext_aia_target_inst 
-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/TMP/tmp_tf_ext_aia_target_inst';
-- 上面是生产环境location，下面是测试环境location
LOCATION 'abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/TMP/tmp_tf_ext_aia_target_inst'
as
select distinct
  MARKET,
  AUDIT_COD
from
  dm.dm_td_ext_aia_org
where
  MR_TRTY_CODE <> 'Vacant'
UNION ALL
SELECT distinct
  'ALL Market',
  AUDIT_COD
from
  dm.dm_td_ext_aia_org
where
  MR_TRTY_CODE <> 'Vacant'

-- COMMAND ----------

CREATE OR REPLACE TEMPORARY VIEW DM_TF_EXT_UNIONALL_SALES_MAPPING
AS 
SELECT
  A.MARKET,
  A.KEY_COMPETITOR,
  A.CLASS,
  A.YYYYMM,
  A.PACK_COD          AS PACK_COD,
  A.CORP_COD          AS CORP_COD,
  A.AUDIT_COD         AS AUDIT_COD,
  A.PLATFORM_TYPE,
  A.STORE_NAME,
  A.STORE_TYPE,
  A.REGION_TYPE,
  A.DATA_SOURCE,
  A.PACK_FLAG,
  A.PROD_FLAG,
  A.DTP_FLAG,
  INT(A.CMPS_FLAG)    AS CMPS_FLAG,
  A.NEW_CODE,
  A.INST_CODE,
  B.AIA_HP_FLAG,
  A.DEPT_NAME,
  ROUND(A.H_LEVEL, 0) AS H_LEVEL,
  A.REIMBURSE,
  A.REIMBURSE_TYPE,
  A.PRESCRIPTION_TYPE,
  A.VA,
  A.UT,
  A.CU,
  A.PT,
  CASE
    WHEN A.SALES_UNIT_CAL = 0 THEN NULL
    ELSE CAST(A.SALES_UNIT_CAL AS DECIMAL(35, 10))
  END AS SALES_UNIT_CAL,
  CASE
    WHEN A.SALES_UNIT_CAL_LY = 0 THEN NULL
    ELSE CAST(A.SALES_UNIT_CAL_LY AS DECIMAL(35, 10))
  END AS SALES_UNIT_CAL_LY,
  CASE
    WHEN A.SALES_VALUE_CAL = 0 THEN NULL
    ELSE CAST(A.SALES_VALUE_CAL AS DECIMAL(35, 10))
  END AS SALES_VALUE_CAL,
  CASE
    WHEN A.SALES_VALUE_CAL_LY = 0 THEN NULL
    ELSE CAST(A.SALES_VALUE_CAL_LY AS DECIMAL(35, 10))
  END AS SALES_VALUE_CAL_LY,
  CASE
    WHEN A.CONUTING_UNIT = 0 THEN NULL
    ELSE CAST(A.CONUTING_UNIT AS DECIMAL(35, 10))
  END AS CONUTING_UNIT,
  CASE
    WHEN A.CONUTING_UNIT_LY = 0 THEN NULL
    ELSE CAST(A.CONUTING_UNIT_LY AS DECIMAL(35, 10))
  END AS CONUTING_UNIT_LY,
  CASE
    WHEN A.PDOT = 0 THEN NULL
    ELSE CAST(A.PDOT AS DECIMAL(35, 10))
  END AS PDOT,
  CASE
    WHEN A.PDOT_LY = 0 THEN NULL
    ELSE CAST(A.PDOT_LY AS DECIMAL(35, 10))
  END AS PDOT_LY,
  CASE
    WHEN A.PRESCRIPTION = 0 THEN NULL
    ELSE CAST(A.PRESCRIPTION AS DECIMAL(35, 10))
  END AS PRESCRIPTION,
  CASE
    WHEN A.PRESCRIPTION_LY = 0 THEN NULL
    ELSE CAST(A.PRESCRIPTION_LY AS DECIMAL(35, 10))
  END PRESCRIPTION_LY
  --AIA remove ins which is wrong
,
  IF (D.ins_code IS NULL, 1, 0) AS AIA_REMOVE
  --Merged Data part
,
  null as MERGEDATA_FLAG
,
  CASE
    when A.DATA_SOURCE='IQVIA-CHPA(Monthly)' and A.AUDIT_COD = 'ROC' THEN 'CHPA'
    WHEN A.DTP_FLAG = '1' THEN 'DTP'
    WHEN F.SOURCENAME IS NULL THEN 'Normal'
    ELSE F.SOURCENAME
  END AS MERGEDATA_FLAG_MKT --mark ins and pack by markettable
,
  CASE
    WHEN A.DTP_FLAG = '1' THEN 1
    WHEN A.DATA_SOURCE = 'IQVIA-CHPA(Monthly)' THEN 1
    ELSE 0
  END AS MERGEDATA_FLAG_CHPA_DTP --mark chap and dtp data
,
  CASE
    WHEN A.DTP_FLAG = '1'
    AND A.AUDIT_COD = 'ROC' THEN 0
    ELSE 1
  END AS MERGEDATA_FLAG_DTP_NOROC --mark dtp roc with 0
,
  CASE WHEN A.DATA_SOURCE<>'AIA(Monthly)' THEN 'Y'
    WHEN G.AUDIT_COD IS NULL THEN 'N'
    ELSE 'Y'
  END AS TARGET_INS    
FROM
  ALL_EXTERNAL_SALES AS A
  LEFT JOIN DM.DM_AIA_HP_FLAG AS B ON A.INST_CODE = B.INST_CODE
  AND A.DATA_SOURCE = B.DATA_SOURCE
  LEFT JOIN dm.dm_ext_aia_data_remove_flag D ON A.market = D.market
  AND A.audit_cod = D.ins_code
  LEFT JOIN dws.dws_td_ext_merge_data_config F ON A.DATA_SOURCE = F.data_source
  AND A.AUDIT_COD = F.audit_cod
  AND A.MARKET = F.market 
  LEFT JOIN tmp.tmp_tf_ext_aia_target_inst G ON
  A.MARKET = G.MARKET AND A.AUDIT_COD = G.AUDIT_COD AND A.DATA_SOURCE ='AIA(Monthly)'

-- COMMAND ----------

-- delete from dm.DM_TF_EXT_UNIONALL_SALES_MAPPING where DATA_SOURCE IN ('Retail(Quarterly)', 'EC(Monthly)')

-- COMMAND ----------

INSERT OVERWRITE TABLE DM.DM_TF_EXT_UNIONALL_SALES_MAPPING 
-- INSERT INTO TABLE DM.DM_TF_EXT_UNIONALL_SALES_MAPPING 
SELECT
  MARKET,
  KEY_COMPETITOR,
  CLASS,
  YYYYMM,
  PACK_COD,
  CORP_COD,
  AUDIT_COD,
  PLATFORM_TYPE,
  STORE_NAME,
  STORE_TYPE,
  REGION_TYPE,
  DATA_SOURCE,
  CAST(PACK_FLAG AS INT) AS PACK_FLAG,
  CAST(PROD_FLAG AS INT) AS PROD_FLAG,
  CAST(DTP_FLAG AS INT) AS DTP_FLAG,
  CAST(CMPS_FLAG AS INT) AS CMPS_FLAG,
  NEW_CODE,
  INST_CODE,
  CAST(AIA_HP_FLAG AS INT) AS AIA_HP_FLAG,
  DEPT_NAME,
  CAST(H_LEVEL AS INT) AS H_LEVEL,
  REIMBURSE,
  REIMBURSE_TYPE,
  PRESCRIPTION_TYPE,
  CAST(VA AS decimal(35,10)) AS VA,
  CAST(UT AS decimal(35,10) ) AS UT,
  CAST(CU AS decimal(35,10) ) AS CU,
  CAST(PT AS decimal(35,10) ) AS PT,
  CAST(SALES_UNIT_CAL AS decimal(35,10) ) AS SALES_UNIT_CAL,
  CAST(SALES_UNIT_CAL_LY AS decimal(35,10) ) AS SALES_UNIT_CAL_LY,
  CAST(SALES_VALUE_CAL AS decimal(35,10) ) AS SALES_VALUE_CAL,
  CAST(SALES_VALUE_CAL_LY AS decimal(35,10) ) AS SALES_VALUE_CAL_LY,
  CAST(CONUTING_UNIT AS decimal(35,10) ) AS CONUTING_UNIT,
  CAST(CONUTING_UNIT_LY AS decimal(35,10) ) AS CONUTING_UNIT_LY,
  CAST(PDOT AS decimal(35,10) ) AS PDOT,
  CAST(PDOT_LY AS decimal(35,10) ) AS PDOT_LY,
  CAST(PRESCRIPTION AS INT) AS PRESCRIPTION,
  CAST(PRESCRIPTION_LY AS INT) AS PRESCRIPTION_LY,
  CAST(AIA_REMOVE AS INT) AS AIA_REMOVE,
  null  MERGEDATA_FLAG,
  MERGEDATA_FLAG_MKT MERGEDATA_FLAG_MKT,
  CAST(MERGEDATA_FLAG_CHPA_DTP AS INT) AS MERGEDATA_FLAG_CHPA_DTP,
  CAST(MERGEDATA_FLAG_DTP_NOROC AS INT) AS MERGEDATA_FLAG_DTP_NOROC,
  null AS MERGED_DATA_DTP_OTHERS,
  TARGET_INS
  -- ,
  -- FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_INSERT_DT,
  -- FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(),'UTC+8') AS ETL_UPDATE_DT
FROM 
  DM_TF_EXT_UNIONALL_SALES_MAPPING

-- WHERE DATA_SOURCE IN ('Retail(Quarterly)', 'EC(Monthly)')