update20260427
This commit is contained in:
421
Retail/06 split_dtp_brand_data.py
Normal file
421
Retail/06 split_dtp_brand_data.py
Normal file
@@ -0,0 +1,421 @@
|
||||
# Databricks notebook source
|
||||
### constant
|
||||
DATA_TYPE_MARKET = 'MARKET'
|
||||
DATA_TYPE_MOLECULE = 'MOLECULE'
|
||||
DATA_TYPE_BRAND = 'BRAND'
|
||||
DATA_TYPE_MULTI_MARKET = 'MULTI_MARKET'
|
||||
DATA_TYPE_MULTI_MOLECULE = 'MULTI_MOLECULE'
|
||||
DATA_TYPE_MULTI_BRAND = 'MULTI_BRAND'
|
||||
DATA_TYPE_DUPLICATE = 'DUPLICATE'
|
||||
DATA_TYPE_HEDGE_TA = 'HEDGE_TA'
|
||||
DATA_TYPE_HEDGE_MARKET = 'HEDGE_MARKET'
|
||||
DATA_TYPE_HEDGE_MOLECULE = 'HEDGE_MOLECULE'
|
||||
DATA_TYPE_HEDGE_BRAND = 'HEDGE_BRAND'
|
||||
|
||||
DATA_FLAG_RAW = 0
|
||||
DATA_FLAG_SUB = 1
|
||||
DATA_FLAG_ADD = 2
|
||||
DATA_FLAG_RATIO = 3
|
||||
DATA_FLAG_AVG = 4
|
||||
DATA_FLAG_HEDGE = 5
|
||||
|
||||
DATA_OTHER_TA = 'OTHER_TA'
|
||||
DATA_OTHER_MARKET = 'OTHER_MARKET'
|
||||
DATA_OTHER_MOLECULE = 'OTHER_MOLECULE'
|
||||
DATA_OTHER_BRAND = 'OTHER_BRAND'
|
||||
DATA_AREA_TYPE_ROC = 'ROC'
|
||||
DATA_LABEL_AREA_ALL = '全国'
|
||||
|
||||
# COMMAND ----------
|
||||
|
||||
############################################################START##############################################################
|
||||
### STEP-1: load data to tmp table
|
||||
### STEP-2: substract data from level market to brand
|
||||
### STEP-3: accumulate data from level brand to market
|
||||
|
||||
# COMMAND ----------
|
||||
|
||||
# MAGIC %md
|
||||
# MAGIC ## STEP-1: load data to tmp table
|
||||
|
||||
# COMMAND ----------
|
||||
|
||||
# MAGIC %sql
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC -- STEP-1: load data to tmp table
|
||||
# MAGIC -- 1.1 load LEVEL_MARKET data to tmp.tmp_retail_dtp_level_market
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC insert overwrite table tmp.tmp_retail_dtp_level_market
|
||||
# MAGIC
|
||||
# MAGIC select
|
||||
# MAGIC a.yyyymm,
|
||||
# MAGIC a.province_city,
|
||||
# MAGIC b.level_ta,
|
||||
# MAGIC b.level_market,
|
||||
# MAGIC a.sales_val,
|
||||
# MAGIC a.sales_vol,
|
||||
# MAGIC 0
|
||||
# MAGIC from tmp.tmp_retail_dtp_brand_rawdata a
|
||||
# MAGIC inner join dwd.dwd_gnd_retail_split_automatic b
|
||||
# MAGIC on a.ta = b.ta
|
||||
# MAGIC and a.market = b.market
|
||||
# MAGIC and a.brand_category = b.brand
|
||||
# MAGIC and ifnull(a.common_name, '') = ifnull(b.common_name, '')
|
||||
# MAGIC where b.level = 'MARKET'
|
||||
|
||||
# COMMAND ----------
|
||||
|
||||
# MAGIC %sql
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC -- STEP-1: load data to tmp table
|
||||
# MAGIC -- 1.2 load LEVEL_MOLECULE data to tmp.tmp_retail_dtp_level_molecule
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC insert overwrite table tmp.tmp_retail_dtp_level_molecule
|
||||
# MAGIC
|
||||
# MAGIC select
|
||||
# MAGIC a.yyyymm,
|
||||
# MAGIC a.province_city,
|
||||
# MAGIC b.level_ta,
|
||||
# MAGIC b.level_market,
|
||||
# MAGIC b.level_molecule,
|
||||
# MAGIC a.sales_val,
|
||||
# MAGIC a.sales_vol,
|
||||
# MAGIC 0
|
||||
# MAGIC from tmp.tmp_retail_dtp_brand_rawdata a
|
||||
# MAGIC inner join dwd.dwd_gnd_retail_split_automatic b
|
||||
# MAGIC on a.ta = b.ta
|
||||
# MAGIC and a.market = b.market
|
||||
# MAGIC and a.brand_category = b.brand
|
||||
# MAGIC and ifnull(a.common_name, '') = ifnull(b.common_name, '')
|
||||
# MAGIC where b.level = 'MOLECULE'
|
||||
|
||||
# COMMAND ----------
|
||||
|
||||
# MAGIC %sql
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC -- STEP-1: load data to tmp table
|
||||
# MAGIC -- 1.3 load LEVEL_BRAND data to tmp.tmp_retail_dtp_level_brand
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC insert overwrite table tmp.tmp_retail_dtp_level_brand
|
||||
# MAGIC
|
||||
# MAGIC select
|
||||
# MAGIC a.yyyymm,
|
||||
# MAGIC a.province_city,
|
||||
# MAGIC b.level_ta,
|
||||
# MAGIC b.level_market,
|
||||
# MAGIC b.level_molecule,
|
||||
# MAGIC b.level_brand,
|
||||
# MAGIC a.sales_val,
|
||||
# MAGIC a.sales_vol,
|
||||
# MAGIC null,
|
||||
# MAGIC null,
|
||||
# MAGIC null,
|
||||
# MAGIC null,
|
||||
# MAGIC 0
|
||||
# MAGIC from tmp.tmp_retail_dtp_brand_rawdata a
|
||||
# MAGIC inner join dwd.dwd_gnd_retail_split_automatic b
|
||||
# MAGIC on a.ta = b.ta
|
||||
# MAGIC and a.market = b.market
|
||||
# MAGIC and a.brand_category = b.brand
|
||||
# MAGIC and ifnull(a.common_name, '') = ifnull(b.common_name, '')
|
||||
# MAGIC where b.level = 'BRAND'
|
||||
|
||||
# COMMAND ----------
|
||||
|
||||
# MAGIC %md
|
||||
# MAGIC ## STEP-2: substract data from level market to brand
|
||||
|
||||
# COMMAND ----------
|
||||
|
||||
# MAGIC %sql
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC -- STEP-2: substract data from level market to brand
|
||||
# MAGIC -- 2.1 substract level market data and add 'OTHER_MOLECULE' data
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC
|
||||
# MAGIC with tmp_aggregate_market as (
|
||||
# MAGIC select
|
||||
# MAGIC yyyymm,
|
||||
# MAGIC province_city,
|
||||
# MAGIC ta,
|
||||
# MAGIC market,
|
||||
# MAGIC sum(sales_val) as sales_val,
|
||||
# MAGIC sum(sales_vol) as sales_vol
|
||||
# MAGIC from tmp.tmp_retail_dtp_level_molecule
|
||||
# MAGIC group by yyyymm, province_city , ta ,market
|
||||
# MAGIC ),tmp_molecule as (
|
||||
# MAGIC select
|
||||
# MAGIC a.yyyymm,
|
||||
# MAGIC a.province_city,
|
||||
# MAGIC a.ta,
|
||||
# MAGIC a.market,
|
||||
# MAGIC a.sales_val - ifnull(b.sales_val, 0) as sales_val,
|
||||
# MAGIC a.sales_vol - ifnull(b.sales_vol, 0) as sales_vol
|
||||
# MAGIC from tmp.tmp_retail_dtp_level_market a
|
||||
# MAGIC left join tmp_aggregate_market b
|
||||
# MAGIC on a.yyyymm = b.yyyymm
|
||||
# MAGIC and a.province_city = b.province_city
|
||||
# MAGIC and a.ta = b.ta
|
||||
# MAGIC and a.market = b.market
|
||||
# MAGIC )
|
||||
# MAGIC
|
||||
# MAGIC -------append OTHER_MOLECULE to tmp.tmp_retail_dtp_level_molecule
|
||||
# MAGIC insert into tmp.tmp_retail_dtp_level_molecule
|
||||
# MAGIC select
|
||||
# MAGIC yyyymm,
|
||||
# MAGIC province_city,
|
||||
# MAGIC ta,
|
||||
# MAGIC market,
|
||||
# MAGIC 'OTHER_MOLECULE',
|
||||
# MAGIC sales_val,
|
||||
# MAGIC sales_vol,
|
||||
# MAGIC 1
|
||||
# MAGIC from tmp_molecule
|
||||
# MAGIC where sales_val > 0 or sales_vol > 0
|
||||
|
||||
# COMMAND ----------
|
||||
|
||||
# MAGIC %sql
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC -- STEP-2: substract data from level market to brand
|
||||
# MAGIC -- 2.2 substract level molecule data and add 'OTHER_BRAND' data
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC with tmp_aggregate_molecule as (
|
||||
# MAGIC select
|
||||
# MAGIC yyyymm,
|
||||
# MAGIC province_city,
|
||||
# MAGIC ta,
|
||||
# MAGIC market,
|
||||
# MAGIC molecule,
|
||||
# MAGIC sum(sales_val) as sales_val,
|
||||
# MAGIC sum(sales_vol) as sales_vol
|
||||
# MAGIC from tmp.tmp_retail_dtp_level_brand
|
||||
# MAGIC group by yyyymm,province_city ,ta ,market,molecule
|
||||
# MAGIC ),tmp_brand as (
|
||||
# MAGIC select
|
||||
# MAGIC a.yyyymm,
|
||||
# MAGIC a.province_city,
|
||||
# MAGIC a.ta,
|
||||
# MAGIC a.market,
|
||||
# MAGIC a.molecule,
|
||||
# MAGIC a.sales_val - ifnull(b.sales_val, 0) as sales_val,
|
||||
# MAGIC a.sales_vol - ifnull(b.sales_vol, 0) as sales_vol
|
||||
# MAGIC from tmp.tmp_retail_dtp_level_molecule a
|
||||
# MAGIC left join tmp_aggregate_molecule b
|
||||
# MAGIC on a.yyyymm = b.yyyymm
|
||||
# MAGIC and a.province_city = b.province_city
|
||||
# MAGIC and a.ta = b.ta
|
||||
# MAGIC and a.market = b.market
|
||||
# MAGIC and a.molecule = b.molecule
|
||||
# MAGIC )
|
||||
# MAGIC
|
||||
# MAGIC -------append OTHER_BRAND to tmp.tmp_retail_dtp_level_brand
|
||||
# MAGIC insert into tmp.tmp_retail_dtp_level_brand
|
||||
# MAGIC select
|
||||
# MAGIC yyyymm,
|
||||
# MAGIC province_city,
|
||||
# MAGIC ta,
|
||||
# MAGIC market,
|
||||
# MAGIC molecule,
|
||||
# MAGIC 'OTHER_BRAND',
|
||||
# MAGIC sales_val,
|
||||
# MAGIC sales_vol,
|
||||
# MAGIC null,
|
||||
# MAGIC null,
|
||||
# MAGIC null,
|
||||
# MAGIC null,
|
||||
# MAGIC 1
|
||||
# MAGIC from tmp_brand
|
||||
# MAGIC where sales_val > 0 or sales_vol > 0
|
||||
|
||||
# COMMAND ----------
|
||||
|
||||
# MAGIC %md
|
||||
# MAGIC ## STEP-3: accumulate data from level brand to market
|
||||
|
||||
# COMMAND ----------
|
||||
|
||||
# MAGIC %sql
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC -- STEP-3: accumulate data from level brand to market
|
||||
# MAGIC -- 3.1 accumulate level molecule data and balance 'OTHER_BRAND' & 'OTHER_MOLECULE' data
|
||||
# MAGIC -- 3.1.1 accumulate level molecule data
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC with tmp_aggregate_molecule as (
|
||||
# MAGIC select
|
||||
# MAGIC yyyymm,
|
||||
# MAGIC province_city,
|
||||
# MAGIC ta,
|
||||
# MAGIC market,
|
||||
# MAGIC molecule,
|
||||
# MAGIC sum(sales_val) as sales_val,
|
||||
# MAGIC sum(sales_vol) as sales_vol
|
||||
# MAGIC from tmp.tmp_retail_dtp_level_brand
|
||||
# MAGIC group by yyyymm,province_city ,ta ,market,molecule
|
||||
# MAGIC ), tmp_append_molecule as (
|
||||
# MAGIC select a.*
|
||||
# MAGIC from tmp_aggregate_molecule a
|
||||
# MAGIC where not exists (
|
||||
# MAGIC select 1
|
||||
# MAGIC from tmp.tmp_retail_dtp_level_molecule b
|
||||
# MAGIC where a.yyyymm = b.yyyymm
|
||||
# MAGIC and a.province_city = b.province_city
|
||||
# MAGIC and a.ta = b.ta
|
||||
# MAGIC and a.market = b.market
|
||||
# MAGIC and a.molecule = b.molecule
|
||||
# MAGIC )
|
||||
# MAGIC )
|
||||
# MAGIC -------append level molecule data accumulate from level brand
|
||||
# MAGIC insert into tmp.tmp_retail_dtp_level_molecule
|
||||
# MAGIC select
|
||||
# MAGIC yyyymm,
|
||||
# MAGIC province_city,
|
||||
# MAGIC ta,
|
||||
# MAGIC market,
|
||||
# MAGIC molecule,
|
||||
# MAGIC sales_val,
|
||||
# MAGIC sales_vol,
|
||||
# MAGIC 2
|
||||
# MAGIC from tmp_append_molecule
|
||||
|
||||
# COMMAND ----------
|
||||
|
||||
# MAGIC %sql
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC -- STEP-3: accumulate data from level brand to market
|
||||
# MAGIC -- 3.1 accumulate level molecule data and balance 'OTHER_BRAND'& 'OTHER_MOLECULE' data
|
||||
# MAGIC -- 3.1.2 balance 'OTHER_MOLECULE' data
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC merge into tmp.tmp_retail_dtp_level_molecule a
|
||||
# MAGIC using(
|
||||
# MAGIC select
|
||||
# MAGIC yyyymm,
|
||||
# MAGIC province_city,
|
||||
# MAGIC ta,
|
||||
# MAGIC market,
|
||||
# MAGIC sum(sales_val) as sales_val,
|
||||
# MAGIC sum(sales_vol) as sales_vol
|
||||
# MAGIC from tmp.tmp_retail_dtp_level_molecule
|
||||
# MAGIC where data_flag = 2
|
||||
# MAGIC group by yyyymm,province_city,ta,market
|
||||
# MAGIC ) as b
|
||||
# MAGIC on a.yyyymm = b.yyyymm
|
||||
# MAGIC and a.province_city = b.province_city
|
||||
# MAGIC and a.ta = b.ta
|
||||
# MAGIC and a.market = b.market
|
||||
# MAGIC and a.molecule = 'OTHER_MOLECULE'
|
||||
# MAGIC and a.data_flag = 1
|
||||
# MAGIC when matched then
|
||||
# MAGIC update set
|
||||
# MAGIC a.sales_val = a.sales_val - b.sales_val,
|
||||
# MAGIC a.sales_vol = a.sales_vol - b.sales_vol
|
||||
# MAGIC
|
||||
|
||||
# COMMAND ----------
|
||||
|
||||
# MAGIC %sql
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC -- STEP-3: accumulate data from level brand to market
|
||||
# MAGIC -- 3.1 accumulate level molecule data and balance 'OTHER_BRAND'& 'OTHER_MOLECULE' data
|
||||
# MAGIC -- 3.1.3 balance 'OTHER_BRAND' data
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC merge into tmp.tmp_retail_dtp_level_brand a
|
||||
# MAGIC using(
|
||||
# MAGIC select
|
||||
# MAGIC yyyymm,
|
||||
# MAGIC province_city,
|
||||
# MAGIC ta,
|
||||
# MAGIC market,
|
||||
# MAGIC sum(sales_val) as sales_val,
|
||||
# MAGIC sum(sales_vol) as sales_vol
|
||||
# MAGIC from tmp.tmp_retail_dtp_level_molecule
|
||||
# MAGIC where data_flag = 2
|
||||
# MAGIC group by yyyymm,province_city,ta,market
|
||||
# MAGIC ) as b
|
||||
# MAGIC on a.yyyymm = b.yyyymm
|
||||
# MAGIC and a.province_city = b.province_city
|
||||
# MAGIC and a.ta = b.ta
|
||||
# MAGIC and a.market = b.market
|
||||
# MAGIC and a.molecule = 'OTHER_MOLECULE'
|
||||
# MAGIC and a.brand = 'OTHER_BRAND'
|
||||
# MAGIC and a.data_flag = 1
|
||||
# MAGIC when matched then
|
||||
# MAGIC update set
|
||||
# MAGIC a.sales_val = a.sales_val - b.sales_val,
|
||||
# MAGIC a.sales_vol = a.sales_vol - b.sales_vol
|
||||
|
||||
# COMMAND ----------
|
||||
|
||||
# MAGIC %sql
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC -- STEP-3: accumulate data from level brand to market
|
||||
# MAGIC -- 3.2 accumulate level market data and balance 'OTHER_MOLECULE' data
|
||||
# MAGIC -- 3.2.1 accumulate level market data
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC
|
||||
# MAGIC with tmp_aggregate_market as (
|
||||
# MAGIC select
|
||||
# MAGIC yyyymm,
|
||||
# MAGIC province_city,
|
||||
# MAGIC ta,
|
||||
# MAGIC market,
|
||||
# MAGIC sum(sales_val) as sales_val,
|
||||
# MAGIC sum(sales_vol) as sales_vol
|
||||
# MAGIC from tmp.tmp_retail_dtp_level_molecule
|
||||
# MAGIC group by yyyymm,province_city ,ta ,market
|
||||
# MAGIC ), tmp_append_market as (
|
||||
# MAGIC select a.*
|
||||
# MAGIC from tmp_aggregate_market a
|
||||
# MAGIC where not exists (
|
||||
# MAGIC select 1
|
||||
# MAGIC from tmp.tmp_retail_dtp_level_market b
|
||||
# MAGIC where a.yyyymm = b.yyyymm
|
||||
# MAGIC and a.province_city = b.province_city
|
||||
# MAGIC and a.ta = b.ta
|
||||
# MAGIC and a.market = b.market
|
||||
# MAGIC )
|
||||
# MAGIC )
|
||||
# MAGIC -------append level market data accumulate from level molecule
|
||||
# MAGIC insert into tmp.tmp_retail_dtp_level_market
|
||||
# MAGIC select
|
||||
# MAGIC yyyymm,
|
||||
# MAGIC province_city,
|
||||
# MAGIC ta,
|
||||
# MAGIC market,
|
||||
# MAGIC sales_val,
|
||||
# MAGIC sales_vol,
|
||||
# MAGIC 2
|
||||
# MAGIC from tmp_append_market
|
||||
|
||||
# COMMAND ----------
|
||||
|
||||
# MAGIC %md
|
||||
# MAGIC ## STEP-4: calculate brand ratio
|
||||
|
||||
# COMMAND ----------
|
||||
|
||||
# MAGIC %sql
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC -- STEP-4: calculate brand ratio
|
||||
# MAGIC -- update total_val&total_vol and ratio_val&ratio_vol
|
||||
# MAGIC -------------------------------------------------------------------------------------
|
||||
# MAGIC merge into tmp.tmp_retail_dtp_level_brand a
|
||||
# MAGIC using(
|
||||
# MAGIC select * from tmp.tmp_retail_dtp_level_brand
|
||||
# MAGIC where province_city = '全国'
|
||||
# MAGIC ) as b
|
||||
# MAGIC on a.yyyymm = b.yyyymm
|
||||
# MAGIC and a.ta = b.ta
|
||||
# MAGIC and a.market = b.market
|
||||
# MAGIC and a.molecule = b.molecule
|
||||
# MAGIC and a.brand = b.brand
|
||||
# MAGIC when matched then
|
||||
# MAGIC update set
|
||||
# MAGIC a.total_val = b.sales_val,
|
||||
# MAGIC a.total_vol = b.sales_vol,
|
||||
# MAGIC a.ratio_val = round(a.sales_val/b.sales_val,10),
|
||||
# MAGIC a.ratio_vol = round(a.sales_vol/b.sales_vol,10)
|
||||
|
||||
# COMMAND ----------
|
||||
|
||||
############################################################END################################################################
|
||||
Reference in New Issue
Block a user