# Databricks notebook source ### constant DATA_TYPE_MARKET = 'MARKET' DATA_TYPE_MOLECULE = 'MOLECULE' DATA_TYPE_BRAND = 'BRAND' DATA_TYPE_MULTI_MARKET = 'MULTI_MARKET' DATA_TYPE_MULTI_MOLECULE = 'MULTI_MOLECULE' DATA_TYPE_MULTI_BRAND = 'MULTI_BRAND' DATA_TYPE_DUPLICATE = 'DUPLICATE' DATA_TYPE_HEDGE_TA = 'HEDGE_TA' DATA_TYPE_HEDGE_MARKET = 'HEDGE_MARKET' DATA_TYPE_HEDGE_MOLECULE = 'HEDGE_MOLECULE' DATA_TYPE_HEDGE_BRAND = 'HEDGE_BRAND' DATA_FLAG_RAW = 0 DATA_FLAG_SUB = 1 DATA_FLAG_ADD = 2 DATA_FLAG_RATIO = 3 DATA_FLAG_AVG = 4 DATA_FLAG_HEDGE = 5 DATA_OTHER_TA = 'OTHER_TA' DATA_OTHER_MARKET = 'OTHER_MARKET' DATA_OTHER_MOLECULE = 'OTHER_MOLECULE' DATA_OTHER_BRAND = 'OTHER_BRAND' DATA_AREA_TYPE_ROC = 'ROC' DATA_LABEL_AREA_ALL = '全国' # COMMAND ---------- ############################################################START############################################################## ### STEP-1: load data to tmp table ### STEP-2: substract data from level market to brand ### STEP-3: accumulate data from level brand to market # COMMAND ---------- # MAGIC %md # MAGIC ## STEP-1: load data to tmp table # COMMAND ---------- # MAGIC %sql # MAGIC ------------------------------------------------------------------------------------- # MAGIC -- STEP-1: load data to tmp table # MAGIC -- 1.1 load LEVEL_MARKET data to tmp.tmp_retail_dtp_level_market # MAGIC ------------------------------------------------------------------------------------- # MAGIC insert overwrite table tmp.tmp_retail_dtp_level_market # MAGIC # MAGIC select # MAGIC a.yyyymm, # MAGIC a.province_city, # MAGIC b.level_ta, # MAGIC b.level_market, # MAGIC a.sales_val, # MAGIC a.sales_vol, # MAGIC 0 # MAGIC from tmp.tmp_retail_dtp_brand_rawdata a # MAGIC inner join dwd.dwd_gnd_retail_split_automatic b # MAGIC on a.ta = b.ta # MAGIC and a.market = b.market # MAGIC and a.brand_category = b.brand # MAGIC and ifnull(a.common_name, '') = ifnull(b.common_name, '') # MAGIC where b.level = 'MARKET' # COMMAND ---------- # MAGIC %sql # MAGIC ------------------------------------------------------------------------------------- # MAGIC -- STEP-1: load data to tmp table # MAGIC -- 1.2 load LEVEL_MOLECULE data to tmp.tmp_retail_dtp_level_molecule # MAGIC ------------------------------------------------------------------------------------- # MAGIC insert overwrite table tmp.tmp_retail_dtp_level_molecule # MAGIC # MAGIC select # MAGIC a.yyyymm, # MAGIC a.province_city, # MAGIC b.level_ta, # MAGIC b.level_market, # MAGIC b.level_molecule, # MAGIC a.sales_val, # MAGIC a.sales_vol, # MAGIC 0 # MAGIC from tmp.tmp_retail_dtp_brand_rawdata a # MAGIC inner join dwd.dwd_gnd_retail_split_automatic b # MAGIC on a.ta = b.ta # MAGIC and a.market = b.market # MAGIC and a.brand_category = b.brand # MAGIC and ifnull(a.common_name, '') = ifnull(b.common_name, '') # MAGIC where b.level = 'MOLECULE' # COMMAND ---------- # MAGIC %sql # MAGIC ------------------------------------------------------------------------------------- # MAGIC -- STEP-1: load data to tmp table # MAGIC -- 1.3 load LEVEL_BRAND data to tmp.tmp_retail_dtp_level_brand # MAGIC ------------------------------------------------------------------------------------- # MAGIC insert overwrite table tmp.tmp_retail_dtp_level_brand # MAGIC # MAGIC select # MAGIC a.yyyymm, # MAGIC a.province_city, # MAGIC b.level_ta, # MAGIC b.level_market, # MAGIC b.level_molecule, # MAGIC b.level_brand, # MAGIC a.sales_val, # MAGIC a.sales_vol, # MAGIC null, # MAGIC null, # MAGIC null, # MAGIC null, # MAGIC 0 # MAGIC from tmp.tmp_retail_dtp_brand_rawdata a # MAGIC inner join dwd.dwd_gnd_retail_split_automatic b # MAGIC on a.ta = b.ta # MAGIC and a.market = b.market # MAGIC and a.brand_category = b.brand # MAGIC and ifnull(a.common_name, '') = ifnull(b.common_name, '') # MAGIC where b.level = 'BRAND' # COMMAND ---------- # MAGIC %md # MAGIC ## STEP-2: substract data from level market to brand # COMMAND ---------- # MAGIC %sql # MAGIC ------------------------------------------------------------------------------------- # MAGIC -- STEP-2: substract data from level market to brand # MAGIC -- 2.1 substract level market data and add 'OTHER_MOLECULE' data # MAGIC ------------------------------------------------------------------------------------- # MAGIC # MAGIC with tmp_aggregate_market as ( # MAGIC select # MAGIC yyyymm, # MAGIC province_city, # MAGIC ta, # MAGIC market, # MAGIC sum(sales_val) as sales_val, # MAGIC sum(sales_vol) as sales_vol # MAGIC from tmp.tmp_retail_dtp_level_molecule # MAGIC group by yyyymm, province_city , ta ,market # MAGIC ),tmp_molecule as ( # MAGIC select # MAGIC a.yyyymm, # MAGIC a.province_city, # MAGIC a.ta, # MAGIC a.market, # MAGIC a.sales_val - ifnull(b.sales_val, 0) as sales_val, # MAGIC a.sales_vol - ifnull(b.sales_vol, 0) as sales_vol # MAGIC from tmp.tmp_retail_dtp_level_market a # MAGIC left join tmp_aggregate_market b # MAGIC on a.yyyymm = b.yyyymm # MAGIC and a.province_city = b.province_city # MAGIC and a.ta = b.ta # MAGIC and a.market = b.market # MAGIC ) # MAGIC # MAGIC -------append OTHER_MOLECULE to tmp.tmp_retail_dtp_level_molecule # MAGIC insert into tmp.tmp_retail_dtp_level_molecule # MAGIC select # MAGIC yyyymm, # MAGIC province_city, # MAGIC ta, # MAGIC market, # MAGIC 'OTHER_MOLECULE', # MAGIC sales_val, # MAGIC sales_vol, # MAGIC 1 # MAGIC from tmp_molecule # MAGIC where sales_val > 0 or sales_vol > 0 # COMMAND ---------- # MAGIC %sql # MAGIC ------------------------------------------------------------------------------------- # MAGIC -- STEP-2: substract data from level market to brand # MAGIC -- 2.2 substract level molecule data and add 'OTHER_BRAND' data # MAGIC ------------------------------------------------------------------------------------- # MAGIC with tmp_aggregate_molecule as ( # MAGIC select # MAGIC yyyymm, # MAGIC province_city, # MAGIC ta, # MAGIC market, # MAGIC molecule, # MAGIC sum(sales_val) as sales_val, # MAGIC sum(sales_vol) as sales_vol # MAGIC from tmp.tmp_retail_dtp_level_brand # MAGIC group by yyyymm,province_city ,ta ,market,molecule # MAGIC ),tmp_brand as ( # MAGIC select # MAGIC a.yyyymm, # MAGIC a.province_city, # MAGIC a.ta, # MAGIC a.market, # MAGIC a.molecule, # MAGIC a.sales_val - ifnull(b.sales_val, 0) as sales_val, # MAGIC a.sales_vol - ifnull(b.sales_vol, 0) as sales_vol # MAGIC from tmp.tmp_retail_dtp_level_molecule a # MAGIC left join tmp_aggregate_molecule b # MAGIC on a.yyyymm = b.yyyymm # MAGIC and a.province_city = b.province_city # MAGIC and a.ta = b.ta # MAGIC and a.market = b.market # MAGIC and a.molecule = b.molecule # MAGIC ) # MAGIC # MAGIC -------append OTHER_BRAND to tmp.tmp_retail_dtp_level_brand # MAGIC insert into tmp.tmp_retail_dtp_level_brand # MAGIC select # MAGIC yyyymm, # MAGIC province_city, # MAGIC ta, # MAGIC market, # MAGIC molecule, # MAGIC 'OTHER_BRAND', # MAGIC sales_val, # MAGIC sales_vol, # MAGIC null, # MAGIC null, # MAGIC null, # MAGIC null, # MAGIC 1 # MAGIC from tmp_brand # MAGIC where sales_val > 0 or sales_vol > 0 # COMMAND ---------- # MAGIC %md # MAGIC ## STEP-3: accumulate data from level brand to market # COMMAND ---------- # MAGIC %sql # MAGIC ------------------------------------------------------------------------------------- # MAGIC -- STEP-3: accumulate data from level brand to market # MAGIC -- 3.1 accumulate level molecule data and balance 'OTHER_BRAND' & 'OTHER_MOLECULE' data # MAGIC -- 3.1.1 accumulate level molecule data # MAGIC ------------------------------------------------------------------------------------- # MAGIC with tmp_aggregate_molecule as ( # MAGIC select # MAGIC yyyymm, # MAGIC province_city, # MAGIC ta, # MAGIC market, # MAGIC molecule, # MAGIC sum(sales_val) as sales_val, # MAGIC sum(sales_vol) as sales_vol # MAGIC from tmp.tmp_retail_dtp_level_brand # MAGIC group by yyyymm,province_city ,ta ,market,molecule # MAGIC ), tmp_append_molecule as ( # MAGIC select a.* # MAGIC from tmp_aggregate_molecule a # MAGIC where not exists ( # MAGIC select 1 # MAGIC from tmp.tmp_retail_dtp_level_molecule b # MAGIC where a.yyyymm = b.yyyymm # MAGIC and a.province_city = b.province_city # MAGIC and a.ta = b.ta # MAGIC and a.market = b.market # MAGIC and a.molecule = b.molecule # MAGIC ) # MAGIC ) # MAGIC -------append level molecule data accumulate from level brand # MAGIC insert into tmp.tmp_retail_dtp_level_molecule # MAGIC select # MAGIC yyyymm, # MAGIC province_city, # MAGIC ta, # MAGIC market, # MAGIC molecule, # MAGIC sales_val, # MAGIC sales_vol, # MAGIC 2 # MAGIC from tmp_append_molecule # COMMAND ---------- # MAGIC %sql # MAGIC ------------------------------------------------------------------------------------- # MAGIC -- STEP-3: accumulate data from level brand to market # MAGIC -- 3.1 accumulate level molecule data and balance 'OTHER_BRAND'& 'OTHER_MOLECULE' data # MAGIC -- 3.1.2 balance 'OTHER_MOLECULE' data # MAGIC ------------------------------------------------------------------------------------- # MAGIC merge into tmp.tmp_retail_dtp_level_molecule a # MAGIC using( # MAGIC select # MAGIC yyyymm, # MAGIC province_city, # MAGIC ta, # MAGIC market, # MAGIC sum(sales_val) as sales_val, # MAGIC sum(sales_vol) as sales_vol # MAGIC from tmp.tmp_retail_dtp_level_molecule # MAGIC where data_flag = 2 # MAGIC group by yyyymm,province_city,ta,market # MAGIC ) as b # MAGIC on a.yyyymm = b.yyyymm # MAGIC and a.province_city = b.province_city # MAGIC and a.ta = b.ta # MAGIC and a.market = b.market # MAGIC and a.molecule = 'OTHER_MOLECULE' # MAGIC and a.data_flag = 1 # MAGIC when matched then # MAGIC update set # MAGIC a.sales_val = a.sales_val - b.sales_val, # MAGIC a.sales_vol = a.sales_vol - b.sales_vol # MAGIC # COMMAND ---------- # MAGIC %sql # MAGIC ------------------------------------------------------------------------------------- # MAGIC -- STEP-3: accumulate data from level brand to market # MAGIC -- 3.1 accumulate level molecule data and balance 'OTHER_BRAND'& 'OTHER_MOLECULE' data # MAGIC -- 3.1.3 balance 'OTHER_BRAND' data # MAGIC ------------------------------------------------------------------------------------- # MAGIC merge into tmp.tmp_retail_dtp_level_brand a # MAGIC using( # MAGIC select # MAGIC yyyymm, # MAGIC province_city, # MAGIC ta, # MAGIC market, # MAGIC sum(sales_val) as sales_val, # MAGIC sum(sales_vol) as sales_vol # MAGIC from tmp.tmp_retail_dtp_level_molecule # MAGIC where data_flag = 2 # MAGIC group by yyyymm,province_city,ta,market # MAGIC ) as b # MAGIC on a.yyyymm = b.yyyymm # MAGIC and a.province_city = b.province_city # MAGIC and a.ta = b.ta # MAGIC and a.market = b.market # MAGIC and a.molecule = 'OTHER_MOLECULE' # MAGIC and a.brand = 'OTHER_BRAND' # MAGIC and a.data_flag = 1 # MAGIC when matched then # MAGIC update set # MAGIC a.sales_val = a.sales_val - b.sales_val, # MAGIC a.sales_vol = a.sales_vol - b.sales_vol # COMMAND ---------- # MAGIC %sql # MAGIC ------------------------------------------------------------------------------------- # MAGIC -- STEP-3: accumulate data from level brand to market # MAGIC -- 3.2 accumulate level market data and balance 'OTHER_MOLECULE' data # MAGIC -- 3.2.1 accumulate level market data # MAGIC ------------------------------------------------------------------------------------- # MAGIC # MAGIC with tmp_aggregate_market as ( # MAGIC select # MAGIC yyyymm, # MAGIC province_city, # MAGIC ta, # MAGIC market, # MAGIC sum(sales_val) as sales_val, # MAGIC sum(sales_vol) as sales_vol # MAGIC from tmp.tmp_retail_dtp_level_molecule # MAGIC group by yyyymm,province_city ,ta ,market # MAGIC ), tmp_append_market as ( # MAGIC select a.* # MAGIC from tmp_aggregate_market a # MAGIC where not exists ( # MAGIC select 1 # MAGIC from tmp.tmp_retail_dtp_level_market b # MAGIC where a.yyyymm = b.yyyymm # MAGIC and a.province_city = b.province_city # MAGIC and a.ta = b.ta # MAGIC and a.market = b.market # MAGIC ) # MAGIC ) # MAGIC -------append level market data accumulate from level molecule # MAGIC insert into tmp.tmp_retail_dtp_level_market # MAGIC select # MAGIC yyyymm, # MAGIC province_city, # MAGIC ta, # MAGIC market, # MAGIC sales_val, # MAGIC sales_vol, # MAGIC 2 # MAGIC from tmp_append_market # COMMAND ---------- # MAGIC %md # MAGIC ## STEP-4: calculate brand ratio # COMMAND ---------- # MAGIC %sql # MAGIC ------------------------------------------------------------------------------------- # MAGIC -- STEP-4: calculate brand ratio # MAGIC -- update total_val&total_vol and ratio_val&ratio_vol # MAGIC ------------------------------------------------------------------------------------- # MAGIC merge into tmp.tmp_retail_dtp_level_brand a # MAGIC using( # MAGIC select * from tmp.tmp_retail_dtp_level_brand # MAGIC where province_city = '全国' # MAGIC ) as b # MAGIC on a.yyyymm = b.yyyymm # MAGIC and a.ta = b.ta # MAGIC and a.market = b.market # MAGIC and a.molecule = b.molecule # MAGIC and a.brand = b.brand # MAGIC when matched then # MAGIC update set # MAGIC a.total_val = b.sales_val, # MAGIC a.total_vol = b.sales_vol, # MAGIC a.ratio_val = round(a.sales_val/b.sales_val,10), # MAGIC a.ratio_vol = round(a.sales_vol/b.sales_vol,10) # COMMAND ---------- ############################################################END################################################################