new file: 01 dm_tf_external_sales.sql
new file: 02 dm_td_external_keycompatitor.sql new file: 03 dm_td_external_brand_market.sql new file: 04 dm_td_external_calendar.sql new file: 05 dm_td_external_exchangerate.sql new file: 07 dm_td_external_packinfo.sql new file: 08 dm_td_external_corp.sql new file: 09 dm_td_external_geo_type.sql new file: 11 DM_TD_EXTERNAL_MARKET_NEW.sql new file: 12 dm_td_external_org.sql new file: 13 external auth.sql new file: 14 dm_tf_external_retail_special_bkp.sql new file: AIA/01 dm_aia_pack_property.sql new file: AIA/02 dm_ext_aia_sales.sql new file: AIA/02 dm_ext_aia_sales_bakup_20230327.sql new file: AIA/03 dm_aia_flag.sql new file: AIA/04 dm_aia_provided_flag.sql new file: AIA/06 DM_TD_EXT_AIA_PACK_PROPERTY.sql new file: AIA/07 DM_TD_EXT_AIA_PACK2MARKET.sql new file: AIA/08 DM_TD_EXT_AIA_MARKET_RATIO.sql new file: AIA/09 DM_TD_EXT_AIA_MARKET_BRAND_RATIO.sql new file: AIA/10 DM_TD_EXT_AIA_MARKET_PACK_MAPPING.sql new file: AIA/11 DM_TD_EXT_AIA_TARGET_INST.sql new file: AIA/11 dm_aia_targethp_flag.sql new file: "AIA/z_03 dm_aia_flag_\345\244\207\344\273\275\347\224\250.sql" new file: CHC/01 dm_chc_pack_property.sql new file: CHC/02 DM_TF_EXT_CHC_SALES.sql new file: CHC/03 DM_TD_EXT_CHC_PACK_PROPERTY.sql new file: CHC/04 DM_TD_EXT_CHC_PACK2MARKET.sql new file: CHC/05 DM_TD_EXT_CHC_MARKET_RATIO.sql new file: CHC/06 DM_TD_EXT_CHC_MARKET_BRAND_RATIO.sql new file: CHC/07 DM_TD_EXT_CHC_MARKET_PACK_MAPPING.sql new file: CHPA/01 dwd_ims_atc_hierarchy.sql new file: CHPA/01 dwd_ims_nfc_hierarchy.sql new file: CHPA/01 dwd_ims_td_manufacturer_corp.sql new file: CHPA/01 dwd_ims_td_pack_property.sql new file: CHPA/01 dwd_update.sql new file: CHPA/01_FB_BLOB_TO_DWD.sql new file: CHPA/02 DWS_IMS_TD_GEO.sql new file: CHPA/02 dws_ims_td_atc_cn.sql new file: CHPA/02 dws_ims_td_corp_cn.sql new file: CHPA/02 dws_ims_td_date.sql new file: CHPA/02 dws_ims_td_manu_cn.sql new file: CHPA/02 dws_ims_td_market.sql new file: CHPA/02 dws_ims_td_market_ta.sql new file: CHPA/02 dws_ims_td_nfc_cn.sql new file: CHPA/02 dws_ims_td_prod_cn.sql new file: CHPA/02 tmp_ims_td_prod_tmp.sql new file: CHPA/02 tmp_ims_tf_fact_sales.sql new file: CHPA/03 DM_TD_EXT_CHPA_MARKET_BRAND_RATIO.sql new file: CHPA/03 DM_TD_EXT_CHPA_MARKET_PACK_MAPPING.sql new file: CHPA/03 DM_TD_EXT_CHPA_MARKET_RATIO.sql new file: CHPA/03 DM_TD_EXT_CHPA_PACK2MARKET.sql new file: CHPA/03 DM_TD_EXT_CHPA_PACK_PROPERTY.sql new file: CHPA/03 DM_TF_EXT_CHPA_SALES.sql new file: CHPA/03 dm_ims_td_calendar.sql new file: CHPA/03 dm_ims_td_geo.sql new file: CHPA/03 dm_ims_td_market_property.sql new file: CHPA/03 dm_ims_td_org.sql new file: CHPA/03 dm_ims_td_org_hvh.sql new file: CHPA/03 dm_ims_td_pack_property.sql new file: CHPA/03 dm_ims_tf_sales.sql new file: CHPA/03 dm_td_chpa_market_definition.sql new file: CHPA/03 dm_td_ims_city_mapping.sql new file: EC/03 ec_load_data.sql new file: EC/04 DM_TD_EXT_EC_PACK_PROPERTY.sql new file: EC/05 DM_TF_EXT_EC_SALES.sql new file: EC/06 DM_TD_EXT_EC_PACK2MARKET.sql new file: EC/07 DM_TD_EXT_EC_MARKET_RATIO.sql new file: EC/08 DM_TD_EXT_EC_MARKET_BRAND_RATIO.sql new file: EC/09 DM_TD_EXT_EC_MARKET_PACK_MAPPING.sql new file: EC/1 (ec)blob_to_dwd.sql new file: EC/2 dwd_inc_gnd_ext_ec_nationnal_pack_union_all.py new file: Merged_Data/Merged_Data_Config_table_bkp.sql new file: Merged_Data/Merged_Data_Config_table_bymonth.sql new file: Merged_Data/dm_tf_exteranl_sales_merged_data_dtp_others_bkp.sql new file: Merged_Data/dm_tf_exteranl_sales_merged_data_dtp_others_bymonth_bkp.sql new file: ORG/DM_TD_EXT_AIA_ORG.sql new file: ORG/DM_TD_EXT_CHC_ORG.sql new file: ORG/DM_TD_EXT_CHPA_ORG.sql new file: ORG/DM_TD_EXT_COUNTY_ORG.sql new file: ORG/DM_TD_EXT_EC_ORG.sql new file: ORG/DM_TD_EXT_RETAIL_ORG.sql new file: ORG/DM_TD_EXT_THC_ORG.sql new file: ORG/DM_TD_EXT_XIEHE_ORG.sql new file: OTHERS/01 dm_td_report_url.sql new file: OTHERS/02 dws_ext_email_warning.sql new file: OTHERS/external_triggered_email.py new file: Retail/01 load_tmp_data.py new file: Retail/02 split_brand_data.py new file: Retail/03 split_pack_data.py new file: Retail/04 map_to_dws_table.py new file: Retail/05 load_dtp_temp_data.py new file: Retail/06 split_dtp_brand_data.py new file: Retail/07 split_dtp_pack_data.py new file: Retail/08 map_to_dtp_dws_table.py new file: Retail/09 dwd_inc_gnd_ext_retail_nataional.py new file: Retail/10 map_to_retail_dm_table.py new file: Retail/11 map_to_overview_dm_table.py new file: Retail/12 dws_tf_external_retail_dtp_special.sql new file: Retail/13 DM_TF_EXT_RETAIL_SALES.sql new file: Retail/14 DM_TF_EXT_RETAIL_DTP_SALES.sql new file: Retail/15 DM_TD_EXT_RETAIL_PACK_PROPERTY.sql new file: Retail/16 DM_TD_EXT_RETAIL_DTP_PACK_PROPERTY.sql new file: Retail/17 DM_TD_EXT_DTP_PACK2MARKET.sql new file: Retail/17 DM_TD_EXT_RETAIL_PACK2MARKET.sql new file: Retail/18 DM_TD_EXT_DTP_MARKET_RATIO.sql new file: Retail/18 DM_TD_EXT_RETAIL_MARKET_RATIO.sql new file: Retail/19 DM_TD_EXT_DTP_MARKET_BRAND_RATIO.sql new file: Retail/19 DM_TD_EXT_RETAIL_MARKET_BRAND_RATIO.sql new file: Retail/20 DM_TD_EXT_DTP_MARKET_PACK_MAPPING.sql new file: Retail/20 DM_TD_EXT_RETAIL_MARKET_PACK_MAPPING.sql new file: "Retail/z1 dwd_inc_gnd_ext_retail_nataional_\344\275\234\345\272\237.py" new file: "Retail/z2 retail_load_data_\344\275\234\345\272\237.sql" new file: "Retail/z3 retail_overview_data_\344\275\234\345\272\237.sql" new file: THC/01 dm_thc_pack_property.sql new file: THC/02 dm_ext_thc_sales.sql new file: THC/02 dm_ext_thc_sales_bakup_20260327.sql new file: THC/03 DM_TF_EXT_THC_MARKET_SALES_CHT.sql new file: THC/04 dm_tf_external_sales_thc.sql new file: THC/05 DM_TD_EXT_THC_PACK_PROPERTY.sql new file: THC/06 DM_TD_EXT_THC_PACK2MARKET.sql new file: THC/07 DM_TD_EXT_THC_MARKET_RATIO.sql new file: THC/08 DM_TD_EXT_THC_MARKET_BRAND_RATIO.sql new file: THC/09 DM_TD_EXT_THC_MARKET_PACK_MAPPING.sql new file: UNIONALL/DM_TD_EXT_UNIONALL_MARKET_PACK_MAPPING.sql new file: UNIONALL/DM_TD_EXT_UNIONALL_PACKINFO.sql new file: UNIONALL/DM_TF_EXT_UNIONALL_MARKET_SALES.sql new file: UNIONALL/DM_TF_EXT_UNIONALL_MARKET_SALES_v1.0.sql new file: UNIONALL/DM_TF_EXT_UNIONALL_SALES.sql new file: UNIONALL/DM_TF_EXT_UNIONALL_SALES_MAPPING.sql new file: UNIONALL/DM_TF_EXT_UNIONALL_SALES_MAPPING_NIAD.sql new file: XIEHE/01 xiehe_blob_to_dwd.py new file: XIEHE/02 dm_xiehe_pack_property.sql new file: XIEHE/03 dm_ext_xiehe_geo.sql new file: XIEHE/04 dm_ext_xiehe_sales.sql new file: XIEHE/05 dm_td_xiehe_core_dept.sql new file: XIEHE/06 DM_TF_EXT_XIEHE_SALES.sql new file: XIEHE/07 DM_TD_EXT_XIEHE_PACK_PROPERTY.sql new file: XIEHE/08 DM_TD_EXT_XIEHE_PACK2MARKET.sql new file: XIEHE/09 DM_TD_EXT_XIEHE_MARKET_RATIO.sql new file: XIEHE/10 DM_TD_EXT_XIEHE_MARKET_BRAND_RATIO.sql new file: XIEHE/11 DM_TD_EXT_XIEHE_MARKET_PACK_MAPPING.sql new file: XIEHE/bkp_01 xiehe_blob2dwd.py new file: XIEHE/bkp_02 dm_ext_xiehe_sales.sql new file: XIEHE/bkp_03 dm_ext_xiehe_pack_property.sql new file: county/01 tmp_ims_county_fact_sales_sum.sql new file: county/02 tmp_imscounty_Result.sql new file: county/03 dm_ims_td_county_geo.sql new file: county/04 dws_ext_county_tf_sales.sql new file: county/05 dm_ext_county_td_pack_property.sql new file: county/06 dm_td_county_pack_region.sql new file: county/07 dm_ext_county_tf_sales_region.sql new file: county/08 DM_TD_EXT_COUNTY_PACK_PROPERTY.sql new file: county/09 DM_TF_EXT_COUNTY_SALES.sql new file: county/10 DM_TD_EXT_COUNTY_PACK2MARKET.sql new file: county/11 DM_TD_EXT_COUNTY_MARKET_RATIO.sql new file: county/12 DM_TD_EXT_COUNTY_MARKET_BRAND_RATIO.sql new file: county/13 DM_TD_EXT_COUNTY_MARKET_PACK_MAPPING.sql new file: for_AIA_Dashboard/01 dm_td_aia_inst_mkt.sql new file: for_AIA_Dashboard/02 dm_td_aia_auth_sales.sql new file: for_AIA_Dashboard/03 dm_td_aia_original_col.sql new file: for_AIA_Dashboard/04 dm_td_aia_nosales_inst.sql new file: for_AIA_Dashboard/05 dm_td_aia_is_eagle_flag.sql new file: for_AIA_Dashboard/06 dm_td_aia_rank.sql new file: for_AIA_Dashboard/07 dm_ext_aia_data_remove_flag.sql new file: for_AIA_Dashboard/07 dm_td_aia_remove_special_ins_bkp.py new file: for_AIA_Dashboard/08 dm_ext_aia_data_quality_flag.sql new file: z 01 dm_tf_external_sales.sql new file: "z 10 dm_td_external_market_pack_mapping_\344\275\234\345\272\237.sql" new file: "z 11 dm_td_external_market_\344\275\234\345\272\237.sql" new file: "\344\270\212\347\272\277\350\204\232\346\234\254.sql" new file: "\346\225\260\346\215\256\351\252\214\350\257\201.sql"
This commit is contained in:
11
OTHERS/01 dm_td_report_url.sql
Normal file
11
OTHERS/01 dm_td_report_url.sql
Normal file
@@ -0,0 +1,11 @@
|
||||
-- Databricks notebook source
|
||||
----------------------------------------------------------
|
||||
--修改时间：20241213
|
||||
--修改人：Fanxujia
|
||||
--修改内容：
|
||||
--新增配置表，里面是MA报告以及AIA报告的url链接，用于PBI上做跳转的链接
|
||||
----------------------------------------------------------
|
||||
insert overwrite table dm.dm_td_report_url
|
||||
(ReportName,URL)
|
||||
select reportname,url
|
||||
from dwd.dwd_gnd_ims_ma_aia_redirection
|
||||
110
OTHERS/02 dws_ext_email_warning.sql
Normal file
110
OTHERS/02 dws_ext_email_warning.sql
Normal file
@@ -0,0 +1,110 @@
|
||||
-- Databricks notebook source
|
||||
-- DBTITLE 1,DDL语句定义表
|
||||
-- CREATE OR REPLACE TABLE dwd.dwd_ext_email_warning (
|
||||
-- data_source STRING COMMENT '数据源名称',
|
||||
-- max_data STRING COMMENT '数据源最新的日期，YYYYMM格式',
|
||||
-- record_date STRING COMMENT '记录数据的日期',
|
||||
-- etl_timestamp TIMESTAMP COMMENT 'ETL运行时间')
|
||||
-- USING delta
|
||||
-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DWD/dwd_ext_email_warning'
|
||||
-- ;
|
||||
-- CREATE OR REPLACE TABLE dws.dws_ext_email_warning (
|
||||
-- data_source STRING COMMENT '数据源名称',
|
||||
-- max_data STRING COMMENT '数据源最新的日期，YYYYMM格式',
|
||||
-- if_update INT COMMENT '0代表没有更新，1代表更新过数据源',
|
||||
-- etl_timestamp TIMESTAMP)
|
||||
-- USING delta
|
||||
-- LOCATION 'abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/DWS/dws_ext_email_warning'
|
||||
|
||||
-- COMMAND ----------
|
||||
|
||||
-- MAGIC %md
|
||||
-- MAGIC ### dwd层
|
||||
|
||||
-- COMMAND ----------
|
||||
|
||||
-- DBTITLE 1,删除今天的数据并插入最新数据
|
||||
delete from dwd.dwd_ext_email_warning
|
||||
where record_date = date_format( from_utc_timestamp( current_timestamp,'UTC+8'),'yyyy-MM-dd');--删除今天的输入记录
|
||||
|
||||
insert into dwd.dwd_ext_email_warning
|
||||
select
|
||||
data_source
|
||||
,max(yyyymm) max_data --数据源最大日期
|
||||
,date_format( from_utc_timestamp( current_timestamp,'UTC+8'),'yyyy-MM-dd') record_date
|
||||
,from_utc_timestamp( current_timestamp,'UTC+8') etl_timestamp
|
||||
from dm.dm_tf_ext_unionall_sales
|
||||
group by DATA_SOURCE
|
||||
|
||||
-- COMMAND ----------
|
||||
|
||||
-- MAGIC %md
|
||||
-- MAGIC ### dws层
|
||||
|
||||
-- COMMAND ----------
|
||||
|
||||
-- DBTITLE 1,计算出各数据是否更新了数据
|
||||
create or replace temporary view temp_datasource_update_info as
|
||||
/*
|
||||
1 获取今天日期的各数据源的记录情况
|
||||
2 获取今天之前的上一个周期记录情况
|
||||
3 今天日期的数据源最新数据月份 与 上一个周期的数据源最新数据月份 是否相等
|
||||
*/
|
||||
with
|
||||
today_record_date (
|
||||
select
|
||||
data_source
|
||||
,max_data
|
||||
,record_date
|
||||
from dwd.dwd_ext_email_warning
|
||||
where record_date = date_format( from_utc_timestamp( current_timestamp,'UTC+8'),'yyyy-MM-dd') --取今天的数据
|
||||
)
|
||||
,previous_record_date (
|
||||
select
|
||||
data_source
|
||||
,max_data
|
||||
,record_date
|
||||
from
|
||||
(
|
||||
select
|
||||
data_source
|
||||
,max_data
|
||||
,record_date
|
||||
,row_number() over( partition by data_source order by record_date desc ) num
|
||||
from dwd.dwd_ext_email_warning
|
||||
where record_date <> date_format( from_utc_timestamp( current_timestamp,'UTC+8'),'yyyy-MM-dd')
|
||||
)
|
||||
where num = 1
|
||||
)
|
||||
select
|
||||
t1.data_source
|
||||
,t1.max_data
|
||||
,case when t1.max_data = t2.max_data then 0 else 1 end as if_update
|
||||
,from_utc_timestamp( current_timestamp,'UTC+8') etl_timestamp
|
||||
from today_record_date t1
|
||||
inner join previous_record_date t2 on t1.data_source = t2.data_source
|
||||
|
||||
|
||||
-- COMMAND ----------
|
||||
|
||||
-- DBTITLE 1,写入dws层
|
||||
INSERT OVERWRITE dws.dws_ext_email_warning
|
||||
select
|
||||
data_source
|
||||
,max_data
|
||||
,if_update
|
||||
,etl_timestamp
|
||||
from temp_datasource_update_info
|
||||
|
||||
-- COMMAND ----------
|
||||
|
||||
-- MAGIC %md
|
||||
-- MAGIC ### ads层
|
||||
|
||||
-- COMMAND ----------
|
||||
|
||||
-- select
|
||||
-- data_source
|
||||
-- ,max_data
|
||||
-- ,if_update
|
||||
-- from dws.dws_ext_email_warning
|
||||
42
OTHERS/external_triggered_email.py
Normal file
42
OTHERS/external_triggered_email.py
Normal file
@@ -0,0 +1,42 @@
|
||||
# Databricks notebook source
|
||||
import base64
|
||||
import time
|
||||
import urllib.parse
|
||||
import requests
|
||||
from Crypto.Cipher import AES
|
||||
from Crypto.Util.Padding import pad
|
||||
|
||||
# === 配置区 ===
|
||||
# Spring Boot 服务端的密钥（Base64编码）
|
||||
SECRET_KEY_BASE64 = "qkLkLHPnqNfJ5d3X6DgLpw=="
|
||||
# 接口地址
|
||||
# https://mdcuat.astrazeneca.cn/earth/app/mail/task/ma/check
|
||||
# https://mdc.astrazeneca.cn/earth/app/mail/task/ma/check
|
||||
API_URL = "https://mdc.astrazeneca.cn/earth/app/mail/task/ma/check"
|
||||
|
||||
# === AES 加密函数 ===
|
||||
def generate_aes_token(secret_key_base64: str) -> str:
|
||||
# Hutool 的 SecureUtil.aes() 默认是 AES/ECB/PKCS5Padding
|
||||
secret_key = base64.b64decode(secret_key_base64)
|
||||
cipher = AES.new(secret_key, AES.MODE_ECB)
|
||||
|
||||
# 加密当前时间戳（毫秒）
|
||||
timestamp = str(int(time.time() * 1000))
|
||||
encrypted = cipher.encrypt(pad(timestamp.encode("utf-8"), AES.block_size))
|
||||
|
||||
# 输出 Base64 编码的 token
|
||||
return base64.b64encode(encrypted).decode("utf-8")
|
||||
|
||||
# === 调用接口 ===
|
||||
def call_check_api():
|
||||
token = generate_aes_token(SECRET_KEY_BASE64)
|
||||
encoded_token = urllib.parse.quote(token)
|
||||
url = f"{API_URL}?token={encoded_token}"
|
||||
|
||||
print(f"请求URL: {url}")
|
||||
response = requests.get(url)
|
||||
print("响应状态码:", response.status_code)
|
||||
print("响应内容:", response.text)
|
||||
|
||||
if __name__ == "__main__":
|
||||
call_check_api()
|
||||
Reference in New Issue
Block a user