chenwu/REFACTOR-MA
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor CHPA 01 and 02 warehouse jobs

Browse Source
This commit is contained in:
chenwu
2026-08-20 19:39:11 +08:00
parent 8c3f45f178
commit fd1355a4fb
26 changed files with 4391 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sql/chpa/01_dwd/01_standardize_gnd_codes.sql
+60
View File
@@ -0,0 +1,60 @@
-- Databricks notebook source
-- =============================================================================
-- Purpose : Standardize GND configuration codes (zero-padding) so join keys
-- line up across the market configuration tables. Kept on the DWD
-- config tables themselves (same targets as legacy).
-- Source : dwd.dwd_gnd_ims_tblbrandratio,
-- dwd.dwd_gnd_tblmarket_bymonth,
-- dwd.dwd_gnd_ims_tblkeycompetitor,
-- dwd.dwd_gnd_ims_tblbrandtype
-- Target : dwd.dwd_gnd_ims_tblbrandratio (UPDATE),
-- dwd.dwd_gnd_tblmarket_bymonth (UPDATE),
-- dwd.dwd_gnd_ims_tblkeycompetitor (UPDATE),
-- dwd.dwd_gnd_ims_tblbrandtype (UPDATE)
-- Grain : N/A -- in-place column updates, row grain unchanged.
-- Write mode : In-place UPDATE (idempotent padding).
-- Replaces : Code-padding block of legacy CHPA/01 dwd_update.sql
-- (time-window defaults moved to
-- 05_normalize_gnd_time_windows.sql).
-- Consumers : IMS market config refresh, DWS pack-property/market and other
-- jobs joining the four updated IMS configuration tables.
-- Notes : Padding rules and literals preserved exactly:
-- - Pack_Code/PACK_COD: left-padded with zeros to 12 digits when
-- the code starts with a digit, otherwise left unchanged.
-- - Product_Code: unconditional 9-digit left zero-pad.
-- - Molecule_Code/CMPS_COD: unconditional 6-digit left zero-pad.
-- Must run before the IMS market snapshot and DWS consumers.
-- =============================================================================
-- COMMAND ----------
UPDATE dwd.dwd_gnd_ims_tblbrandratio
SET
PACK_COD = if(CAST(PACK_COD AS string) REGEXP '^[0-9]', right(concat('000000000000', CAST(PACK_COD AS string)), 12), CAST(PACK_COD AS string)),
CMPS_COD = RIGHT(concat('000000', CAST(CMPS_COD AS string)), 6)
;
-- COMMAND ----------
UPDATE dwd.dwd_gnd_tblmarket_bymonth
SET
Pack_Code = if(Pack_Code REGEXP '^[0-9]', right(concat('000000000000', Pack_Code), 12), Pack_Code),
Product_Code = RIGHT(concat('000000000', Product_Code), 9),
Molecule_Code = RIGHT(concat('000000', Molecule_Code), 6)
;
-- COMMAND ----------
UPDATE dwd.dwd_gnd_ims_tblkeycompetitor
SET
Pack_Code = if(Pack_Code REGEXP '^[0-9]', right(concat('000000000000', Pack_Code), 12), Pack_Code),
Product_Code = RIGHT(concat('000000000', Product_Code), 9),
Molecule_Code = RIGHT(concat('000000', Molecule_Code), 6)
;
-- COMMAND ----------
UPDATE dwd.dwd_gnd_ims_tblbrandtype
SET
PACK_COD = if(PACK_COD REGEXP '^[0-9]', right(concat('000000000000', PACK_COD), 12), PACK_COD)
;
sql/chpa/01_dwd/02_refresh_ims_market_config.sql
+79
View File
@@ -0,0 +1,79 @@
-- Databricks notebook source
-- =============================================================================
-- Purpose : Refresh the IMS market configuration master from the by-month
-- staging table (full snapshot, deduplicated).
-- Source : dwd.dwd_gnd_tblmarket_bymonth (codes pre-padded by
-- 01_standardize_gnd_codes.sql)
-- Target : dwd.dwd_gnd_ims_tblmarket
-- Grain : One row per distinct market definition (market_no, market, bu,
-- ATC/NFC, pack/product/molecule, corporation/manufacturer).
-- Write mode : Full refresh (INSERT OVERWRITE).
-- Replaces : IMS market snapshot block of legacy CHPA/01 dwd_update.sql.
-- Consumers : 02_dws pack-property/market jobs, DM market datasets.
-- Notes : Target/source column contract is the legacy 24-column list,
-- preserved verbatim. The target is the existing legacy config
-- master; codes must be padded before this job runs.
-- =============================================================================
-- COMMAND ----------
-- One-time DDL: the target is the existing legacy config master. If it must be
-- recreated, restore the verified 24-column legacy contract listed below; do
-- not LIKE the by-month staging table (it additionally carries starttime/
-- endtime).
-- COMMAND ----------
INSERT OVERWRITE TABLE dwd.dwd_gnd_ims_tblmarket (
market_no,
market,
bu,
atc1_code,
atc2_code,
atc3_code,
atc4_code,
nfc1_code,
nfc2_code,
nfc3_code,
pack_code,
pack_desc,
strength,
product_code,
product_desc,
molecule_code,
molecule_desc,
not_in_flag,
extend_market,
extend_market_ratio,
corporation_code,
corporation_desc,
manufacturer_code,
manufacturer_desc
)
SELECT DISTINCT
market_no,
market,
bu,
atc1_code,
atc2_code,
atc3_code,
atc4_code,
nfc1_code,
nfc2_code,
nfc3_code,
pack_code,
pack_desc,
strength,
product_code,
product_desc,
molecule_code,
molecule_desc,
not_in_flag,
extend_market,
extend_market_ratio,
corporation_code,
corporation_desc,
manufacturer_code,
manufacturer_desc
FROM dwd.dwd_gnd_tblmarket_bymonth
;
sql/chpa/01_dwd/03_refresh_retail_market_config.sql
+78
View File
@@ -0,0 +1,78 @@
-- Databricks notebook source
-- =============================================================================
-- Purpose : Refresh the retail (ext) market configuration master from the
-- by-month staging table (full snapshot, deduplicated).
-- Source : dwd.dwd_gnd_retail_tblmarket_bymonth
-- Target : dwd.DWD_gnd_ext_retail_tblmarket (legacy table name preserved,
-- including the historical uppercase DWD_ prefix)
-- Grain : One row per distinct market definition (market_no, market, bu,
-- ATC/NFC, pack/product/molecule, corporation/manufacturer).
-- Write mode : Full refresh (INSERT OVERWRITE).
-- Replaces : Retail market snapshot block of legacy CHPA/01 dwd_update.sql.
-- Consumers : retail-side market/pack jobs.
-- Notes : Target/source column contract is the legacy 24-column list,
-- preserved verbatim. Codes must be padded before this job runs.
-- =============================================================================
-- COMMAND ----------
-- One-time DDL: the target is the existing legacy config master. If it must be
-- recreated, restore the verified 24-column legacy contract listed below; do
-- not LIKE the by-month staging table (it additionally carries starttime/
-- endtime).
-- COMMAND ----------
INSERT OVERWRITE TABLE dwd.DWD_gnd_ext_retail_tblmarket (
market_no,
market,
bu,
atc1_code,
atc2_code,
atc3_code,
atc4_code,
nfc1_code,
nfc2_code,
nfc3_code,
pack_code,
pack_desc,
strength,
product_code,
product_desc,
molecule_code,
molecule_desc,
not_in_flag,
extend_market,
extend_market_ratio,
corporation_code,
corporation_desc,
manufacturer_code,
manufacturer_desc
)
SELECT DISTINCT
market_no,
market,
bu,
atc1_code,
atc2_code,
atc3_code,
atc4_code,
nfc1_code,
nfc2_code,
nfc3_code,
pack_code,
pack_desc,
strength,
product_code,
product_desc,
molecule_code,
molecule_desc,
not_in_flag,
extend_market,
extend_market_ratio,
corporation_code,
corporation_desc,
manufacturer_code,
manufacturer_desc
FROM dwd.dwd_gnd_retail_tblmarket_bymonth
;
sql/chpa/01_dwd/04_refresh_dtp_market_config.sql
+78
View File
@@ -0,0 +1,78 @@
-- Databricks notebook source
-- =============================================================================
-- Purpose : Refresh the DTP market configuration master from the by-month
-- staging table (full snapshot, deduplicated).
-- Source : dwd.dwd_gnd_dtp_tblmarket_bymonth
-- Target : DWD.dwd_gnd_dtp_tblmarket (legacy table name preserved,
-- including the historical uppercase DWD schema prefix)
-- Grain : One row per distinct market definition (market_no, market, bu,
-- ATC/NFC, pack/product/molecule, corporation/manufacturer).
-- Write mode : Full refresh (INSERT OVERWRITE).
-- Replaces : DTP market snapshot block of legacy CHPA/01 dwd_update.sql.
-- Consumers : DTP-side market/pack jobs.
-- Notes : Target/source column contract is the legacy 24-column list,
-- preserved verbatim. Codes must be padded before this job runs.
-- =============================================================================
-- COMMAND ----------
-- One-time DDL: the target is the existing legacy config master. If it must be
-- recreated, restore the verified 24-column legacy contract listed below; do
-- not LIKE the by-month staging table (it additionally carries starttime/
-- endtime).
-- COMMAND ----------
INSERT OVERWRITE TABLE DWD.dwd_gnd_dtp_tblmarket (
market_no,
market,
bu,
atc1_code,
atc2_code,
atc3_code,
atc4_code,
nfc1_code,
nfc2_code,
nfc3_code,
pack_code,
pack_desc,
strength,
product_code,
product_desc,
molecule_code,
molecule_desc,
not_in_flag,
extend_market,
extend_market_ratio,
corporation_code,
corporation_desc,
manufacturer_code,
manufacturer_desc
)
SELECT DISTINCT
market_no,
market,
bu,
atc1_code,
atc2_code,
atc3_code,
atc4_code,
nfc1_code,
nfc2_code,
nfc3_code,
pack_code,
pack_desc,
strength,
product_code,
product_desc,
molecule_code,
molecule_desc,
not_in_flag,
extend_market,
extend_market_ratio,
corporation_code,
corporation_desc,
manufacturer_code,
manufacturer_desc
FROM dwd.dwd_gnd_dtp_tblmarket_bymonth
;
sql/chpa/01_dwd/05_normalize_gnd_time_windows.sql
+95
View File
@@ -0,0 +1,95 @@
-- Databricks notebook source
-- =============================================================================
-- Purpose : Normalize the GND market/config time windows. The legacy job
-- applied two different default conventions; both are preserved
-- verbatim, including the historical difference between
-- 190001/209901 and 200001/299912 (do not unify without business
-- sign-off).
-- Source : dwd.dwd_gnd_ims_tblbrandratio,
-- dwd.dwd_gnd_tblmarket_bymonth,
-- dwd.dwd_gnd_retail_tblmarket_bymonth,
-- dwd.dwd_gnd_ec_tblmarket_bymonth,
-- dwd.dwd_gnd_dtp_tblmarket_bymonth
-- Target : the same five DWD tables (in-place UPDATE).
-- Grain : N/A -- in-place column updates, row grain unchanged.
-- Write mode : In-place UPDATE.
-- Replaces : Time-window blocks of legacy CHPA/01 dwd_update.sql.
-- Consumers : downstream market/pack jobs that filter on starttime/endtime.
-- Notes : Statement order is significant and matches legacy execution:
-- 1) brandratio defaults via CASE (UPPER(...)='ALL' ->
-- 190001/209901);
-- 2) blanket defaults 200001/299912 for all five tables where the
-- value is NULL/''/'All'.
-- Because step 1 runs first, brandratio rows originally
-- NULL/'ALL'/'All'/'all' end at 190001/209901, while '' rows end
-- at 200001/299912 -- exactly as legacy. The literal checks
-- ('All' vs UPPER(...)='ALL') are preserved case-sensitively.
-- =============================================================================
-- COMMAND ----------
UPDATE dwd.dwd_gnd_ims_tblbrandratio
SET
StartTime = CASE WHEN StartTime IS NULL OR UPPER(StartTime) = 'ALL' THEN '190001' ELSE StartTime END,
EndTime = CASE WHEN EndTime IS NULL OR UPPER(EndTime) = 'ALL' THEN '209901' ELSE EndTime END
;
-- COMMAND ----------
UPDATE DWD.dwd_gnd_tblmarket_bymonth
SET starttime = '200001'
WHERE starttime IS NULL OR starttime = '' OR starttime = 'All';
-- COMMAND ----------
UPDATE DWD.dwd_gnd_tblmarket_bymonth
SET endtime = '299912'
WHERE endtime IS NULL OR endtime = '' OR endtime = 'All';
-- COMMAND ----------
UPDATE DWD.dwd_gnd_ims_tblbrandratio
SET starttime = '200001'
WHERE starttime IS NULL OR starttime = '' OR starttime = 'All';
-- COMMAND ----------
UPDATE DWD.dwd_gnd_ims_tblbrandratio
SET endtime = '299912'
WHERE endtime IS NULL OR endtime = '' OR endtime = 'All';
-- COMMAND ----------
UPDATE DWD.dwd_gnd_retail_tblmarket_bymonth
SET starttime = '200001'
WHERE starttime IS NULL OR starttime = '' OR starttime = 'All';
-- COMMAND ----------
UPDATE DWD.dwd_gnd_retail_tblmarket_bymonth
SET endtime = '299912'
WHERE endtime IS NULL OR endtime = '' OR endtime = 'All';
-- COMMAND ----------
UPDATE DWD.dwd_gnd_ec_tblmarket_bymonth
SET starttime = '200001'
WHERE starttime IS NULL OR starttime = '' OR starttime = 'All';
-- COMMAND ----------
UPDATE DWD.dwd_gnd_ec_tblmarket_bymonth
SET endtime = '299912'
WHERE endtime IS NULL OR endtime = '' OR endtime = 'All';
-- COMMAND ----------
UPDATE DWD.dwd_gnd_dtp_tblmarket_bymonth
SET starttime = '200001'
WHERE starttime IS NULL OR starttime = '' OR starttime = 'All';
-- COMMAND ----------
UPDATE DWD.dwd_gnd_dtp_tblmarket_bymonth
SET endtime = '299912'
WHERE endtime IS NULL OR endtime = '' OR endtime = 'All';
sql/chpa/01_dwd/06_fix_ims_manufacturer_type.sql
+36
View File
@@ -0,0 +1,36 @@
-- Databricks notebook source
-- =============================================================================
-- Purpose : DWD data-quality fix: manufacturers whose ManufacturerType_ID is
-- 0 cannot be matched to a ManufacturerType. The verified subset
-- of such rows (or the rows themselves) are MNC, so 0 is remapped
-- to 2 (MNC) so reports classify them correctly.
-- Source : dwd.dwd_ims_td_manufacturer (in place)
-- Target : dwd.dwd_ims_td_manufacturer
-- Grain : N/A -- in-place column update; only rows with
-- ManufacturerType_ID = 0.
-- Write mode : In-place UPDATE.
-- Replaces : UPDATE block of legacy CHPA/01 dwd_ims_td_manufacturer_corp.sql
-- (the mapping build moved to
-- 02_dws/03_dws_ext_td_ims_manufacturer_corporation.sql).
-- Consumers : 02_dws/03_dws_ext_td_ims_manufacturer_corporation (and through
-- it the corporation/manufacturer CN dims and pack-property).
-- Notes : Historical manual fixes for manufacturer_id = '93' (SH/ escape
-- issue) are intentionally NOT reapplied -- the 20240904 review
-- concluded the upstream value no longer requires them. Kept below
-- as commented history for traceability.
-- =============================================================================
-- COMMAND ----------
UPDATE dwd.dwd_ims_td_manufacturer
SET ManufacturerType_ID = 2
WHERE ManufacturerType_ID = 0;
-- COMMAND ----------
-- Historical manual corrections (20240904 review: no longer needed; kept as
-- commented history only):
-- update dwd.dwd_ims_td_manufacturer set manufacturer_abbr = 'SH/' where manufacturer_id = '93';
-- update dwd.dwd_ims_td_manufacturer set corporation_id = '93' where manufacturer_id = '93';
-- update dwd.dwd_ims_td_manufacturer set corporation_code = '00221' where manufacturer_id = '93';
-- update dwd.dwd_ims_td_manufacturer set manufacturertype_id = '1' where manufacturer_id = '93';
sql/chpa/01_dwd/07_ingest_pharbers_province_fact.sql
+260
View File
@@ -0,0 +1,260 @@
-- Databricks notebook source
-- =============================================================================
-- Purpose : Daily ingestion of the Pharbers provincial sales fact file
-- (Pharbers_PROV_Fact*.csv) from the ADLS user-upload blob into
-- the DWD province-level sales fact.
-- Source : ADLS blob CSV files (Pharbers_PROV_Fact*.csv under
-- <...>/ODS/GND/UserUpload/<YYYY/MM/DD>/, environment-specific),
-- tmp.tmp_chpa_raw_data (staging)
-- Target : dwd.dwd_gnd_pharbers_prov_fact
-- Grain : One row per province x month x drug as provided by the source
-- file; grain to be confirmed against the CSV contract.
-- Write mode : Full refresh (INSERT OVERWRITE).
-- Replaces : Legacy CHPA/01_FB_BLOB_TO_DWD.sql (same DWD target).
-- Consumers : 02 tmp_ims_tf_fact_sales -> DM sales.
-- Notes : Two engineering fixes vs legacy (per migration doc):
-- 1) Fail fast when the day's upload path or matching files are
-- missing, instead of silently republishing stale staging.
-- 2) withColumnRenamed results are now assigned back to the frame
-- (legacy discarded them); renames are applied only to columns
-- that exist, preserving legacy tolerance of missing columns.
-- Everything else is preserved: paths, file filter regex,
-- CSV options, staging flow, NULL lineage columns, and the UTC+8
-- insert timestamp.
-- =============================================================================
-- COMMAND ----------
-- MAGIC %run ../../../Common/config
-- COMMAND ----------
-- MAGIC %md
-- MAGIC ### 从 blob 读取 csv 文件作为 CHPA 法伯省级事实表
-- COMMAND ----------
-- MAGIC %python
-- MAGIC from datetime import datetime, timedelta
-- MAGIC import pandas as pd
-- COMMAND ----------
-- MAGIC %python
-- MAGIC if ENVIRONMENT == PRD_ENVIRONMENT_VALUE:
-- MAGIC factsales_file_path_template = "abfss://master@azcdatalakeprd.dfs.core.chinacloudapi.cn/ODS/GND/UserUpload/"
-- MAGIC elif ENVIRONMENT == TEST_ENVIRONMENT_VALUE:
-- MAGIC factsales_file_path_template = "abfss://master@retaildlstoragetest.dfs.core.chinacloudapi.cn/ODS/GND/UserUpload/"
-- COMMAND ----------
-- MAGIC %python
-- MAGIC # 路径是否存在
-- MAGIC def path_exists(path):
-- MAGIC try:
-- MAGIC dbutils.fs.ls(path)
-- MAGIC return True
-- MAGIC except Exception as e:
-- MAGIC if "java.io.FileNotFoundException" in str(e):
-- MAGIC return False
-- MAGIC else:
-- MAGIC print(f"检查路径 {path} 时出错: {e}")
-- MAGIC raise
-- COMMAND ----------
-- MAGIC %python
-- MAGIC # 列出 blob 上的文件列表
-- MAGIC def list_file_name(path):
-- MAGIC first_path_list = [i.path for i in dbutils.fs.ls(path)]
-- MAGIC second_path_list = [dbutils.fs.ls(i)[0] for i in first_path_list]
-- MAGIC return second_path_list
-- COMMAND ----------
-- MAGIC %python
-- MAGIC # 从 blob 下载文件到 local
-- MAGIC def download_file(file_path, local_path):
-- MAGIC dbutils.fs.cp(file_path, local_path)
-- MAGIC print(f"已下载 {file_path} 到 {local_path}")
-- MAGIC return local_path
-- COMMAND ----------
-- MAGIC %python
-- MAGIC # 计算时间得到当天的路径
-- MAGIC current_date = datetime.utcnow() + timedelta(hours=8)
-- MAGIC date_path = current_date.strftime("%Y/%m/%d/")
-- MAGIC base_path = factsales_file_path_template + date_path
-- COMMAND ----------
-- MAGIC %md
-- MAGIC ### 获取路径下的文件名称，并挑出符合条件的文件路径
-- MAGIC - 无文件时直接失败，避免旧 staging 数据被再次覆盖到 DWD
-- COMMAND ----------
-- MAGIC %python
-- MAGIC if not path_exists(base_path):
-- MAGIC raise FileNotFoundError(f"上传路径不存在，任务失败，拒绝沿用旧 staging 数据: {base_path}")
-- MAGIC
-- MAGIC all_file_list = list_file_name(base_path)
-- MAGIC
-- MAGIC # 生成 df 来筛选内容
-- MAGIC files_df = pd.DataFrame([{
-- MAGIC 'path': f.path,
-- MAGIC 'modificationtime': f.modificationTime,
-- MAGIC 'name': f.name
-- MAGIC } for f in all_file_list])
-- MAGIC
-- MAGIC # 同名文件保留修改时间最新的一份
-- MAGIC files_df = files_df.sort_values('modificationtime', ascending=False).drop_duplicates('name').sort_index()
-- MAGIC files_df = files_df[files_df['name'].str.match(r'^Pharbers_PROV_Fact.*\.csv$')]
-- MAGIC files_df = files_df.reset_index(drop=True)
-- MAGIC
-- MAGIC if files_df.empty:
-- MAGIC raise RuntimeError("未找到符合条件的数据文件 (Pharbers_PROV_Fact*.csv)，任务失败，拒绝沿用旧 staging 数据")
-- MAGIC
-- MAGIC print(f"找到 {len(files_df)} 个符合条件的数据文件")
-- COMMAND ----------
-- MAGIC %python
-- MAGIC import os
-- MAGIC
-- MAGIC # 下载数据到 local，读取并清洗，逐文件收集
-- MAGIC df_all = []
-- MAGIC for file in files_df['path'].tolist():
-- MAGIC local_path = download_file(file, f"/Volumes/{NGBI_CATALOG}/tmp/volume_tmp/tmp/{os.path.basename(file)}")
-- MAGIC file_df = (spark.read
-- MAGIC .option("header", "true")
-- MAGIC .option("quote", '"')
-- MAGIC .option("escape", '"')
-- MAGIC .option("multiLine", "true")
-- MAGIC .option("mode", "PERMISSIVE")
-- MAGIC .csv(local_path))
-- MAGIC # 与旧脚本一致：丢弃 TA / Market 两列
-- MAGIC file_df = file_df.drop("TA", "Market")
-- MAGIC
-- MAGIC # 修复历史 bug：旧脚本调用 withColumnRenamed 后未把结果赋回原变量，
-- MAGIC # 重命名实际从未生效。这里正确赋值；且仅对确实存在的列重命名，
-- MAGIC # 保持旧任务对缺失列名的容忍，不因列不存在而失败。
-- MAGIC rename_map = {
-- MAGIC 'IMS.药品ID': 'IMS_DRUG_ID',
-- MAGIC '是否法伯编码': 'IS_HOSP_CODE',
-- MAGIC '规格': 'SPEC',
-- MAGIC '转换比': 'CONVERSION_RATIO',
-- MAGIC '剂型': 'DOSAGE_FORM',
-- MAGIC '价格': 'PRICE',
-- MAGIC }
-- MAGIC for old_name, new_name in rename_map.items():
-- MAGIC if old_name in file_df.columns:
-- MAGIC file_df = file_df.withColumnRenamed(old_name, new_name)
-- MAGIC
-- MAGIC print(f"已读取 {local_path}")
-- MAGIC df_all.append(file_df)
-- MAGIC
-- MAGIC if not df_all:
-- MAGIC raise RuntimeError("没有读取到任何数据文件，任务失败")
-- COMMAND ----------
-- MAGIC %python
-- MAGIC # 先清空 staging，避免旧数据残留，随后逐文件追加
-- MAGIC spark.sql("TRUNCATE TABLE tmp.tmp_chpa_raw_data")
-- MAGIC for num, file_df in enumerate(df_all, start=1):
-- MAGIC file_df.createOrReplaceTempView("fact_sales")
-- MAGIC spark.sql("INSERT INTO tmp.tmp_chpa_raw_data SELECT * FROM fact_sales")
-- MAGIC print(f"第{num}个")
-- COMMAND ----------
-- 全量覆盖
INSERT OVERWRITE TABLE dwd.dwd_gnd_pharbers_prov_fact (
year,
ym,
province_c,
ims_drug_id,
is_hosp_code,
prod_corp,
prod_cod,
pack_cod,
phcd,
prod_des,
cmps_des,
corp_des,
mnfl_cod,
prod_des_c,
cmps_c,
corp_des_c,
pack_des,
spec,
conversion_ratio,
dosage_form,
atc4_cod,
app1_cod,
app1_des,
app1_des_c,
app2_cod,
app2_des,
app2_des_c,
app3_cod,
app3_des,
app3_des_c,
vbp_batch,
vbp,
value,
totalunit,
countingunit,
price,
manu_des,
manu_des_c,
source_file_path,
source_file_name,
etl_insert_dt
)
SELECT
year,
ym,
province_c,
ims_drug_id,
is_hosp_code,
prod_corp,
prod_cod,
pack_cod,
phcd,
prod_des,
cmps_des,
corp_des,
mnfl_cod,
prod_des_c,
cmps_c,
corp_des_c,
pack_des,
spec,
conversion_ratio,
dosage_form,
atc4_cod,
app1_cod,
app1_des,
app1_des_c,
app2_cod,
app2_des,
app2_des_c,
app3_cod,
app3_des,
app3_des_c,
vbp_batch,
vbp,
value,
totalunit,
countingunit,
price,
manu_des,
manu_des_c,
NULL AS source_file_path,
NULL AS source_file_name,
FROM_UTC_TIMESTAMP(CURRENT_TIMESTAMP(), 'UTC+8') AS etl_insert_dt
FROM tmp.tmp_chpa_raw_data
;