V1.0
This commit is contained in:
commit
9bff0a7c7b
2
.gitattributes
vendored
Normal file
2
.gitattributes
vendored
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
# Auto detect text files and perform LF normalization
|
||||||
|
* text=auto
|
BIN
Evaluate_Result/admin_example_1.xlsx
Normal file
BIN
Evaluate_Result/admin_example_1.xlsx
Normal file
Binary file not shown.
BIN
Evaluate_Result/admin_example_2.xlsx
Normal file
BIN
Evaluate_Result/admin_example_2.xlsx
Normal file
Binary file not shown.
BIN
ExamplePBIX/Examples.pbix
Normal file
BIN
ExamplePBIX/Examples.pbix
Normal file
Binary file not shown.
1
Export_Json/example.json
Normal file
1
Export_Json/example.json
Normal file
@ -0,0 +1 @@
|
|||||||
|
{"version":"1.1.0","events":[{"name":"User Action","component":"Report Canvas","start":"2024-12-21T17:14:40.987Z","id":"515c279a49002a814996","metrics":{"sourceLabel":"UserAction_StartedMonitoring"}},{"name":"User Action","component":"Report Canvas","start":"2024-12-21T17:14:42.623Z","id":"79b174c0d02ca69ada00","metrics":{"sourceLabel":"UserAction_Refresh"}},{"name":"Visual Container Lifecycle","start":"2024-12-21T17:14:42.629Z","end":"2024-12-21T17:14:42.682Z","id":"24c30acf43d0ab30dd40-b004a31d5ccd6a4a5385","metrics":{"status":1,"visualTitle":"Table","visualId":"24c30acf43d0ab30dd40","visualType":"tableEx","initialLoad":false}},{"name":"Resolve Parameters","start":"2024-12-21T17:14:42.630Z","end":"2024-12-21T17:14:42.631Z","id":"865f086f-a553-4057-9963-f386c6ea310f","parentId":"24c30acf43d0ab30dd40-b004a31d5ccd6a4a5385"},{"name":"Query","start":"2024-12-21T17:14:42.631Z","end":"2024-12-21T17:14:42.662Z","id":"f374c5d4-aeae-49e4-a63f-a086480fbad1","parentId":"24c30acf43d0ab30dd40-b004a31d5ccd6a4a5385"},{"name":"Render","start":"2024-12-21T17:14:42.661Z","end":"2024-12-21T17:14:42.682Z","id":"bf797551-b336-43c1-8a29-eda4f83f224e","parentId":"24c30acf43d0ab30dd40-b004a31d5ccd6a4a5385"},{"name":"Data View Transform","start":"2024-12-21T17:14:42.670Z","end":"2024-12-21T17:14:42.673Z","id":"2b79f57f-9353-48e9-afe8-0c1f446f8636","parentId":"bf797551-b336-43c1-8a29-eda4f83f224e"},{"name":"Visual Update Async","start":"2024-12-21T17:14:42.673Z","end":"2024-12-21T17:14:42.682Z","id":"ca7f9db7-b9c5-4739-aced-6a8a5560f0a8","parentId":"bf797551-b336-43c1-8a29-eda4f83f224e"},{"name":"Visual Update","start":"2024-12-21T17:14:42.673Z","end":"2024-12-21T17:14:42.673Z","id":"29d7c7a6-ef0d-416a-973f-cf4ddacedc8a","parentId":"bf797551-b336-43c1-8a29-eda4f83f224e"},{"name":"Execute Semantic Query","component":"DSE","start":"2024-12-21T17:14:42.649Z","end":"2024-12-21T17:14:42.656Z","id":"47aeb947-80d7-4096-9f2d-3b16c8b5b5e9","parentId":"f374c5d4-aeae-49e4-a63f-a086480fbad1"},{"name":"Query Generation","start":"2024-12-21T17:14:42.633Z","end":"2024-12-21T17:14:42.634Z","id":"4fa46d04-aa1d-46ea-933b-49002a24c309","parentId":"f374c5d4-aeae-49e4-a63f-a086480fbad1"},{"name":"Query Pending","start":"2024-12-21T17:14:42.648Z","end":"2024-12-21T17:14:42.649Z","id":"178f1281-20eb-4c78-a3d1-bdc9f6735a7c","parentId":"f374c5d4-aeae-49e4-a63f-a086480fbad1"},{"name":"Parse Query Result","start":"2024-12-21T17:14:42.658Z","end":"2024-12-21T17:14:42.660Z","id":"33e79f0c-bc13-4951-a18e-6fdbd767e4f8","parentId":"f374c5d4-aeae-49e4-a63f-a086480fbad1"},{"name":"Execute DAX Query","component":"DSE","start":"2024-12-21T17:14:42.651Z","end":"2024-12-21T17:14:42.655Z","id":"e458b6a8-0b2b-474a-a8f1-294856878497","parentId":"47aeb947-80d7-4096-9f2d-3b16c8b5b5e9","metrics":{"QueryText":"DEFINE\r\n\tVAR __DS0Core = \r\n\t\tSUMMARIZECOLUMNS(\r\n\t\t\tROLLUPADDISSUBTOTAL('financials'[Country], \"IsGrandTotalRowTotal\"),\r\n\t\t\t\"Sumv_Sales\", CALCULATE(SUM('financials'[ Sales]))\r\n\t\t)\r\n\r\n\tVAR __DS0PrimaryWindowed = \r\n\t\tTOPN(502, __DS0Core, [IsGrandTotalRowTotal], 0, 'financials'[Country], 1)\r\n\r\nEVALUATE\r\n\t__DS0PrimaryWindowed\r\n\r\nORDER BY\r\n\t[IsGrandTotalRowTotal] DESC, 'financials'[Country]","RowCount":6}},{"name":"Execute Query","component":"AS","start":"2024-12-21T17:14:42.653Z","end":"2024-12-21T17:14:42.657Z","id":"3BC9FA84-D491-49ED-B2A1-7DC0D4F32A4C","parentId":"e458b6a8-0b2b-474a-a8f1-294856878497"},{"name":"Serialize Rowset","component":"AS","start":"2024-12-21T17:14:42.657Z","end":"2024-12-21T17:14:42.657Z","id":"57BE528D-04A5-4696-A0AF-3327CD0CE702","parentId":"3BC9FA84-D491-49ED-B2A1-7DC0D4F32A4C"}],"sessionId":"14d9da6f-0492-428d-bcff-ece9e2ac26f6"}
|
Binary file not shown.
BIN
__pycache__/app.cpython-312.pyc
Normal file
BIN
__pycache__/app.cpython-312.pyc
Normal file
Binary file not shown.
BIN
__pycache__/compareData.cpython-312.pyc
Normal file
BIN
__pycache__/compareData.cpython-312.pyc
Normal file
Binary file not shown.
BIN
__pycache__/config.cpython-312.pyc
Normal file
BIN
__pycache__/config.cpython-312.pyc
Normal file
Binary file not shown.
BIN
__pycache__/getDataFromAS.cpython-312.pyc
Normal file
BIN
__pycache__/getDataFromAS.cpython-312.pyc
Normal file
Binary file not shown.
BIN
__pycache__/getQueries.cpython-312.pyc
Normal file
BIN
__pycache__/getQueries.cpython-312.pyc
Normal file
Binary file not shown.
BIN
__pycache__/writeDataToExcel.cpython-312.pyc
Normal file
BIN
__pycache__/writeDataToExcel.cpython-312.pyc
Normal file
Binary file not shown.
138
app.py
Normal file
138
app.py
Normal file
@ -0,0 +1,138 @@
|
|||||||
|
from flask import Flask, request, render_template, redirect, url_for
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
import config
|
||||||
|
from getDataFromAS import ASDataFetcher
|
||||||
|
from getQueries import getQueries_From_json
|
||||||
|
from compareData import Comparator
|
||||||
|
|
||||||
|
app = Flask(__name__)
|
||||||
|
|
||||||
|
result = ""
|
||||||
|
|
||||||
|
@app.route('/')
|
||||||
|
def index():
|
||||||
|
global result
|
||||||
|
return render_template('index.html', config=config, result=result)
|
||||||
|
|
||||||
|
@app.route('/update_config', methods=['POST'])
|
||||||
|
def update_config():
|
||||||
|
for key, value in request.form.items():
|
||||||
|
if hasattr(config, key) and key not in ['CURRENT_DIR_PATH', 'os', 'NET_FOLDER']:
|
||||||
|
setattr(config, key, value)
|
||||||
|
return redirect(url_for('index'))
|
||||||
|
|
||||||
|
@app.route('/run', methods=['POST'])
|
||||||
|
def run():
|
||||||
|
global result
|
||||||
|
result = ""
|
||||||
|
|
||||||
|
# 拼接导出页面查询json文件夹路径
|
||||||
|
json_folder_path = os.path.join(config.CURRENT_DIR_PATH, "Export_Json")
|
||||||
|
# 拼接查询结果excel文件夹路径
|
||||||
|
result_folder_path = os.path.join(config.CURRENT_DIR_PATH, "Evaluate_Result")
|
||||||
|
# 检查文件夹是否存在,如果不存在则创建它
|
||||||
|
if not os.path.exists(json_folder_path):
|
||||||
|
os.makedirs(json_folder_path)
|
||||||
|
if not os.path.exists(result_folder_path):
|
||||||
|
os.makedirs(result_folder_path)
|
||||||
|
|
||||||
|
# 拼接导出页面查询json文件路径
|
||||||
|
export_page_json_path_1 = os.path.join(json_folder_path, config.EXPORT_PAGE_JSON_NAME_1)
|
||||||
|
export_page_json_path_2 = os.path.join(json_folder_path, config.EXPORT_PAGE_JSON_NAME_2)
|
||||||
|
# 读取json文件中的查询
|
||||||
|
queries_1 = getQueries_From_json(export_page_json_path_1)
|
||||||
|
queries_2 = getQueries_From_json(export_page_json_path_2)
|
||||||
|
|
||||||
|
# 创建 ASDataFetcher 实例
|
||||||
|
fetcher_1 = ASDataFetcher(
|
||||||
|
workspace=config.WORKSPACE_1,
|
||||||
|
username=config.USERNAME_1,
|
||||||
|
password=config.PASSWORD_1,
|
||||||
|
queries=queries_1,
|
||||||
|
json_name=config.EXPORT_PAGE_JSON_NAME_1,
|
||||||
|
result_folder_path=result_folder_path,
|
||||||
|
isAdmin=config.IS_ADMIN_1,
|
||||||
|
catalog=config.CATALOG_1,
|
||||||
|
effective_username=config.EFFECTIVE_USERNAME_1,
|
||||||
|
customdata=config.CUSTOMER_DATA_1,
|
||||||
|
role=config.ROLE_1,
|
||||||
|
model_number=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 创建 ASDataFetcher 实例
|
||||||
|
fetcher_2 = ASDataFetcher(
|
||||||
|
workspace=config.WORKSPACE_2,
|
||||||
|
username=config.USERNAME_2,
|
||||||
|
password=config.PASSWORD_2,
|
||||||
|
queries=queries_2,
|
||||||
|
json_name=config.EXPORT_PAGE_JSON_NAME_2,
|
||||||
|
result_folder_path=result_folder_path,
|
||||||
|
isAdmin=config.IS_ADMIN_2,
|
||||||
|
catalog=config.CATALOG_2,
|
||||||
|
effective_username=config.EFFECTIVE_USERNAME_2,
|
||||||
|
customdata=config.CUSTOMER_DATA_2,
|
||||||
|
role=config.ROLE_2,
|
||||||
|
model_number=2,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 定义线程任务
|
||||||
|
def run_fetcher(fetcher, print_lock):
|
||||||
|
global result
|
||||||
|
try:
|
||||||
|
fetcher.writeToExcel(print_lock)
|
||||||
|
with print_lock:
|
||||||
|
result += f"\n-------------------------\n{fetcher.json_name} 查询完成\n"
|
||||||
|
result += "\n".join(fetcher.log_messages) + "\n"
|
||||||
|
except Exception as e:
|
||||||
|
with print_lock:
|
||||||
|
result += f"错误提示: {e}\n"
|
||||||
|
result += "\n".join(fetcher.log_messages) + "\n"
|
||||||
|
return
|
||||||
|
|
||||||
|
# 创建锁
|
||||||
|
print_lock = threading.Lock()
|
||||||
|
|
||||||
|
# 创建线程
|
||||||
|
thread_1 = threading.Thread(target=run_fetcher, args=(fetcher_1, print_lock))
|
||||||
|
thread_2 = threading.Thread(target=run_fetcher, args=(fetcher_2, print_lock))
|
||||||
|
|
||||||
|
try:
|
||||||
|
result += "---*********************************************************---\n"
|
||||||
|
result += " 开始查询\n"
|
||||||
|
result += "---*********************************************************---\n"
|
||||||
|
|
||||||
|
# 启动线程
|
||||||
|
thread_1.start()
|
||||||
|
thread_2.start()
|
||||||
|
|
||||||
|
# 等待线程完成
|
||||||
|
thread_1.join()
|
||||||
|
thread_2.join()
|
||||||
|
|
||||||
|
result += "\n---*********************************************************---\n"
|
||||||
|
result += " 查询完成\n"
|
||||||
|
result += "---*********************************************************---\n"
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
result += f"错误提示: {e}\n"
|
||||||
|
return redirect(url_for('index'))
|
||||||
|
|
||||||
|
try:
|
||||||
|
result += "\n---------------------------------------------------------------\n"
|
||||||
|
result += " 开始比较\n"
|
||||||
|
result += "---------------------------------------------------------------\n"
|
||||||
|
|
||||||
|
comparator = Comparator(fetcher_1.result_full_excel_name, fetcher_2.result_full_excel_name)
|
||||||
|
comparator.compare_ExcelFiles()
|
||||||
|
result += "\n".join(comparator.log_messages) + "\n"
|
||||||
|
result += "\n---------------------------------------------------------------\n"
|
||||||
|
result += " 比较完成\n"
|
||||||
|
result += "---------------------------------------------------------------\n"
|
||||||
|
except Exception as e:
|
||||||
|
result += f"错误提示: {e}\n"
|
||||||
|
|
||||||
|
return redirect(url_for('index'))
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
app.run(debug=True)
|
65
compareData.py
Normal file
65
compareData.py
Normal file
@ -0,0 +1,65 @@
|
|||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
|
||||||
|
class Comparator:
|
||||||
|
def __init__(self, file1, file2):
|
||||||
|
self.file1 = file1
|
||||||
|
self.file2 = file2
|
||||||
|
self.log_messages = []
|
||||||
|
|
||||||
|
def compare_ExcelFiles(self):
|
||||||
|
# 读取Excel文件
|
||||||
|
xls1 = pd.ExcelFile(f"{self.file1}.xlsx", engine="openpyxl")
|
||||||
|
xls2 = pd.ExcelFile(f"{self.file2}.xlsx", engine="openpyxl")
|
||||||
|
|
||||||
|
# 获取所有工作表的名称
|
||||||
|
sheet_names1 = set(xls1.sheet_names)
|
||||||
|
sheet_names2 = set(xls2.sheet_names)
|
||||||
|
all_sheet_names = sheet_names1.union(sheet_names2)
|
||||||
|
message = f"文件1有{len(sheet_names1)}个sheet,文件2有{len(sheet_names2)}个sheet"
|
||||||
|
print(message)
|
||||||
|
self.log_messages.append(f"\n{message}")
|
||||||
|
|
||||||
|
# 读取Excel文件中的特定工作表
|
||||||
|
for i in all_sheet_names:
|
||||||
|
if i not in sheet_names1:
|
||||||
|
message = f"文件1中不存在sheet: {i}"
|
||||||
|
print(message)
|
||||||
|
self.log_messages.append(f"\n{message}")
|
||||||
|
continue
|
||||||
|
if i not in sheet_names2:
|
||||||
|
message = f"文件2中不存在sheet: {i}"
|
||||||
|
print(message)
|
||||||
|
self.log_messages.append(f"\n{message}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
df1 = pd.read_excel(
|
||||||
|
f"{self.file1}.xlsx", sheet_name=i, engine="openpyxl", skiprows=1
|
||||||
|
)
|
||||||
|
df2 = pd.read_excel(
|
||||||
|
f"{self.file2}.xlsx", sheet_name=i, engine="openpyxl", skiprows=1
|
||||||
|
)
|
||||||
|
# 比较两个DataFrame中的数据
|
||||||
|
if df1.equals(df2):
|
||||||
|
message = f"第{i}个sheet的数据相同"
|
||||||
|
print(message)
|
||||||
|
self.log_messages.append(f"\n{message}")
|
||||||
|
else:
|
||||||
|
message = f"第{i}个sheet的数据不同\n----------------------------\n不同数据如下:"
|
||||||
|
print(message)
|
||||||
|
self.log_messages.append(f"\n{message}")
|
||||||
|
try:
|
||||||
|
message = df1.compare(df2)
|
||||||
|
print(message)
|
||||||
|
self.log_messages.append(f"\n{message}")
|
||||||
|
except ValueError:
|
||||||
|
message = "数据列名称不同"
|
||||||
|
print(message)
|
||||||
|
self.log_messages.append(f"\n{message}")
|
||||||
|
message = "----------------------------"
|
||||||
|
print(message)
|
||||||
|
self.log_messages.append(f"\n{message}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
pass
|
57
config.py
Normal file
57
config.py
Normal file
@ -0,0 +1,57 @@
|
|||||||
|
import os
|
||||||
|
|
||||||
|
# 获取当前文件所在的文件夹路径
|
||||||
|
CURRENT_DIR_PATH = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
# MS.NET包的路径
|
||||||
|
NET_FOLDER = "MS.NET_Package"
|
||||||
|
|
||||||
|
# --------------------------------------------------------------
|
||||||
|
# ------------------------请填写以下信息-------------------------
|
||||||
|
# --------------------------------------------------------------
|
||||||
|
|
||||||
|
# 连接信息
|
||||||
|
### 第一个Analysis Services的连接信息
|
||||||
|
WORKSPACE_1 = "localhost:9088"
|
||||||
|
USERNAME_1 = "xxxx"
|
||||||
|
PASSWORD_1 = "xxxx"
|
||||||
|
CATALOG_1 = None
|
||||||
|
### 第二个Analysis Services的连接信息
|
||||||
|
WORKSPACE_2 = "localhost:9088"
|
||||||
|
USERNAME_2 = "xxxx"
|
||||||
|
PASSWORD_2 = "xxxx"
|
||||||
|
CATALOG_2 = None
|
||||||
|
|
||||||
|
# 导出页面查询的json文件的名称
|
||||||
|
EXPORT_PAGE_JSON_NAME_1 = "example"
|
||||||
|
EXPORT_PAGE_JSON_NAME_2 = "example"
|
||||||
|
|
||||||
|
# 是否是管理员方式运行,如果是,则为True,否则为False
|
||||||
|
IS_ADMIN_1 = True
|
||||||
|
IS_ADMIN_2 = True
|
||||||
|
# 自定义数据和角色
|
||||||
|
### 如果定义IS_ADMIN为False,则需要填写以下信息
|
||||||
|
### EFFECTIVE_USERNAME和CUSTOMER_DATA都填写只有一个生效,如果不需要生效的请填写None。
|
||||||
|
### 如果都填写,优先生效的顺序为: EFFECTIVE_USERNAME -> CUSTOMER_DATA
|
||||||
|
|
||||||
|
### 第一个Analysis Services的自定义数据和角色
|
||||||
|
ROLE_1 = "1"
|
||||||
|
EFFECTIVE_USERNAME_1 = None
|
||||||
|
CUSTOMER_DATA_1 = "1"
|
||||||
|
### 第二个Analysis Services的自定义数据和角色
|
||||||
|
ROLE_2 = "2"
|
||||||
|
EFFECTIVE_USERNAME_2 = None
|
||||||
|
CUSTOMER_DATA_2 = "2"
|
||||||
|
|
||||||
|
# --------------------------------------------------------------
|
||||||
|
# ------------------------请填写以上信息-------------------------
|
||||||
|
# --------------------------------------------------------------
|
||||||
|
|
||||||
|
desc_WORKSPACE = "Analysis Services的连接信息,如果是powerbi,则连接字符串中包含catalog"
|
||||||
|
desc_USERNAME = "用户名"
|
||||||
|
desc_PASSWORD = "密码"
|
||||||
|
desc_CATALOG = "如果是powerbi,则catalog为语义模型的名称,默认为None"
|
||||||
|
desc_EXPORT_PAGE_JSON_NAME = "导出页面查询的json文件的名称"
|
||||||
|
desc_IS_ADMIN = "是否是管理员方式运行,如果是,则为True,否则为False"
|
||||||
|
desc_ROLE = "角色名称"
|
||||||
|
desc_EFFECTIVE_USERNAME = "username()获取字段"
|
||||||
|
desc_CUSTOMER_DATA = "customerdata()获取字段"
|
189
getDataFromAS.py
Normal file
189
getDataFromAS.py
Normal file
@ -0,0 +1,189 @@
|
|||||||
|
import re
|
||||||
|
import pandas as pd
|
||||||
|
import clr
|
||||||
|
import os
|
||||||
|
from config import CURRENT_DIR_PATH, NET_FOLDER
|
||||||
|
|
||||||
|
folder = os.path.join(CURRENT_DIR_PATH, NET_FOLDER)
|
||||||
|
clr.AddReference(
|
||||||
|
folder
|
||||||
|
+ r"\Microsoft.AnalysisServices.AdomdClient\v4.0_15.0.0.0__89845dcd8080cc91\Microsoft.AnalysisServices.AdomdClient.DLL"
|
||||||
|
)
|
||||||
|
|
||||||
|
from Microsoft.AnalysisServices.AdomdClient import AdomdConnection, AdomdCommand # type: ignore
|
||||||
|
|
||||||
|
class ASDataFetcher:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
workspace,
|
||||||
|
username,
|
||||||
|
password,
|
||||||
|
queries,
|
||||||
|
json_name,
|
||||||
|
result_folder_path,
|
||||||
|
isAdmin=True,
|
||||||
|
catalog=None,
|
||||||
|
effective_username=None,
|
||||||
|
customdata=None,
|
||||||
|
role=None,
|
||||||
|
model_number=None,
|
||||||
|
):
|
||||||
|
self.workspace = workspace
|
||||||
|
self.username = username
|
||||||
|
self.password = password
|
||||||
|
self.queries = queries
|
||||||
|
self.isAdmin = isAdmin
|
||||||
|
self.effective_username = effective_username
|
||||||
|
self.customdata = customdata
|
||||||
|
self.role = role
|
||||||
|
self.result_folder_path = result_folder_path
|
||||||
|
self.error_list = []
|
||||||
|
self.json_name = json_name
|
||||||
|
self.model_number = model_number
|
||||||
|
self.log_messages = []
|
||||||
|
self.catalog = catalog
|
||||||
|
|
||||||
|
# 先将当前对象的 connString 方法的返回值赋值给当前对象的 conn_string 属性, 以便后续使用
|
||||||
|
self.conn_string = self.connString()
|
||||||
|
|
||||||
|
def connString(self) -> str:
|
||||||
|
# 初始化连接字符串,包含数据源、用户名和密码
|
||||||
|
# 如果workspace开头是powerbi,则连接字符串中包含catalog
|
||||||
|
if self.workspace.startswith("powerbi://"):
|
||||||
|
conn = f"DataSource={self.workspace};User ID={self.username};Password={self.password};Initial Catalog={self.catalog}"
|
||||||
|
else:
|
||||||
|
conn = f"DataSource={self.workspace};User ID={self.username};Password={self.password}"
|
||||||
|
|
||||||
|
# 如果是管理员,直接返回连接字符串
|
||||||
|
if self.isAdmin:
|
||||||
|
self.result_excel_name = f"admin_{self.json_name}_{self.model_number}"
|
||||||
|
self.result_full_excel_name = os.path.join(
|
||||||
|
self.result_folder_path, self.result_excel_name
|
||||||
|
)
|
||||||
|
return conn
|
||||||
|
# 如果不是管理员,且提供了角色信息
|
||||||
|
elif self.role:
|
||||||
|
# 如果提供了有效的用户名,返回包含有效用户名和角色的连接字符串
|
||||||
|
if self.effective_username:
|
||||||
|
self.result_excel_name = (
|
||||||
|
f"{self.role}_{self.effective_username}_{self.json_name}_{self.model_number}"
|
||||||
|
)
|
||||||
|
self.result_full_excel_name = os.path.join(
|
||||||
|
self.result_folder_path, self.result_excel_name
|
||||||
|
)
|
||||||
|
return f"{conn};EffectiveUserName={self.effective_username};Roles={self.role};"
|
||||||
|
|
||||||
|
# 如果提供了自定义数据,返回包含自定义数据和角色的连接字符串
|
||||||
|
elif self.customdata:
|
||||||
|
self.result_excel_name = (
|
||||||
|
f"{self.role}_{self.customdata}_{self.json_name}_{self.model_number}"
|
||||||
|
)
|
||||||
|
self.result_full_excel_name = os.path.join(
|
||||||
|
self.result_folder_path, self.result_excel_name
|
||||||
|
)
|
||||||
|
return f"{conn};CustomData={self.customdata};Roles={self.role};"
|
||||||
|
|
||||||
|
# 如果只提供了角色,返回包含角色的连接字符串
|
||||||
|
else:
|
||||||
|
self.result_excel_name = f"{self.role}_-NoUser-_{self.json_name}_{self.model_number}"
|
||||||
|
self.result_full_excel_name = os.path.join(
|
||||||
|
self.result_folder_path, self.result_excel_name
|
||||||
|
)
|
||||||
|
return f"{conn};Roles={self.role};"
|
||||||
|
# 如果不是管理员且未提供角色信息,抛出异常
|
||||||
|
else:
|
||||||
|
raise ValueError("非管理员用户必须提供角色信息")
|
||||||
|
|
||||||
|
def readData(self, reader):
|
||||||
|
# 获取列名
|
||||||
|
column_names = [reader.GetName(i) for i in range(reader.FieldCount)]
|
||||||
|
# 初始化数据存储
|
||||||
|
data = []
|
||||||
|
while reader.Read():
|
||||||
|
# 读取每一行的数据
|
||||||
|
row_data = [reader[i] for i in range(reader.FieldCount)]
|
||||||
|
# 将行数据添加到数据列表中
|
||||||
|
data.append(row_data)
|
||||||
|
# 将数据转换为 DataFrame
|
||||||
|
df = pd.DataFrame(data, columns=column_names)
|
||||||
|
# 返回DataFrame和下一个结果集
|
||||||
|
return df, reader.NextResult()
|
||||||
|
|
||||||
|
def getDataFromAS(self, query):
|
||||||
|
|
||||||
|
# 创建AdomdConnection对象
|
||||||
|
conn = AdomdConnection(self.conn_string)
|
||||||
|
try:
|
||||||
|
# 建立连接
|
||||||
|
conn.Open()
|
||||||
|
# 创建命令
|
||||||
|
cmd = AdomdCommand(query, conn)
|
||||||
|
# 执行命令
|
||||||
|
reader = cmd.ExecuteReader()
|
||||||
|
IsNext = True
|
||||||
|
while IsNext:
|
||||||
|
# 将结果集转换为DataFrame
|
||||||
|
df, IsNext = self.readData(reader)
|
||||||
|
# 重命名列名
|
||||||
|
df.columns = [
|
||||||
|
(
|
||||||
|
re.search(r"\[(.*?)\]", col).group(1)
|
||||||
|
if re.search(r"\[(.*?)\]", col)
|
||||||
|
else col
|
||||||
|
)
|
||||||
|
for col in df.columns
|
||||||
|
]
|
||||||
|
yield df
|
||||||
|
except Exception as ex:
|
||||||
|
message = f"错误信息: {ex}"
|
||||||
|
print(message)
|
||||||
|
self.log_messages.append(f"\n{message}")
|
||||||
|
finally:
|
||||||
|
# 关闭连接
|
||||||
|
conn.Close()
|
||||||
|
|
||||||
|
def writeToExcel(self, print_lock):
|
||||||
|
# 创建一个ExcelWriter对象
|
||||||
|
with pd.ExcelWriter(
|
||||||
|
f"{self.result_full_excel_name}.xlsx", engine="openpyxl"
|
||||||
|
) as writer:
|
||||||
|
sheet_name_id = 1
|
||||||
|
# 将DataFrame写入不同的工作表
|
||||||
|
for query in self.queries:
|
||||||
|
for df in self.getDataFromAS(query):
|
||||||
|
try:
|
||||||
|
# 访问工作簿和工作表
|
||||||
|
workbook = writer.book
|
||||||
|
worksheet = workbook.create_sheet(str(sheet_name_id))
|
||||||
|
worksheet["A1"] = query
|
||||||
|
|
||||||
|
# 将DataFrame写入Excel文件
|
||||||
|
df.to_excel(
|
||||||
|
writer,
|
||||||
|
sheet_name=str(sheet_name_id),
|
||||||
|
index=True,
|
||||||
|
float_format="%.2f",
|
||||||
|
startrow=1,
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
with print_lock:
|
||||||
|
message = f"写入工作表 {sheet_name_id} 失败: {e}"
|
||||||
|
print(message)
|
||||||
|
self.log_messages.append(f"\n{message}")
|
||||||
|
self.error_list.append(sheet_name_id)
|
||||||
|
else:
|
||||||
|
with print_lock:
|
||||||
|
message = f"{self.result_excel_name} \n表格大小:{df.shape} \n写入工作表 {sheet_name_id}"
|
||||||
|
print(message)
|
||||||
|
self.log_messages.append(f"\n{message}")
|
||||||
|
finally:
|
||||||
|
with print_lock:
|
||||||
|
message = f"-------------------------"
|
||||||
|
print(message)
|
||||||
|
self.log_messages.append(f"{message}")
|
||||||
|
sheet_name_id += 1
|
||||||
|
return self.error_list
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
pass
|
37
getQueries.py
Normal file
37
getQueries.py
Normal file
@ -0,0 +1,37 @@
|
|||||||
|
import json
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
|
||||||
|
def getQueries_From_json(json_name):
|
||||||
|
# 打开并读取JSON文件
|
||||||
|
with open(json_name + ".json", "r", encoding="utf-8-sig") as file:
|
||||||
|
data = json.load(file)
|
||||||
|
# 从JSON数据中提取所有名为"Execute DAX Query"的事件中的QueryText
|
||||||
|
queries = [
|
||||||
|
event["metrics"]["QueryText"]
|
||||||
|
for event in data["events"]
|
||||||
|
if event["name"] == "Execute DAX Query"
|
||||||
|
]
|
||||||
|
return queries
|
||||||
|
|
||||||
|
|
||||||
|
def getQueries_From_excel(excel_name):
|
||||||
|
# 使用pandas读取Excel文件
|
||||||
|
df = pd.read_excel(excel_name, engine="openpyxl")
|
||||||
|
# 在DataFrame中添加一个新的列"NAME",其值为"EVALUATE '" + 原始的"NAME"列值 + "'"
|
||||||
|
df["NAME"] = "EVALUATE '" + df["NAME"] + "'"
|
||||||
|
# 将新的"NAME"列转换为列表
|
||||||
|
queries = df["NAME"].to_list()
|
||||||
|
return queries
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
import os
|
||||||
|
from config import CURRENT_DIR_PATH, EXPORT_PAGE_JSON_NAME_1
|
||||||
|
|
||||||
|
# 获取当前脚本所在的目录路径
|
||||||
|
EXPORT_PAGE_JSON_PATH_1 = os.path.join(
|
||||||
|
CURRENT_DIR_PATH, "Export_Json", EXPORT_PAGE_JSON_NAME_1
|
||||||
|
)
|
||||||
|
queries = getQueries_From_json(EXPORT_PAGE_JSON_PATH_1)
|
||||||
|
print(queries)
|
BIN
requirements.txt
Normal file
BIN
requirements.txt
Normal file
Binary file not shown.
112
run.py
Normal file
112
run.py
Normal file
@ -0,0 +1,112 @@
|
|||||||
|
import os
|
||||||
|
import threading
|
||||||
|
from compareData import Comparator
|
||||||
|
from getDataFromAS import ASDataFetcher
|
||||||
|
from getQueries import getQueries_From_json
|
||||||
|
import config
|
||||||
|
|
||||||
|
# 拼接导出页面查询json文件夹路径
|
||||||
|
json_folder_path = os.path.join(config.CURRENT_DIR_PATH, "Export_Json")
|
||||||
|
# 拼接查询结果excel文件夹路径
|
||||||
|
result_folder_path = os.path.join(config.CURRENT_DIR_PATH, "Evaluate_Result")
|
||||||
|
# 检查文件夹是否存在,如果不存在则创建它
|
||||||
|
if not os.path.exists(json_folder_path):
|
||||||
|
os.makedirs(json_folder_path)
|
||||||
|
if not os.path.exists(result_folder_path):
|
||||||
|
os.makedirs(result_folder_path)
|
||||||
|
|
||||||
|
# 拼接导出页面查询json文件路径
|
||||||
|
export_page_json_path_1 = os.path.join(json_folder_path, config.EXPORT_PAGE_JSON_NAME_1)
|
||||||
|
export_page_json_path_2 = os.path.join(json_folder_path, config.EXPORT_PAGE_JSON_NAME_2)
|
||||||
|
# 读取json文件中的查询
|
||||||
|
queries_1 = getQueries_From_json(export_page_json_path_1)
|
||||||
|
queries_2 = getQueries_From_json(export_page_json_path_2)
|
||||||
|
|
||||||
|
# 创建 ASDataFetcher 实例
|
||||||
|
fetcher_1 = ASDataFetcher(
|
||||||
|
workspace=config.WORKSPACE_1,
|
||||||
|
username=config.USERNAME_1,
|
||||||
|
password=config.PASSWORD_1,
|
||||||
|
queries=queries_1,
|
||||||
|
json_name=config.EXPORT_PAGE_JSON_NAME_1,
|
||||||
|
result_folder_path=result_folder_path,
|
||||||
|
isAdmin=config.IS_ADMIN_1,
|
||||||
|
catalog=config.CATALOG_1,
|
||||||
|
effective_username=config.EFFECTIVE_USERNAME_1,
|
||||||
|
customdata=config.CUSTOMER_DATA_1,
|
||||||
|
role=config.ROLE_1,
|
||||||
|
model_number=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 创建 ASDataFetcher 实例
|
||||||
|
fetcher_2 = ASDataFetcher(
|
||||||
|
workspace=config.WORKSPACE_2,
|
||||||
|
username=config.USERNAME_2,
|
||||||
|
password=config.PASSWORD_2,
|
||||||
|
queries=queries_2,
|
||||||
|
json_name=config.EXPORT_PAGE_JSON_NAME_2,
|
||||||
|
result_folder_path=result_folder_path,
|
||||||
|
isAdmin=config.IS_ADMIN_2,
|
||||||
|
catalog=config.CATALOG_2,
|
||||||
|
effective_username=config.EFFECTIVE_USERNAME_2,
|
||||||
|
customdata=config.CUSTOMER_DATA_2,
|
||||||
|
role=config.ROLE_2,
|
||||||
|
model_number=2,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 定义线程任务
|
||||||
|
def run_fetcher(fetcher, print_lock):
|
||||||
|
fetcher.writeToExcel(print_lock)
|
||||||
|
|
||||||
|
# 创建锁
|
||||||
|
print_lock = threading.Lock()
|
||||||
|
|
||||||
|
# 创建线程
|
||||||
|
thread_1 = threading.Thread(target=run_fetcher, args=(fetcher_1, print_lock))
|
||||||
|
thread_2 = threading.Thread(target=run_fetcher, args=(fetcher_2, print_lock))
|
||||||
|
|
||||||
|
try:
|
||||||
|
print(
|
||||||
|
f"""
|
||||||
|
---*********************************************************---
|
||||||
|
开始查询
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
# 启动线程
|
||||||
|
thread_1.start()
|
||||||
|
thread_2.start()
|
||||||
|
|
||||||
|
# 等待线程完成
|
||||||
|
thread_1.join()
|
||||||
|
thread_2.join()
|
||||||
|
|
||||||
|
print(
|
||||||
|
f"""
|
||||||
|
查询完成
|
||||||
|
---*********************************************************---
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"错误提示: {e}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
print(
|
||||||
|
"""
|
||||||
|
---------------------------------------------------------------
|
||||||
|
开始比较
|
||||||
|
---------------------------------------------------------------
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
comparator = Comparator(
|
||||||
|
fetcher_1.result_full_excel_name, fetcher_2.result_full_excel_name
|
||||||
|
)
|
||||||
|
comparator.compare_ExcelFiles()
|
||||||
|
print(
|
||||||
|
"""
|
||||||
|
----------------------------------------------------------------
|
||||||
|
比较完成
|
||||||
|
----------------------------------------------------------------
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"错误提示: {e}")
|
111
templates/index.html
Normal file
111
templates/index.html
Normal file
@ -0,0 +1,111 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="zh-cn">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<title>数据对比</title>
|
||||||
|
<style>
|
||||||
|
.container {
|
||||||
|
height: 100vh;
|
||||||
|
display: flex;
|
||||||
|
}
|
||||||
|
.left {
|
||||||
|
flex: 40%; /* 左边 div 占 40% */
|
||||||
|
display: table;
|
||||||
|
flex-direction: column;
|
||||||
|
background-color: #f0f0f0;
|
||||||
|
padding: 10px;
|
||||||
|
box-sizing: border-box;
|
||||||
|
overflow-y: auto;
|
||||||
|
}
|
||||||
|
.config {
|
||||||
|
flex: 1; /* 上半部分 */
|
||||||
|
display: inline-table;
|
||||||
|
flex-direction: row;
|
||||||
|
}
|
||||||
|
.config-content {
|
||||||
|
display: flex;
|
||||||
|
flex: 100%;
|
||||||
|
}
|
||||||
|
.buttons {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: revert-layer;
|
||||||
|
align-items: center;
|
||||||
|
margin-top: 10px;
|
||||||
|
flex-shrink: 0; /* 防止按钮区域收缩 */
|
||||||
|
justify-content: space-around;
|
||||||
|
}
|
||||||
|
.result {
|
||||||
|
flex: 60%; /* 右边 div 占 60% */
|
||||||
|
background-color: #f9eded;
|
||||||
|
padding: 10px;
|
||||||
|
box-sizing: border-box;
|
||||||
|
overflow-y: auto;
|
||||||
|
}
|
||||||
|
.description {
|
||||||
|
color: gray;
|
||||||
|
}
|
||||||
|
.config .config1, .config2 {
|
||||||
|
flex: 50%; /* 修改 flex 值为 50% */
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
}
|
||||||
|
.config-item {
|
||||||
|
display: block;
|
||||||
|
grid-template-columns: 1fr 1fr;
|
||||||
|
gap: 10px;
|
||||||
|
margin-bottom: 15px;
|
||||||
|
}
|
||||||
|
.config-keys {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: 1fr 1fr;
|
||||||
|
gap: 10px;
|
||||||
|
}
|
||||||
|
.config-desc {
|
||||||
|
grid-column: 1 / -1;
|
||||||
|
text-align: left;
|
||||||
|
}
|
||||||
|
.config-inputs {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: 1fr 1fr;
|
||||||
|
gap: 10px;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<div class="container">
|
||||||
|
<div class="left">
|
||||||
|
<div class="config">
|
||||||
|
<h2>配置项</h2>
|
||||||
|
<form action="{{ url_for('update_config') }}" method="post">
|
||||||
|
{% for key1 in config.__dict__.keys() if key1.endswith('_1') %}
|
||||||
|
{% set base_key = key1[:-2] %}
|
||||||
|
{% set key2 = base_key + '_2' %}
|
||||||
|
{% set desc_key = 'desc_' + base_key %}
|
||||||
|
<div class="config-item">
|
||||||
|
<div class="config-keys">
|
||||||
|
<label for="{{ key1 }}">{{ key1 }}</label>
|
||||||
|
<label for="{{ key2 }}">{{ key2 }}</label>
|
||||||
|
</div>
|
||||||
|
<div class="config-desc">
|
||||||
|
<span class="description">{{ config.__dict__.get(desc_key, '') }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="config-inputs">
|
||||||
|
<input type="text" id="{{ key1 }}" name="{{ key1 }}" value="{{ config.__dict__[key1] }}">
|
||||||
|
<input type="text" id="{{ key2 }}" name="{{ key2 }}" value="{{ config.__dict__[key2] }}">
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endfor %}
|
||||||
|
<div class="buttons">
|
||||||
|
<button type="submit">保存配置</button>
|
||||||
|
<button type="submit" formaction="{{ url_for('run') }}">运行</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="result">
|
||||||
|
<h1>运行结果</h1>
|
||||||
|
<pre>{{ result }}</pre>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</body>
|
||||||
|
</html>
|
Loading…
Reference in New Issue
Block a user