V1.0
This commit is contained in:
commit
9bff0a7c7b
2
.gitattributes
vendored
Normal file
2
.gitattributes
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
# Auto detect text files and perform LF normalization
|
||||
* text=auto
|
BIN
Evaluate_Result/admin_example_1.xlsx
Normal file
BIN
Evaluate_Result/admin_example_1.xlsx
Normal file
Binary file not shown.
BIN
Evaluate_Result/admin_example_2.xlsx
Normal file
BIN
Evaluate_Result/admin_example_2.xlsx
Normal file
Binary file not shown.
BIN
ExamplePBIX/Examples.pbix
Normal file
BIN
ExamplePBIX/Examples.pbix
Normal file
Binary file not shown.
1
Export_Json/example.json
Normal file
1
Export_Json/example.json
Normal file
@ -0,0 +1 @@
|
||||
{"version":"1.1.0","events":[{"name":"User Action","component":"Report Canvas","start":"2024-12-21T17:14:40.987Z","id":"515c279a49002a814996","metrics":{"sourceLabel":"UserAction_StartedMonitoring"}},{"name":"User Action","component":"Report Canvas","start":"2024-12-21T17:14:42.623Z","id":"79b174c0d02ca69ada00","metrics":{"sourceLabel":"UserAction_Refresh"}},{"name":"Visual Container Lifecycle","start":"2024-12-21T17:14:42.629Z","end":"2024-12-21T17:14:42.682Z","id":"24c30acf43d0ab30dd40-b004a31d5ccd6a4a5385","metrics":{"status":1,"visualTitle":"Table","visualId":"24c30acf43d0ab30dd40","visualType":"tableEx","initialLoad":false}},{"name":"Resolve Parameters","start":"2024-12-21T17:14:42.630Z","end":"2024-12-21T17:14:42.631Z","id":"865f086f-a553-4057-9963-f386c6ea310f","parentId":"24c30acf43d0ab30dd40-b004a31d5ccd6a4a5385"},{"name":"Query","start":"2024-12-21T17:14:42.631Z","end":"2024-12-21T17:14:42.662Z","id":"f374c5d4-aeae-49e4-a63f-a086480fbad1","parentId":"24c30acf43d0ab30dd40-b004a31d5ccd6a4a5385"},{"name":"Render","start":"2024-12-21T17:14:42.661Z","end":"2024-12-21T17:14:42.682Z","id":"bf797551-b336-43c1-8a29-eda4f83f224e","parentId":"24c30acf43d0ab30dd40-b004a31d5ccd6a4a5385"},{"name":"Data View Transform","start":"2024-12-21T17:14:42.670Z","end":"2024-12-21T17:14:42.673Z","id":"2b79f57f-9353-48e9-afe8-0c1f446f8636","parentId":"bf797551-b336-43c1-8a29-eda4f83f224e"},{"name":"Visual Update Async","start":"2024-12-21T17:14:42.673Z","end":"2024-12-21T17:14:42.682Z","id":"ca7f9db7-b9c5-4739-aced-6a8a5560f0a8","parentId":"bf797551-b336-43c1-8a29-eda4f83f224e"},{"name":"Visual Update","start":"2024-12-21T17:14:42.673Z","end":"2024-12-21T17:14:42.673Z","id":"29d7c7a6-ef0d-416a-973f-cf4ddacedc8a","parentId":"bf797551-b336-43c1-8a29-eda4f83f224e"},{"name":"Execute Semantic Query","component":"DSE","start":"2024-12-21T17:14:42.649Z","end":"2024-12-21T17:14:42.656Z","id":"47aeb947-80d7-4096-9f2d-3b16c8b5b5e9","parentId":"f374c5d4-aeae-49e4-a63f-a086480fbad1"},{"name":"Query Generation","start":"2024-12-21T17:14:42.633Z","end":"2024-12-21T17:14:42.634Z","id":"4fa46d04-aa1d-46ea-933b-49002a24c309","parentId":"f374c5d4-aeae-49e4-a63f-a086480fbad1"},{"name":"Query Pending","start":"2024-12-21T17:14:42.648Z","end":"2024-12-21T17:14:42.649Z","id":"178f1281-20eb-4c78-a3d1-bdc9f6735a7c","parentId":"f374c5d4-aeae-49e4-a63f-a086480fbad1"},{"name":"Parse Query Result","start":"2024-12-21T17:14:42.658Z","end":"2024-12-21T17:14:42.660Z","id":"33e79f0c-bc13-4951-a18e-6fdbd767e4f8","parentId":"f374c5d4-aeae-49e4-a63f-a086480fbad1"},{"name":"Execute DAX Query","component":"DSE","start":"2024-12-21T17:14:42.651Z","end":"2024-12-21T17:14:42.655Z","id":"e458b6a8-0b2b-474a-a8f1-294856878497","parentId":"47aeb947-80d7-4096-9f2d-3b16c8b5b5e9","metrics":{"QueryText":"DEFINE\r\n\tVAR __DS0Core = \r\n\t\tSUMMARIZECOLUMNS(\r\n\t\t\tROLLUPADDISSUBTOTAL('financials'[Country], \"IsGrandTotalRowTotal\"),\r\n\t\t\t\"Sumv_Sales\", CALCULATE(SUM('financials'[ Sales]))\r\n\t\t)\r\n\r\n\tVAR __DS0PrimaryWindowed = \r\n\t\tTOPN(502, __DS0Core, [IsGrandTotalRowTotal], 0, 'financials'[Country], 1)\r\n\r\nEVALUATE\r\n\t__DS0PrimaryWindowed\r\n\r\nORDER BY\r\n\t[IsGrandTotalRowTotal] DESC, 'financials'[Country]","RowCount":6}},{"name":"Execute Query","component":"AS","start":"2024-12-21T17:14:42.653Z","end":"2024-12-21T17:14:42.657Z","id":"3BC9FA84-D491-49ED-B2A1-7DC0D4F32A4C","parentId":"e458b6a8-0b2b-474a-a8f1-294856878497"},{"name":"Serialize Rowset","component":"AS","start":"2024-12-21T17:14:42.657Z","end":"2024-12-21T17:14:42.657Z","id":"57BE528D-04A5-4696-A0AF-3327CD0CE702","parentId":"3BC9FA84-D491-49ED-B2A1-7DC0D4F32A4C"}],"sessionId":"14d9da6f-0492-428d-bcff-ece9e2ac26f6"}
|
Binary file not shown.
BIN
__pycache__/app.cpython-312.pyc
Normal file
BIN
__pycache__/app.cpython-312.pyc
Normal file
Binary file not shown.
BIN
__pycache__/compareData.cpython-312.pyc
Normal file
BIN
__pycache__/compareData.cpython-312.pyc
Normal file
Binary file not shown.
BIN
__pycache__/config.cpython-312.pyc
Normal file
BIN
__pycache__/config.cpython-312.pyc
Normal file
Binary file not shown.
BIN
__pycache__/getDataFromAS.cpython-312.pyc
Normal file
BIN
__pycache__/getDataFromAS.cpython-312.pyc
Normal file
Binary file not shown.
BIN
__pycache__/getQueries.cpython-312.pyc
Normal file
BIN
__pycache__/getQueries.cpython-312.pyc
Normal file
Binary file not shown.
BIN
__pycache__/writeDataToExcel.cpython-312.pyc
Normal file
BIN
__pycache__/writeDataToExcel.cpython-312.pyc
Normal file
Binary file not shown.
138
app.py
Normal file
138
app.py
Normal file
@ -0,0 +1,138 @@
|
||||
from flask import Flask, request, render_template, redirect, url_for
|
||||
import os
|
||||
import threading
|
||||
import config
|
||||
from getDataFromAS import ASDataFetcher
|
||||
from getQueries import getQueries_From_json
|
||||
from compareData import Comparator
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
result = ""
|
||||
|
||||
@app.route('/')
|
||||
def index():
|
||||
global result
|
||||
return render_template('index.html', config=config, result=result)
|
||||
|
||||
@app.route('/update_config', methods=['POST'])
|
||||
def update_config():
|
||||
for key, value in request.form.items():
|
||||
if hasattr(config, key) and key not in ['CURRENT_DIR_PATH', 'os', 'NET_FOLDER']:
|
||||
setattr(config, key, value)
|
||||
return redirect(url_for('index'))
|
||||
|
||||
@app.route('/run', methods=['POST'])
|
||||
def run():
|
||||
global result
|
||||
result = ""
|
||||
|
||||
# 拼接导出页面查询json文件夹路径
|
||||
json_folder_path = os.path.join(config.CURRENT_DIR_PATH, "Export_Json")
|
||||
# 拼接查询结果excel文件夹路径
|
||||
result_folder_path = os.path.join(config.CURRENT_DIR_PATH, "Evaluate_Result")
|
||||
# 检查文件夹是否存在,如果不存在则创建它
|
||||
if not os.path.exists(json_folder_path):
|
||||
os.makedirs(json_folder_path)
|
||||
if not os.path.exists(result_folder_path):
|
||||
os.makedirs(result_folder_path)
|
||||
|
||||
# 拼接导出页面查询json文件路径
|
||||
export_page_json_path_1 = os.path.join(json_folder_path, config.EXPORT_PAGE_JSON_NAME_1)
|
||||
export_page_json_path_2 = os.path.join(json_folder_path, config.EXPORT_PAGE_JSON_NAME_2)
|
||||
# 读取json文件中的查询
|
||||
queries_1 = getQueries_From_json(export_page_json_path_1)
|
||||
queries_2 = getQueries_From_json(export_page_json_path_2)
|
||||
|
||||
# 创建 ASDataFetcher 实例
|
||||
fetcher_1 = ASDataFetcher(
|
||||
workspace=config.WORKSPACE_1,
|
||||
username=config.USERNAME_1,
|
||||
password=config.PASSWORD_1,
|
||||
queries=queries_1,
|
||||
json_name=config.EXPORT_PAGE_JSON_NAME_1,
|
||||
result_folder_path=result_folder_path,
|
||||
isAdmin=config.IS_ADMIN_1,
|
||||
catalog=config.CATALOG_1,
|
||||
effective_username=config.EFFECTIVE_USERNAME_1,
|
||||
customdata=config.CUSTOMER_DATA_1,
|
||||
role=config.ROLE_1,
|
||||
model_number=1,
|
||||
)
|
||||
|
||||
# 创建 ASDataFetcher 实例
|
||||
fetcher_2 = ASDataFetcher(
|
||||
workspace=config.WORKSPACE_2,
|
||||
username=config.USERNAME_2,
|
||||
password=config.PASSWORD_2,
|
||||
queries=queries_2,
|
||||
json_name=config.EXPORT_PAGE_JSON_NAME_2,
|
||||
result_folder_path=result_folder_path,
|
||||
isAdmin=config.IS_ADMIN_2,
|
||||
catalog=config.CATALOG_2,
|
||||
effective_username=config.EFFECTIVE_USERNAME_2,
|
||||
customdata=config.CUSTOMER_DATA_2,
|
||||
role=config.ROLE_2,
|
||||
model_number=2,
|
||||
)
|
||||
|
||||
# 定义线程任务
|
||||
def run_fetcher(fetcher, print_lock):
|
||||
global result
|
||||
try:
|
||||
fetcher.writeToExcel(print_lock)
|
||||
with print_lock:
|
||||
result += f"\n-------------------------\n{fetcher.json_name} 查询完成\n"
|
||||
result += "\n".join(fetcher.log_messages) + "\n"
|
||||
except Exception as e:
|
||||
with print_lock:
|
||||
result += f"错误提示: {e}\n"
|
||||
result += "\n".join(fetcher.log_messages) + "\n"
|
||||
return
|
||||
|
||||
# 创建锁
|
||||
print_lock = threading.Lock()
|
||||
|
||||
# 创建线程
|
||||
thread_1 = threading.Thread(target=run_fetcher, args=(fetcher_1, print_lock))
|
||||
thread_2 = threading.Thread(target=run_fetcher, args=(fetcher_2, print_lock))
|
||||
|
||||
try:
|
||||
result += "---*********************************************************---\n"
|
||||
result += " 开始查询\n"
|
||||
result += "---*********************************************************---\n"
|
||||
|
||||
# 启动线程
|
||||
thread_1.start()
|
||||
thread_2.start()
|
||||
|
||||
# 等待线程完成
|
||||
thread_1.join()
|
||||
thread_2.join()
|
||||
|
||||
result += "\n---*********************************************************---\n"
|
||||
result += " 查询完成\n"
|
||||
result += "---*********************************************************---\n"
|
||||
|
||||
except Exception as e:
|
||||
result += f"错误提示: {e}\n"
|
||||
return redirect(url_for('index'))
|
||||
|
||||
try:
|
||||
result += "\n---------------------------------------------------------------\n"
|
||||
result += " 开始比较\n"
|
||||
result += "---------------------------------------------------------------\n"
|
||||
|
||||
comparator = Comparator(fetcher_1.result_full_excel_name, fetcher_2.result_full_excel_name)
|
||||
comparator.compare_ExcelFiles()
|
||||
result += "\n".join(comparator.log_messages) + "\n"
|
||||
result += "\n---------------------------------------------------------------\n"
|
||||
result += " 比较完成\n"
|
||||
result += "---------------------------------------------------------------\n"
|
||||
except Exception as e:
|
||||
result += f"错误提示: {e}\n"
|
||||
|
||||
return redirect(url_for('index'))
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(debug=True)
|
65
compareData.py
Normal file
65
compareData.py
Normal file
@ -0,0 +1,65 @@
|
||||
import pandas as pd
|
||||
|
||||
|
||||
class Comparator:
|
||||
def __init__(self, file1, file2):
|
||||
self.file1 = file1
|
||||
self.file2 = file2
|
||||
self.log_messages = []
|
||||
|
||||
def compare_ExcelFiles(self):
|
||||
# 读取Excel文件
|
||||
xls1 = pd.ExcelFile(f"{self.file1}.xlsx", engine="openpyxl")
|
||||
xls2 = pd.ExcelFile(f"{self.file2}.xlsx", engine="openpyxl")
|
||||
|
||||
# 获取所有工作表的名称
|
||||
sheet_names1 = set(xls1.sheet_names)
|
||||
sheet_names2 = set(xls2.sheet_names)
|
||||
all_sheet_names = sheet_names1.union(sheet_names2)
|
||||
message = f"文件1有{len(sheet_names1)}个sheet,文件2有{len(sheet_names2)}个sheet"
|
||||
print(message)
|
||||
self.log_messages.append(f"\n{message}")
|
||||
|
||||
# 读取Excel文件中的特定工作表
|
||||
for i in all_sheet_names:
|
||||
if i not in sheet_names1:
|
||||
message = f"文件1中不存在sheet: {i}"
|
||||
print(message)
|
||||
self.log_messages.append(f"\n{message}")
|
||||
continue
|
||||
if i not in sheet_names2:
|
||||
message = f"文件2中不存在sheet: {i}"
|
||||
print(message)
|
||||
self.log_messages.append(f"\n{message}")
|
||||
continue
|
||||
|
||||
df1 = pd.read_excel(
|
||||
f"{self.file1}.xlsx", sheet_name=i, engine="openpyxl", skiprows=1
|
||||
)
|
||||
df2 = pd.read_excel(
|
||||
f"{self.file2}.xlsx", sheet_name=i, engine="openpyxl", skiprows=1
|
||||
)
|
||||
# 比较两个DataFrame中的数据
|
||||
if df1.equals(df2):
|
||||
message = f"第{i}个sheet的数据相同"
|
||||
print(message)
|
||||
self.log_messages.append(f"\n{message}")
|
||||
else:
|
||||
message = f"第{i}个sheet的数据不同\n----------------------------\n不同数据如下:"
|
||||
print(message)
|
||||
self.log_messages.append(f"\n{message}")
|
||||
try:
|
||||
message = df1.compare(df2)
|
||||
print(message)
|
||||
self.log_messages.append(f"\n{message}")
|
||||
except ValueError:
|
||||
message = "数据列名称不同"
|
||||
print(message)
|
||||
self.log_messages.append(f"\n{message}")
|
||||
message = "----------------------------"
|
||||
print(message)
|
||||
self.log_messages.append(f"\n{message}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pass
|
57
config.py
Normal file
57
config.py
Normal file
@ -0,0 +1,57 @@
|
||||
import os
|
||||
|
||||
# 获取当前文件所在的文件夹路径
|
||||
CURRENT_DIR_PATH = os.path.dirname(os.path.abspath(__file__))
|
||||
# MS.NET包的路径
|
||||
NET_FOLDER = "MS.NET_Package"
|
||||
|
||||
# --------------------------------------------------------------
|
||||
# ------------------------请填写以下信息-------------------------
|
||||
# --------------------------------------------------------------
|
||||
|
||||
# 连接信息
|
||||
### 第一个Analysis Services的连接信息
|
||||
WORKSPACE_1 = "localhost:9088"
|
||||
USERNAME_1 = "xxxx"
|
||||
PASSWORD_1 = "xxxx"
|
||||
CATALOG_1 = None
|
||||
### 第二个Analysis Services的连接信息
|
||||
WORKSPACE_2 = "localhost:9088"
|
||||
USERNAME_2 = "xxxx"
|
||||
PASSWORD_2 = "xxxx"
|
||||
CATALOG_2 = None
|
||||
|
||||
# 导出页面查询的json文件的名称
|
||||
EXPORT_PAGE_JSON_NAME_1 = "example"
|
||||
EXPORT_PAGE_JSON_NAME_2 = "example"
|
||||
|
||||
# 是否是管理员方式运行,如果是,则为True,否则为False
|
||||
IS_ADMIN_1 = True
|
||||
IS_ADMIN_2 = True
|
||||
# 自定义数据和角色
|
||||
### 如果定义IS_ADMIN为False,则需要填写以下信息
|
||||
### EFFECTIVE_USERNAME和CUSTOMER_DATA都填写只有一个生效,如果不需要生效的请填写None。
|
||||
### 如果都填写,优先生效的顺序为: EFFECTIVE_USERNAME -> CUSTOMER_DATA
|
||||
|
||||
### 第一个Analysis Services的自定义数据和角色
|
||||
ROLE_1 = "1"
|
||||
EFFECTIVE_USERNAME_1 = None
|
||||
CUSTOMER_DATA_1 = "1"
|
||||
### 第二个Analysis Services的自定义数据和角色
|
||||
ROLE_2 = "2"
|
||||
EFFECTIVE_USERNAME_2 = None
|
||||
CUSTOMER_DATA_2 = "2"
|
||||
|
||||
# --------------------------------------------------------------
|
||||
# ------------------------请填写以上信息-------------------------
|
||||
# --------------------------------------------------------------
|
||||
|
||||
desc_WORKSPACE = "Analysis Services的连接信息,如果是powerbi,则连接字符串中包含catalog"
|
||||
desc_USERNAME = "用户名"
|
||||
desc_PASSWORD = "密码"
|
||||
desc_CATALOG = "如果是powerbi,则catalog为语义模型的名称,默认为None"
|
||||
desc_EXPORT_PAGE_JSON_NAME = "导出页面查询的json文件的名称"
|
||||
desc_IS_ADMIN = "是否是管理员方式运行,如果是,则为True,否则为False"
|
||||
desc_ROLE = "角色名称"
|
||||
desc_EFFECTIVE_USERNAME = "username()获取字段"
|
||||
desc_CUSTOMER_DATA = "customerdata()获取字段"
|
189
getDataFromAS.py
Normal file
189
getDataFromAS.py
Normal file
@ -0,0 +1,189 @@
|
||||
import re
|
||||
import pandas as pd
|
||||
import clr
|
||||
import os
|
||||
from config import CURRENT_DIR_PATH, NET_FOLDER
|
||||
|
||||
folder = os.path.join(CURRENT_DIR_PATH, NET_FOLDER)
|
||||
clr.AddReference(
|
||||
folder
|
||||
+ r"\Microsoft.AnalysisServices.AdomdClient\v4.0_15.0.0.0__89845dcd8080cc91\Microsoft.AnalysisServices.AdomdClient.DLL"
|
||||
)
|
||||
|
||||
from Microsoft.AnalysisServices.AdomdClient import AdomdConnection, AdomdCommand # type: ignore
|
||||
|
||||
class ASDataFetcher:
|
||||
def __init__(
|
||||
self,
|
||||
workspace,
|
||||
username,
|
||||
password,
|
||||
queries,
|
||||
json_name,
|
||||
result_folder_path,
|
||||
isAdmin=True,
|
||||
catalog=None,
|
||||
effective_username=None,
|
||||
customdata=None,
|
||||
role=None,
|
||||
model_number=None,
|
||||
):
|
||||
self.workspace = workspace
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.queries = queries
|
||||
self.isAdmin = isAdmin
|
||||
self.effective_username = effective_username
|
||||
self.customdata = customdata
|
||||
self.role = role
|
||||
self.result_folder_path = result_folder_path
|
||||
self.error_list = []
|
||||
self.json_name = json_name
|
||||
self.model_number = model_number
|
||||
self.log_messages = []
|
||||
self.catalog = catalog
|
||||
|
||||
# 先将当前对象的 connString 方法的返回值赋值给当前对象的 conn_string 属性, 以便后续使用
|
||||
self.conn_string = self.connString()
|
||||
|
||||
def connString(self) -> str:
|
||||
# 初始化连接字符串,包含数据源、用户名和密码
|
||||
# 如果workspace开头是powerbi,则连接字符串中包含catalog
|
||||
if self.workspace.startswith("powerbi://"):
|
||||
conn = f"DataSource={self.workspace};User ID={self.username};Password={self.password};Initial Catalog={self.catalog}"
|
||||
else:
|
||||
conn = f"DataSource={self.workspace};User ID={self.username};Password={self.password}"
|
||||
|
||||
# 如果是管理员,直接返回连接字符串
|
||||
if self.isAdmin:
|
||||
self.result_excel_name = f"admin_{self.json_name}_{self.model_number}"
|
||||
self.result_full_excel_name = os.path.join(
|
||||
self.result_folder_path, self.result_excel_name
|
||||
)
|
||||
return conn
|
||||
# 如果不是管理员,且提供了角色信息
|
||||
elif self.role:
|
||||
# 如果提供了有效的用户名,返回包含有效用户名和角色的连接字符串
|
||||
if self.effective_username:
|
||||
self.result_excel_name = (
|
||||
f"{self.role}_{self.effective_username}_{self.json_name}_{self.model_number}"
|
||||
)
|
||||
self.result_full_excel_name = os.path.join(
|
||||
self.result_folder_path, self.result_excel_name
|
||||
)
|
||||
return f"{conn};EffectiveUserName={self.effective_username};Roles={self.role};"
|
||||
|
||||
# 如果提供了自定义数据,返回包含自定义数据和角色的连接字符串
|
||||
elif self.customdata:
|
||||
self.result_excel_name = (
|
||||
f"{self.role}_{self.customdata}_{self.json_name}_{self.model_number}"
|
||||
)
|
||||
self.result_full_excel_name = os.path.join(
|
||||
self.result_folder_path, self.result_excel_name
|
||||
)
|
||||
return f"{conn};CustomData={self.customdata};Roles={self.role};"
|
||||
|
||||
# 如果只提供了角色,返回包含角色的连接字符串
|
||||
else:
|
||||
self.result_excel_name = f"{self.role}_-NoUser-_{self.json_name}_{self.model_number}"
|
||||
self.result_full_excel_name = os.path.join(
|
||||
self.result_folder_path, self.result_excel_name
|
||||
)
|
||||
return f"{conn};Roles={self.role};"
|
||||
# 如果不是管理员且未提供角色信息,抛出异常
|
||||
else:
|
||||
raise ValueError("非管理员用户必须提供角色信息")
|
||||
|
||||
def readData(self, reader):
|
||||
# 获取列名
|
||||
column_names = [reader.GetName(i) for i in range(reader.FieldCount)]
|
||||
# 初始化数据存储
|
||||
data = []
|
||||
while reader.Read():
|
||||
# 读取每一行的数据
|
||||
row_data = [reader[i] for i in range(reader.FieldCount)]
|
||||
# 将行数据添加到数据列表中
|
||||
data.append(row_data)
|
||||
# 将数据转换为 DataFrame
|
||||
df = pd.DataFrame(data, columns=column_names)
|
||||
# 返回DataFrame和下一个结果集
|
||||
return df, reader.NextResult()
|
||||
|
||||
def getDataFromAS(self, query):
|
||||
|
||||
# 创建AdomdConnection对象
|
||||
conn = AdomdConnection(self.conn_string)
|
||||
try:
|
||||
# 建立连接
|
||||
conn.Open()
|
||||
# 创建命令
|
||||
cmd = AdomdCommand(query, conn)
|
||||
# 执行命令
|
||||
reader = cmd.ExecuteReader()
|
||||
IsNext = True
|
||||
while IsNext:
|
||||
# 将结果集转换为DataFrame
|
||||
df, IsNext = self.readData(reader)
|
||||
# 重命名列名
|
||||
df.columns = [
|
||||
(
|
||||
re.search(r"\[(.*?)\]", col).group(1)
|
||||
if re.search(r"\[(.*?)\]", col)
|
||||
else col
|
||||
)
|
||||
for col in df.columns
|
||||
]
|
||||
yield df
|
||||
except Exception as ex:
|
||||
message = f"错误信息: {ex}"
|
||||
print(message)
|
||||
self.log_messages.append(f"\n{message}")
|
||||
finally:
|
||||
# 关闭连接
|
||||
conn.Close()
|
||||
|
||||
def writeToExcel(self, print_lock):
|
||||
# 创建一个ExcelWriter对象
|
||||
with pd.ExcelWriter(
|
||||
f"{self.result_full_excel_name}.xlsx", engine="openpyxl"
|
||||
) as writer:
|
||||
sheet_name_id = 1
|
||||
# 将DataFrame写入不同的工作表
|
||||
for query in self.queries:
|
||||
for df in self.getDataFromAS(query):
|
||||
try:
|
||||
# 访问工作簿和工作表
|
||||
workbook = writer.book
|
||||
worksheet = workbook.create_sheet(str(sheet_name_id))
|
||||
worksheet["A1"] = query
|
||||
|
||||
# 将DataFrame写入Excel文件
|
||||
df.to_excel(
|
||||
writer,
|
||||
sheet_name=str(sheet_name_id),
|
||||
index=True,
|
||||
float_format="%.2f",
|
||||
startrow=1,
|
||||
)
|
||||
except Exception as e:
|
||||
with print_lock:
|
||||
message = f"写入工作表 {sheet_name_id} 失败: {e}"
|
||||
print(message)
|
||||
self.log_messages.append(f"\n{message}")
|
||||
self.error_list.append(sheet_name_id)
|
||||
else:
|
||||
with print_lock:
|
||||
message = f"{self.result_excel_name} \n表格大小:{df.shape} \n写入工作表 {sheet_name_id}"
|
||||
print(message)
|
||||
self.log_messages.append(f"\n{message}")
|
||||
finally:
|
||||
with print_lock:
|
||||
message = f"-------------------------"
|
||||
print(message)
|
||||
self.log_messages.append(f"{message}")
|
||||
sheet_name_id += 1
|
||||
return self.error_list
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pass
|
37
getQueries.py
Normal file
37
getQueries.py
Normal file
@ -0,0 +1,37 @@
|
||||
import json
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def getQueries_From_json(json_name):
|
||||
# 打开并读取JSON文件
|
||||
with open(json_name + ".json", "r", encoding="utf-8-sig") as file:
|
||||
data = json.load(file)
|
||||
# 从JSON数据中提取所有名为"Execute DAX Query"的事件中的QueryText
|
||||
queries = [
|
||||
event["metrics"]["QueryText"]
|
||||
for event in data["events"]
|
||||
if event["name"] == "Execute DAX Query"
|
||||
]
|
||||
return queries
|
||||
|
||||
|
||||
def getQueries_From_excel(excel_name):
|
||||
# 使用pandas读取Excel文件
|
||||
df = pd.read_excel(excel_name, engine="openpyxl")
|
||||
# 在DataFrame中添加一个新的列"NAME",其值为"EVALUATE '" + 原始的"NAME"列值 + "'"
|
||||
df["NAME"] = "EVALUATE '" + df["NAME"] + "'"
|
||||
# 将新的"NAME"列转换为列表
|
||||
queries = df["NAME"].to_list()
|
||||
return queries
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import os
|
||||
from config import CURRENT_DIR_PATH, EXPORT_PAGE_JSON_NAME_1
|
||||
|
||||
# 获取当前脚本所在的目录路径
|
||||
EXPORT_PAGE_JSON_PATH_1 = os.path.join(
|
||||
CURRENT_DIR_PATH, "Export_Json", EXPORT_PAGE_JSON_NAME_1
|
||||
)
|
||||
queries = getQueries_From_json(EXPORT_PAGE_JSON_PATH_1)
|
||||
print(queries)
|
BIN
requirements.txt
Normal file
BIN
requirements.txt
Normal file
Binary file not shown.
112
run.py
Normal file
112
run.py
Normal file
@ -0,0 +1,112 @@
|
||||
import os
|
||||
import threading
|
||||
from compareData import Comparator
|
||||
from getDataFromAS import ASDataFetcher
|
||||
from getQueries import getQueries_From_json
|
||||
import config
|
||||
|
||||
# 拼接导出页面查询json文件夹路径
|
||||
json_folder_path = os.path.join(config.CURRENT_DIR_PATH, "Export_Json")
|
||||
# 拼接查询结果excel文件夹路径
|
||||
result_folder_path = os.path.join(config.CURRENT_DIR_PATH, "Evaluate_Result")
|
||||
# 检查文件夹是否存在,如果不存在则创建它
|
||||
if not os.path.exists(json_folder_path):
|
||||
os.makedirs(json_folder_path)
|
||||
if not os.path.exists(result_folder_path):
|
||||
os.makedirs(result_folder_path)
|
||||
|
||||
# 拼接导出页面查询json文件路径
|
||||
export_page_json_path_1 = os.path.join(json_folder_path, config.EXPORT_PAGE_JSON_NAME_1)
|
||||
export_page_json_path_2 = os.path.join(json_folder_path, config.EXPORT_PAGE_JSON_NAME_2)
|
||||
# 读取json文件中的查询
|
||||
queries_1 = getQueries_From_json(export_page_json_path_1)
|
||||
queries_2 = getQueries_From_json(export_page_json_path_2)
|
||||
|
||||
# 创建 ASDataFetcher 实例
|
||||
fetcher_1 = ASDataFetcher(
|
||||
workspace=config.WORKSPACE_1,
|
||||
username=config.USERNAME_1,
|
||||
password=config.PASSWORD_1,
|
||||
queries=queries_1,
|
||||
json_name=config.EXPORT_PAGE_JSON_NAME_1,
|
||||
result_folder_path=result_folder_path,
|
||||
isAdmin=config.IS_ADMIN_1,
|
||||
catalog=config.CATALOG_1,
|
||||
effective_username=config.EFFECTIVE_USERNAME_1,
|
||||
customdata=config.CUSTOMER_DATA_1,
|
||||
role=config.ROLE_1,
|
||||
model_number=1,
|
||||
)
|
||||
|
||||
# 创建 ASDataFetcher 实例
|
||||
fetcher_2 = ASDataFetcher(
|
||||
workspace=config.WORKSPACE_2,
|
||||
username=config.USERNAME_2,
|
||||
password=config.PASSWORD_2,
|
||||
queries=queries_2,
|
||||
json_name=config.EXPORT_PAGE_JSON_NAME_2,
|
||||
result_folder_path=result_folder_path,
|
||||
isAdmin=config.IS_ADMIN_2,
|
||||
catalog=config.CATALOG_2,
|
||||
effective_username=config.EFFECTIVE_USERNAME_2,
|
||||
customdata=config.CUSTOMER_DATA_2,
|
||||
role=config.ROLE_2,
|
||||
model_number=2,
|
||||
)
|
||||
|
||||
# 定义线程任务
|
||||
def run_fetcher(fetcher, print_lock):
|
||||
fetcher.writeToExcel(print_lock)
|
||||
|
||||
# 创建锁
|
||||
print_lock = threading.Lock()
|
||||
|
||||
# 创建线程
|
||||
thread_1 = threading.Thread(target=run_fetcher, args=(fetcher_1, print_lock))
|
||||
thread_2 = threading.Thread(target=run_fetcher, args=(fetcher_2, print_lock))
|
||||
|
||||
try:
|
||||
print(
|
||||
f"""
|
||||
---*********************************************************---
|
||||
开始查询
|
||||
"""
|
||||
)
|
||||
# 启动线程
|
||||
thread_1.start()
|
||||
thread_2.start()
|
||||
|
||||
# 等待线程完成
|
||||
thread_1.join()
|
||||
thread_2.join()
|
||||
|
||||
print(
|
||||
f"""
|
||||
查询完成
|
||||
---*********************************************************---
|
||||
"""
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"错误提示: {e}")
|
||||
|
||||
try:
|
||||
print(
|
||||
"""
|
||||
---------------------------------------------------------------
|
||||
开始比较
|
||||
---------------------------------------------------------------
|
||||
"""
|
||||
)
|
||||
comparator = Comparator(
|
||||
fetcher_1.result_full_excel_name, fetcher_2.result_full_excel_name
|
||||
)
|
||||
comparator.compare_ExcelFiles()
|
||||
print(
|
||||
"""
|
||||
----------------------------------------------------------------
|
||||
比较完成
|
||||
----------------------------------------------------------------
|
||||
"""
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"错误提示: {e}")
|
111
templates/index.html
Normal file
111
templates/index.html
Normal file
@ -0,0 +1,111 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-cn">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>数据对比</title>
|
||||
<style>
|
||||
.container {
|
||||
height: 100vh;
|
||||
display: flex;
|
||||
}
|
||||
.left {
|
||||
flex: 40%; /* 左边 div 占 40% */
|
||||
display: table;
|
||||
flex-direction: column;
|
||||
background-color: #f0f0f0;
|
||||
padding: 10px;
|
||||
box-sizing: border-box;
|
||||
overflow-y: auto;
|
||||
}
|
||||
.config {
|
||||
flex: 1; /* 上半部分 */
|
||||
display: inline-table;
|
||||
flex-direction: row;
|
||||
}
|
||||
.config-content {
|
||||
display: flex;
|
||||
flex: 100%;
|
||||
}
|
||||
.buttons {
|
||||
display: flex;
|
||||
flex-direction: revert-layer;
|
||||
align-items: center;
|
||||
margin-top: 10px;
|
||||
flex-shrink: 0; /* 防止按钮区域收缩 */
|
||||
justify-content: space-around;
|
||||
}
|
||||
.result {
|
||||
flex: 60%; /* 右边 div 占 60% */
|
||||
background-color: #f9eded;
|
||||
padding: 10px;
|
||||
box-sizing: border-box;
|
||||
overflow-y: auto;
|
||||
}
|
||||
.description {
|
||||
color: gray;
|
||||
}
|
||||
.config .config1, .config2 {
|
||||
flex: 50%; /* 修改 flex 值为 50% */
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
.config-item {
|
||||
display: block;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 10px;
|
||||
margin-bottom: 15px;
|
||||
}
|
||||
.config-keys {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 10px;
|
||||
}
|
||||
.config-desc {
|
||||
grid-column: 1 / -1;
|
||||
text-align: left;
|
||||
}
|
||||
.config-inputs {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 10px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="left">
|
||||
<div class="config">
|
||||
<h2>配置项</h2>
|
||||
<form action="{{ url_for('update_config') }}" method="post">
|
||||
{% for key1 in config.__dict__.keys() if key1.endswith('_1') %}
|
||||
{% set base_key = key1[:-2] %}
|
||||
{% set key2 = base_key + '_2' %}
|
||||
{% set desc_key = 'desc_' + base_key %}
|
||||
<div class="config-item">
|
||||
<div class="config-keys">
|
||||
<label for="{{ key1 }}">{{ key1 }}</label>
|
||||
<label for="{{ key2 }}">{{ key2 }}</label>
|
||||
</div>
|
||||
<div class="config-desc">
|
||||
<span class="description">{{ config.__dict__.get(desc_key, '') }}</span>
|
||||
</div>
|
||||
<div class="config-inputs">
|
||||
<input type="text" id="{{ key1 }}" name="{{ key1 }}" value="{{ config.__dict__[key1] }}">
|
||||
<input type="text" id="{{ key2 }}" name="{{ key2 }}" value="{{ config.__dict__[key2] }}">
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
<div class="buttons">
|
||||
<button type="submit">保存配置</button>
|
||||
<button type="submit" formaction="{{ url_for('run') }}">运行</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<div class="result">
|
||||
<h1>运行结果</h1>
|
||||
<pre>{{ result }}</pre>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
Loading…
Reference in New Issue
Block a user