commit 9bff0a7c7b09301193a82302ed182e13bdf85c9d Author: chenwu Date: Sun Dec 22 16:46:52 2024 +0800 V1.0 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dfe0770 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Auto detect text files and perform LF normalization +* text=auto diff --git a/Evaluate_Result/admin_example_1.xlsx b/Evaluate_Result/admin_example_1.xlsx new file mode 100644 index 0000000..ec878b4 Binary files /dev/null and b/Evaluate_Result/admin_example_1.xlsx differ diff --git a/Evaluate_Result/admin_example_2.xlsx b/Evaluate_Result/admin_example_2.xlsx new file mode 100644 index 0000000..ec878b4 Binary files /dev/null and b/Evaluate_Result/admin_example_2.xlsx differ diff --git a/ExamplePBIX/Examples.pbix b/ExamplePBIX/Examples.pbix new file mode 100644 index 0000000..da7ba38 Binary files /dev/null and b/ExamplePBIX/Examples.pbix differ diff --git a/Export_Json/example.json b/Export_Json/example.json new file mode 100644 index 0000000..bb842c0 --- /dev/null +++ b/Export_Json/example.json @@ -0,0 +1 @@ +{"version":"1.1.0","events":[{"name":"User Action","component":"Report Canvas","start":"2024-12-21T17:14:40.987Z","id":"515c279a49002a814996","metrics":{"sourceLabel":"UserAction_StartedMonitoring"}},{"name":"User Action","component":"Report Canvas","start":"2024-12-21T17:14:42.623Z","id":"79b174c0d02ca69ada00","metrics":{"sourceLabel":"UserAction_Refresh"}},{"name":"Visual Container Lifecycle","start":"2024-12-21T17:14:42.629Z","end":"2024-12-21T17:14:42.682Z","id":"24c30acf43d0ab30dd40-b004a31d5ccd6a4a5385","metrics":{"status":1,"visualTitle":"Table","visualId":"24c30acf43d0ab30dd40","visualType":"tableEx","initialLoad":false}},{"name":"Resolve Parameters","start":"2024-12-21T17:14:42.630Z","end":"2024-12-21T17:14:42.631Z","id":"865f086f-a553-4057-9963-f386c6ea310f","parentId":"24c30acf43d0ab30dd40-b004a31d5ccd6a4a5385"},{"name":"Query","start":"2024-12-21T17:14:42.631Z","end":"2024-12-21T17:14:42.662Z","id":"f374c5d4-aeae-49e4-a63f-a086480fbad1","parentId":"24c30acf43d0ab30dd40-b004a31d5ccd6a4a5385"},{"name":"Render","start":"2024-12-21T17:14:42.661Z","end":"2024-12-21T17:14:42.682Z","id":"bf797551-b336-43c1-8a29-eda4f83f224e","parentId":"24c30acf43d0ab30dd40-b004a31d5ccd6a4a5385"},{"name":"Data View Transform","start":"2024-12-21T17:14:42.670Z","end":"2024-12-21T17:14:42.673Z","id":"2b79f57f-9353-48e9-afe8-0c1f446f8636","parentId":"bf797551-b336-43c1-8a29-eda4f83f224e"},{"name":"Visual Update Async","start":"2024-12-21T17:14:42.673Z","end":"2024-12-21T17:14:42.682Z","id":"ca7f9db7-b9c5-4739-aced-6a8a5560f0a8","parentId":"bf797551-b336-43c1-8a29-eda4f83f224e"},{"name":"Visual Update","start":"2024-12-21T17:14:42.673Z","end":"2024-12-21T17:14:42.673Z","id":"29d7c7a6-ef0d-416a-973f-cf4ddacedc8a","parentId":"bf797551-b336-43c1-8a29-eda4f83f224e"},{"name":"Execute Semantic Query","component":"DSE","start":"2024-12-21T17:14:42.649Z","end":"2024-12-21T17:14:42.656Z","id":"47aeb947-80d7-4096-9f2d-3b16c8b5b5e9","parentId":"f374c5d4-aeae-49e4-a63f-a086480fbad1"},{"name":"Query Generation","start":"2024-12-21T17:14:42.633Z","end":"2024-12-21T17:14:42.634Z","id":"4fa46d04-aa1d-46ea-933b-49002a24c309","parentId":"f374c5d4-aeae-49e4-a63f-a086480fbad1"},{"name":"Query Pending","start":"2024-12-21T17:14:42.648Z","end":"2024-12-21T17:14:42.649Z","id":"178f1281-20eb-4c78-a3d1-bdc9f6735a7c","parentId":"f374c5d4-aeae-49e4-a63f-a086480fbad1"},{"name":"Parse Query Result","start":"2024-12-21T17:14:42.658Z","end":"2024-12-21T17:14:42.660Z","id":"33e79f0c-bc13-4951-a18e-6fdbd767e4f8","parentId":"f374c5d4-aeae-49e4-a63f-a086480fbad1"},{"name":"Execute DAX Query","component":"DSE","start":"2024-12-21T17:14:42.651Z","end":"2024-12-21T17:14:42.655Z","id":"e458b6a8-0b2b-474a-a8f1-294856878497","parentId":"47aeb947-80d7-4096-9f2d-3b16c8b5b5e9","metrics":{"QueryText":"DEFINE\r\n\tVAR __DS0Core = \r\n\t\tSUMMARIZECOLUMNS(\r\n\t\t\tROLLUPADDISSUBTOTAL('financials'[Country], \"IsGrandTotalRowTotal\"),\r\n\t\t\t\"Sumv_Sales\", CALCULATE(SUM('financials'[ Sales]))\r\n\t\t)\r\n\r\n\tVAR __DS0PrimaryWindowed = \r\n\t\tTOPN(502, __DS0Core, [IsGrandTotalRowTotal], 0, 'financials'[Country], 1)\r\n\r\nEVALUATE\r\n\t__DS0PrimaryWindowed\r\n\r\nORDER BY\r\n\t[IsGrandTotalRowTotal] DESC, 'financials'[Country]","RowCount":6}},{"name":"Execute Query","component":"AS","start":"2024-12-21T17:14:42.653Z","end":"2024-12-21T17:14:42.657Z","id":"3BC9FA84-D491-49ED-B2A1-7DC0D4F32A4C","parentId":"e458b6a8-0b2b-474a-a8f1-294856878497"},{"name":"Serialize Rowset","component":"AS","start":"2024-12-21T17:14:42.657Z","end":"2024-12-21T17:14:42.657Z","id":"57BE528D-04A5-4696-A0AF-3327CD0CE702","parentId":"3BC9FA84-D491-49ED-B2A1-7DC0D4F32A4C"}],"sessionId":"14d9da6f-0492-428d-bcff-ece9e2ac26f6"} \ No newline at end of file diff --git a/MS.NET_Package/Microsoft.AnalysisServices.AdomdClient/v4.0_15.0.0.0__89845dcd8080cc91/Microsoft.AnalysisServices.AdomdClient.dll b/MS.NET_Package/Microsoft.AnalysisServices.AdomdClient/v4.0_15.0.0.0__89845dcd8080cc91/Microsoft.AnalysisServices.AdomdClient.dll new file mode 100644 index 0000000..9730631 Binary files /dev/null and b/MS.NET_Package/Microsoft.AnalysisServices.AdomdClient/v4.0_15.0.0.0__89845dcd8080cc91/Microsoft.AnalysisServices.AdomdClient.dll differ diff --git a/__pycache__/app.cpython-312.pyc b/__pycache__/app.cpython-312.pyc new file mode 100644 index 0000000..d98985a Binary files /dev/null and b/__pycache__/app.cpython-312.pyc differ diff --git a/__pycache__/compareData.cpython-312.pyc b/__pycache__/compareData.cpython-312.pyc new file mode 100644 index 0000000..01d2065 Binary files /dev/null and b/__pycache__/compareData.cpython-312.pyc differ diff --git a/__pycache__/config.cpython-312.pyc b/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000..c71e6ba Binary files /dev/null and b/__pycache__/config.cpython-312.pyc differ diff --git a/__pycache__/getDataFromAS.cpython-312.pyc b/__pycache__/getDataFromAS.cpython-312.pyc new file mode 100644 index 0000000..1265892 Binary files /dev/null and b/__pycache__/getDataFromAS.cpython-312.pyc differ diff --git a/__pycache__/getQueries.cpython-312.pyc b/__pycache__/getQueries.cpython-312.pyc new file mode 100644 index 0000000..58a421f Binary files /dev/null and b/__pycache__/getQueries.cpython-312.pyc differ diff --git a/__pycache__/writeDataToExcel.cpython-312.pyc b/__pycache__/writeDataToExcel.cpython-312.pyc new file mode 100644 index 0000000..c19c1ca Binary files /dev/null and b/__pycache__/writeDataToExcel.cpython-312.pyc differ diff --git a/app.py b/app.py new file mode 100644 index 0000000..edf06fb --- /dev/null +++ b/app.py @@ -0,0 +1,138 @@ +from flask import Flask, request, render_template, redirect, url_for +import os +import threading +import config +from getDataFromAS import ASDataFetcher +from getQueries import getQueries_From_json +from compareData import Comparator + +app = Flask(__name__) + +result = "" + +@app.route('/') +def index(): + global result + return render_template('index.html', config=config, result=result) + +@app.route('/update_config', methods=['POST']) +def update_config(): + for key, value in request.form.items(): + if hasattr(config, key) and key not in ['CURRENT_DIR_PATH', 'os', 'NET_FOLDER']: + setattr(config, key, value) + return redirect(url_for('index')) + +@app.route('/run', methods=['POST']) +def run(): + global result + result = "" + + # 拼接导出页面查询json文件夹路径 + json_folder_path = os.path.join(config.CURRENT_DIR_PATH, "Export_Json") + # 拼接查询结果excel文件夹路径 + result_folder_path = os.path.join(config.CURRENT_DIR_PATH, "Evaluate_Result") + # 检查文件夹是否存在，如果不存在则创建它 + if not os.path.exists(json_folder_path): + os.makedirs(json_folder_path) + if not os.path.exists(result_folder_path): + os.makedirs(result_folder_path) + + # 拼接导出页面查询json文件路径 + export_page_json_path_1 = os.path.join(json_folder_path, config.EXPORT_PAGE_JSON_NAME_1) + export_page_json_path_2 = os.path.join(json_folder_path, config.EXPORT_PAGE_JSON_NAME_2) + # 读取json文件中的查询 + queries_1 = getQueries_From_json(export_page_json_path_1) + queries_2 = getQueries_From_json(export_page_json_path_2) + + # 创建 ASDataFetcher 实例 + fetcher_1 = ASDataFetcher( + workspace=config.WORKSPACE_1, + username=config.USERNAME_1, + password=config.PASSWORD_1, + queries=queries_1, + json_name=config.EXPORT_PAGE_JSON_NAME_1, + result_folder_path=result_folder_path, + isAdmin=config.IS_ADMIN_1, + catalog=config.CATALOG_1, + effective_username=config.EFFECTIVE_USERNAME_1, + customdata=config.CUSTOMER_DATA_1, + role=config.ROLE_1, + model_number=1, + ) + + # 创建 ASDataFetcher 实例 + fetcher_2 = ASDataFetcher( + workspace=config.WORKSPACE_2, + username=config.USERNAME_2, + password=config.PASSWORD_2, + queries=queries_2, + json_name=config.EXPORT_PAGE_JSON_NAME_2, + result_folder_path=result_folder_path, + isAdmin=config.IS_ADMIN_2, + catalog=config.CATALOG_2, + effective_username=config.EFFECTIVE_USERNAME_2, + customdata=config.CUSTOMER_DATA_2, + role=config.ROLE_2, + model_number=2, + ) + + # 定义线程任务 + def run_fetcher(fetcher, print_lock): + global result + try: + fetcher.writeToExcel(print_lock) + with print_lock: + result += f"\n-------------------------\n{fetcher.json_name} 查询完成\n" + result += "\n".join(fetcher.log_messages) + "\n" + except Exception as e: + with print_lock: + result += f"错误提示: {e}\n" + result += "\n".join(fetcher.log_messages) + "\n" + return + + # 创建锁 + print_lock = threading.Lock() + + # 创建线程 + thread_1 = threading.Thread(target=run_fetcher, args=(fetcher_1, print_lock)) + thread_2 = threading.Thread(target=run_fetcher, args=(fetcher_2, print_lock)) + + try: + result += "---*********************************************************---\n" + result += " 开始查询\n" + result += "---*********************************************************---\n" + + # 启动线程 + thread_1.start() + thread_2.start() + + # 等待线程完成 + thread_1.join() + thread_2.join() + + result += "\n---*********************************************************---\n" + result += " 查询完成\n" + result += "---*********************************************************---\n" + + except Exception as e: + result += f"错误提示: {e}\n" + return redirect(url_for('index')) + + try: + result += "\n---------------------------------------------------------------\n" + result += " 开始比较\n" + result += "---------------------------------------------------------------\n" + + comparator = Comparator(fetcher_1.result_full_excel_name, fetcher_2.result_full_excel_name) + comparator.compare_ExcelFiles() + result += "\n".join(comparator.log_messages) + "\n" + result += "\n---------------------------------------------------------------\n" + result += " 比较完成\n" + result += "---------------------------------------------------------------\n" + except Exception as e: + result += f"错误提示: {e}\n" + + return redirect(url_for('index')) + +if __name__ == '__main__': + app.run(debug=True) diff --git a/compareData.py b/compareData.py new file mode 100644 index 0000000..74e10f9 --- /dev/null +++ b/compareData.py @@ -0,0 +1,65 @@ +import pandas as pd + + +class Comparator: + def __init__(self, file1, file2): + self.file1 = file1 + self.file2 = file2 + self.log_messages = [] + + def compare_ExcelFiles(self): + # 读取Excel文件 + xls1 = pd.ExcelFile(f"{self.file1}.xlsx", engine="openpyxl") + xls2 = pd.ExcelFile(f"{self.file2}.xlsx", engine="openpyxl") + + # 获取所有工作表的名称 + sheet_names1 = set(xls1.sheet_names) + sheet_names2 = set(xls2.sheet_names) + all_sheet_names = sheet_names1.union(sheet_names2) + message = f"文件1有{len(sheet_names1)}个sheet，文件2有{len(sheet_names2)}个sheet" + print(message) + self.log_messages.append(f"\n{message}") + + # 读取Excel文件中的特定工作表 + for i in all_sheet_names: + if i not in sheet_names1: + message = f"文件1中不存在sheet: {i}" + print(message) + self.log_messages.append(f"\n{message}") + continue + if i not in sheet_names2: + message = f"文件2中不存在sheet: {i}" + print(message) + self.log_messages.append(f"\n{message}") + continue + + df1 = pd.read_excel( + f"{self.file1}.xlsx", sheet_name=i, engine="openpyxl", skiprows=1 + ) + df2 = pd.read_excel( + f"{self.file2}.xlsx", sheet_name=i, engine="openpyxl", skiprows=1 + ) + # 比较两个DataFrame中的数据 + if df1.equals(df2): + message = f"第{i}个sheet的数据相同" + print(message) + self.log_messages.append(f"\n{message}") + else: + message = f"第{i}个sheet的数据不同\n----------------------------\n不同数据如下：" + print(message) + self.log_messages.append(f"\n{message}") + try: + message = df1.compare(df2) + print(message) + self.log_messages.append(f"\n{message}") + except ValueError: + message = "数据列名称不同" + print(message) + self.log_messages.append(f"\n{message}") + message = "----------------------------" + print(message) + self.log_messages.append(f"\n{message}") + + +if __name__ == "__main__": + pass diff --git a/config.py b/config.py new file mode 100644 index 0000000..789b52a --- /dev/null +++ b/config.py @@ -0,0 +1,57 @@ +import os + +# 获取当前文件所在的文件夹路径 +CURRENT_DIR_PATH = os.path.dirname(os.path.abspath(__file__)) +# MS.NET包的路径 +NET_FOLDER = "MS.NET_Package" + +# -------------------------------------------------------------- +# ------------------------请填写以下信息------------------------- +# -------------------------------------------------------------- + +# 连接信息 +### 第一个Analysis Services的连接信息 +WORKSPACE_1 = "localhost:9088" +USERNAME_1 = "xxxx" +PASSWORD_1 = "xxxx" +CATALOG_1 = None +### 第二个Analysis Services的连接信息 +WORKSPACE_2 = "localhost:9088" +USERNAME_2 = "xxxx" +PASSWORD_2 = "xxxx" +CATALOG_2 = None + +# 导出页面查询的json文件的名称 +EXPORT_PAGE_JSON_NAME_1 = "example" +EXPORT_PAGE_JSON_NAME_2 = "example" + +# 是否是管理员方式运行，如果是，则为True，否则为False +IS_ADMIN_1 = True +IS_ADMIN_2 = True +# 自定义数据和角色 +### 如果定义IS_ADMIN为False，则需要填写以下信息 +### EFFECTIVE_USERNAME和CUSTOMER_DATA都填写只有一个生效，如果不需要生效的请填写None。 +### 如果都填写，优先生效的顺序为： EFFECTIVE_USERNAME -> CUSTOMER_DATA + +### 第一个Analysis Services的自定义数据和角色 +ROLE_1 = "1" +EFFECTIVE_USERNAME_1 = None +CUSTOMER_DATA_1 = "1" +### 第二个Analysis Services的自定义数据和角色 +ROLE_2 = "2" +EFFECTIVE_USERNAME_2 = None +CUSTOMER_DATA_2 = "2" + +# -------------------------------------------------------------- +# ------------------------请填写以上信息------------------------- +# -------------------------------------------------------------- + +desc_WORKSPACE = "Analysis Services的连接信息，如果是powerbi，则连接字符串中包含catalog" +desc_USERNAME = "用户名" +desc_PASSWORD = "密码" +desc_CATALOG = "如果是powerbi，则catalog为语义模型的名称，默认为None" +desc_EXPORT_PAGE_JSON_NAME = "导出页面查询的json文件的名称" +desc_IS_ADMIN = "是否是管理员方式运行，如果是，则为True，否则为False" +desc_ROLE = "角色名称" +desc_EFFECTIVE_USERNAME = "username()获取字段" +desc_CUSTOMER_DATA = "customerdata()获取字段" \ No newline at end of file diff --git a/getDataFromAS.py b/getDataFromAS.py new file mode 100644 index 0000000..c49daf4 --- /dev/null +++ b/getDataFromAS.py @@ -0,0 +1,189 @@ +import re +import pandas as pd +import clr +import os +from config import CURRENT_DIR_PATH, NET_FOLDER + +folder = os.path.join(CURRENT_DIR_PATH, NET_FOLDER) +clr.AddReference( + folder + + r"\Microsoft.AnalysisServices.AdomdClient\v4.0_15.0.0.0__89845dcd8080cc91\Microsoft.AnalysisServices.AdomdClient.DLL" +) + +from Microsoft.AnalysisServices.AdomdClient import AdomdConnection, AdomdCommand # type: ignore + +class ASDataFetcher: + def __init__( + self, + workspace, + username, + password, + queries, + json_name, + result_folder_path, + isAdmin=True, + catalog=None, + effective_username=None, + customdata=None, + role=None, + model_number=None, + ): + self.workspace = workspace + self.username = username + self.password = password + self.queries = queries + self.isAdmin = isAdmin + self.effective_username = effective_username + self.customdata = customdata + self.role = role + self.result_folder_path = result_folder_path + self.error_list = [] + self.json_name = json_name + self.model_number = model_number + self.log_messages = [] + self.catalog = catalog + + # 先将当前对象的 connString 方法的返回值赋值给当前对象的 conn_string 属性, 以便后续使用 + self.conn_string = self.connString() + + def connString(self) -> str: + # 初始化连接字符串，包含数据源、用户名和密码 + # 如果workspace开头是powerbi，则连接字符串中包含catalog + if self.workspace.startswith("powerbi://"): + conn = f"DataSource={self.workspace};User ID={self.username};Password={self.password};Initial Catalog={self.catalog}" + else: + conn = f"DataSource={self.workspace};User ID={self.username};Password={self.password}" + + # 如果是管理员，直接返回连接字符串 + if self.isAdmin: + self.result_excel_name = f"admin_{self.json_name}_{self.model_number}" + self.result_full_excel_name = os.path.join( + self.result_folder_path, self.result_excel_name + ) + return conn + # 如果不是管理员，且提供了角色信息 + elif self.role: + # 如果提供了有效的用户名，返回包含有效用户名和角色的连接字符串 + if self.effective_username: + self.result_excel_name = ( + f"{self.role}_{self.effective_username}_{self.json_name}_{self.model_number}" + ) + self.result_full_excel_name = os.path.join( + self.result_folder_path, self.result_excel_name + ) + return f"{conn};EffectiveUserName={self.effective_username};Roles={self.role};" + + # 如果提供了自定义数据，返回包含自定义数据和角色的连接字符串 + elif self.customdata: + self.result_excel_name = ( + f"{self.role}_{self.customdata}_{self.json_name}_{self.model_number}" + ) + self.result_full_excel_name = os.path.join( + self.result_folder_path, self.result_excel_name + ) + return f"{conn};CustomData={self.customdata};Roles={self.role};" + + # 如果只提供了角色，返回包含角色的连接字符串 + else: + self.result_excel_name = f"{self.role}_-NoUser-_{self.json_name}_{self.model_number}" + self.result_full_excel_name = os.path.join( + self.result_folder_path, self.result_excel_name + ) + return f"{conn};Roles={self.role};" + # 如果不是管理员且未提供角色信息，抛出异常 + else: + raise ValueError("非管理员用户必须提供角色信息") + + def readData(self, reader): + # 获取列名 + column_names = [reader.GetName(i) for i in range(reader.FieldCount)] + # 初始化数据存储 + data = [] + while reader.Read(): + # 读取每一行的数据 + row_data = [reader[i] for i in range(reader.FieldCount)] + # 将行数据添加到数据列表中 + data.append(row_data) + # 将数据转换为 DataFrame + df = pd.DataFrame(data, columns=column_names) + # 返回DataFrame和下一个结果集 + return df, reader.NextResult() + + def getDataFromAS(self, query): + + # 创建AdomdConnection对象 + conn = AdomdConnection(self.conn_string) + try: + # 建立连接 + conn.Open() + # 创建命令 + cmd = AdomdCommand(query, conn) + # 执行命令 + reader = cmd.ExecuteReader() + IsNext = True + while IsNext: + # 将结果集转换为DataFrame + df, IsNext = self.readData(reader) + # 重命名列名 + df.columns = [ + ( + re.search(r"\[(.*?)\]", col).group(1) + if re.search(r"\[(.*?)\]", col) + else col + ) + for col in df.columns + ] + yield df + except Exception as ex: + message = f"错误信息: {ex}" + print(message) + self.log_messages.append(f"\n{message}") + finally: + # 关闭连接 + conn.Close() + + def writeToExcel(self, print_lock): + # 创建一个ExcelWriter对象 + with pd.ExcelWriter( + f"{self.result_full_excel_name}.xlsx", engine="openpyxl" + ) as writer: + sheet_name_id = 1 + # 将DataFrame写入不同的工作表 + for query in self.queries: + for df in self.getDataFromAS(query): + try: + # 访问工作簿和工作表 + workbook = writer.book + worksheet = workbook.create_sheet(str(sheet_name_id)) + worksheet["A1"] = query + + # 将DataFrame写入Excel文件 + df.to_excel( + writer, + sheet_name=str(sheet_name_id), + index=True, + float_format="%.2f", + startrow=1, + ) + except Exception as e: + with print_lock: + message = f"写入工作表 {sheet_name_id} 失败: {e}" + print(message) + self.log_messages.append(f"\n{message}") + self.error_list.append(sheet_name_id) + else: + with print_lock: + message = f"{self.result_excel_name} \n表格大小：{df.shape} \n写入工作表 {sheet_name_id}" + print(message) + self.log_messages.append(f"\n{message}") + finally: + with print_lock: + message = f"-------------------------" + print(message) + self.log_messages.append(f"{message}") + sheet_name_id += 1 + return self.error_list + + +if __name__ == "__main__": + pass diff --git a/getQueries.py b/getQueries.py new file mode 100644 index 0000000..5e76438 --- /dev/null +++ b/getQueries.py @@ -0,0 +1,37 @@ +import json +import pandas as pd + + +def getQueries_From_json(json_name): + # 打开并读取JSON文件 + with open(json_name + ".json", "r", encoding="utf-8-sig") as file: + data = json.load(file) + # 从JSON数据中提取所有名为"Execute DAX Query"的事件中的QueryText + queries = [ + event["metrics"]["QueryText"] + for event in data["events"] + if event["name"] == "Execute DAX Query" + ] + return queries + + +def getQueries_From_excel(excel_name): + # 使用pandas读取Excel文件 + df = pd.read_excel(excel_name, engine="openpyxl") + # 在DataFrame中添加一个新的列"NAME"，其值为"EVALUATE '" + 原始的"NAME"列值 + "'" + df["NAME"] = "EVALUATE '" + df["NAME"] + "'" + # 将新的"NAME"列转换为列表 + queries = df["NAME"].to_list() + return queries + + +if __name__ == "__main__": + import os + from config import CURRENT_DIR_PATH, EXPORT_PAGE_JSON_NAME_1 + + # 获取当前脚本所在的目录路径 + EXPORT_PAGE_JSON_PATH_1 = os.path.join( + CURRENT_DIR_PATH, "Export_Json", EXPORT_PAGE_JSON_NAME_1 + ) + queries = getQueries_From_json(EXPORT_PAGE_JSON_PATH_1) + print(queries) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..fb5c6bc Binary files /dev/null and b/requirements.txt differ diff --git a/run.py b/run.py new file mode 100644 index 0000000..1382c6d --- /dev/null +++ b/run.py @@ -0,0 +1,112 @@ +import os +import threading +from compareData import Comparator +from getDataFromAS import ASDataFetcher +from getQueries import getQueries_From_json +import config + +# 拼接导出页面查询json文件夹路径 +json_folder_path = os.path.join(config.CURRENT_DIR_PATH, "Export_Json") +# 拼接查询结果excel文件夹路径 +result_folder_path = os.path.join(config.CURRENT_DIR_PATH, "Evaluate_Result") +# 检查文件夹是否存在，如果不存在则创建它 +if not os.path.exists(json_folder_path): + os.makedirs(json_folder_path) +if not os.path.exists(result_folder_path): + os.makedirs(result_folder_path) + +# 拼接导出页面查询json文件路径 +export_page_json_path_1 = os.path.join(json_folder_path, config.EXPORT_PAGE_JSON_NAME_1) +export_page_json_path_2 = os.path.join(json_folder_path, config.EXPORT_PAGE_JSON_NAME_2) +# 读取json文件中的查询 +queries_1 = getQueries_From_json(export_page_json_path_1) +queries_2 = getQueries_From_json(export_page_json_path_2) + +# 创建 ASDataFetcher 实例 +fetcher_1 = ASDataFetcher( + workspace=config.WORKSPACE_1, + username=config.USERNAME_1, + password=config.PASSWORD_1, + queries=queries_1, + json_name=config.EXPORT_PAGE_JSON_NAME_1, + result_folder_path=result_folder_path, + isAdmin=config.IS_ADMIN_1, + catalog=config.CATALOG_1, + effective_username=config.EFFECTIVE_USERNAME_1, + customdata=config.CUSTOMER_DATA_1, + role=config.ROLE_1, + model_number=1, +) + +# 创建 ASDataFetcher 实例 +fetcher_2 = ASDataFetcher( + workspace=config.WORKSPACE_2, + username=config.USERNAME_2, + password=config.PASSWORD_2, + queries=queries_2, + json_name=config.EXPORT_PAGE_JSON_NAME_2, + result_folder_path=result_folder_path, + isAdmin=config.IS_ADMIN_2, + catalog=config.CATALOG_2, + effective_username=config.EFFECTIVE_USERNAME_2, + customdata=config.CUSTOMER_DATA_2, + role=config.ROLE_2, + model_number=2, +) + +# 定义线程任务 +def run_fetcher(fetcher, print_lock): + fetcher.writeToExcel(print_lock) + +# 创建锁 +print_lock = threading.Lock() + +# 创建线程 +thread_1 = threading.Thread(target=run_fetcher, args=(fetcher_1, print_lock)) +thread_2 = threading.Thread(target=run_fetcher, args=(fetcher_2, print_lock)) + +try: + print( + f""" + ---*********************************************************--- + 开始查询 + """ + ) + # 启动线程 + thread_1.start() + thread_2.start() + + # 等待线程完成 + thread_1.join() + thread_2.join() + + print( + f""" + 查询完成 + ---*********************************************************--- + """ + ) +except Exception as e: + print(f"错误提示: {e}") + +try: + print( + """ + --------------------------------------------------------------- + 开始比较 + --------------------------------------------------------------- + """ + ) + comparator = Comparator( + fetcher_1.result_full_excel_name, fetcher_2.result_full_excel_name + ) + comparator.compare_ExcelFiles() + print( + """ + ---------------------------------------------------------------- + 比较完成 + ---------------------------------------------------------------- + """ + ) +except Exception as e: + print(f"错误提示: {e}") diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..b6584e6 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,111 @@ + + + + + 数据对比 + + + +
+
+
+

配置项

+
+ {% for key1 in config.__dict__.keys() if key1.endswith('_1') %} + {% set base_key = key1[:-2] %} + {% set key2 = base_key + '_2' %} + {% set desc_key = 'desc_' + base_key %} +
+
+ + +
+
+ {{ config.__dict__.get(desc_key, '') }} +
+
+ + +
+
+ {% endfor %} +
+ + +
+
+
+
+
+

运行结果

+ 
{{ result }}
+
+
+ +