execute-py.js 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879
  1. /**
  2. * 执行 Python 相关操作模块(通用功能)
  3. * 负责图像匹配、OCR识别等通用功能(通过调用 Python/JS 实现)
  4. * 注意:聊天记录提取等业务相关功能在 read-and-write.js 中
  5. */
  6. import { ipcMain } from 'electron';
  7. import { writeFile, mkdir, rm, readdir, stat } from 'fs/promises';
  8. import fs from 'fs';
  9. import { join, dirname, isAbsolute } from 'path';
  10. import { fileURLToPath } from 'url';
  11. import { exec } from 'child_process';
  12. import { promisify } from 'util';
  13. import { captureScreenshot } from './adb/screenshot.js';
  14. import { getDeviceResolution } from './adb/device-info.js';
  15. import { matchImage } from './func/image-center-location.js';
  16. import { findTextLocation } from './func/string-reg-location.js';
  17. import { ocrFullScreen as ocrFullScreenFromFunc, getLastMessage as getLastMessageFromFunc } from './func/ocr-chat.js';
  18. const execAsync = promisify(exec);
  19. const __filename = fileURLToPath(import.meta.url);
  20. const __dirname = dirname(__filename);
  21. /**
  22. * 递归计算目录大小
  23. * @param {string} dirPath - 目录路径
  24. * @returns {Promise<number>} 目录总大小(字节)
  25. */
  26. async function calculateDirSize(dirPath) {
  27. let totalSize = 0;
  28. try {
  29. const entries = await readdir(dirPath, { withFileTypes: true });
  30. for (const entry of entries) {
  31. const entryPath = join(dirPath, entry.name);
  32. try {
  33. if (entry.isFile()) {
  34. const stats = await stat(entryPath);
  35. totalSize += stats.size;
  36. } else if (entry.isDirectory()) {
  37. totalSize += await calculateDirSize(entryPath);
  38. }
  39. } catch (e) {
  40. // 忽略无法访问的文件/目录
  41. }
  42. }
  43. } catch (e) {
  44. // 忽略无法读取的目录
  45. }
  46. return totalSize;
  47. }
  48. /**
  49. * 执行图像匹配:截图、调用 Python 脚本、返回坐标
  50. * @param {string} ipPort - 设备 ID/IP:Port
  51. * @param {string} templateImagePath - 模板图片路径
  52. * @returns {Promise<{success: boolean, error?: string, coordinate?: Object, clickPosition?: Object}>}
  53. */
  54. export async function matchImageAndGetCoordinate(ipPort, templateImagePath) {
  55. try {
  56. if (!ipPort) {
  57. return { success: false, error: '缺少设备 ID' };
  58. }
  59. if (!templateImagePath) {
  60. return { success: false, error: '缺少模板图片路径' };
  61. }
  62. // 将相对路径转换为绝对路径
  63. let absoluteTemplatePath = templateImagePath;
  64. if (!isAbsolute(templateImagePath)) {
  65. absoluteTemplatePath = join(__dirname, '..', templateImagePath);
  66. }
  67. // 1. 获取设备分辨率
  68. const resolutionResult = await getDeviceResolution(ipPort);
  69. if (!resolutionResult.success) {
  70. return { success: false, error: '获取设备分辨率失败' };
  71. }
  72. const { width, height } = resolutionResult;
  73. // 2. 获取屏幕截图
  74. const screenshotResult = await captureScreenshot(ipPort, { format: 'png' });
  75. if (!screenshotResult.success || !screenshotResult.data) {
  76. return { success: false, error: '获取屏幕截图失败' };
  77. }
  78. // 3. 保存截图到临时文件
  79. const tempDir = join(__dirname, '..');
  80. const screenshotPath = join(tempDir, 'temp_screenshot.png');
  81. const screenshotBuffer = Buffer.from(screenshotResult.data, 'base64');
  82. await writeFile(screenshotPath, screenshotBuffer);
  83. // 4. 调用 JS 函数进行图像匹配
  84. const matchResult = await matchImage(screenshotPath, absoluteTemplatePath, width, height);
  85. if (!matchResult.success) {
  86. return { success: false, error: matchResult.error || '图像匹配失败' };
  87. }
  88. // 5. 返回匹配结果
  89. if (matchResult.success && matchResult.x !== undefined) {
  90. const { x, y, width: w, height: h } = matchResult;
  91. // 计算点击位置(中心点)
  92. const clickX = Math.round(x + w / 2);
  93. const clickY = Math.round(y + h / 2);
  94. return {
  95. success: true,
  96. coordinate: { x, y, width: w, height: h },
  97. clickPosition: { x: clickX, y: clickY }
  98. };
  99. } else {
  100. return {
  101. success: false,
  102. error: matchResult.error || '图像匹配失败'
  103. };
  104. }
  105. } catch (error) {
  106. return { success: false, error: error.message };
  107. }
  108. }
  109. /**
  110. * 图像区域定位:在完整截图中查找区域截图的位置,返回四个顶点坐标
  111. * @param {string} screenshotPath - 完整截图路径
  112. * @param {string} regionPath - 区域截图路径
  113. * @param {string} device - 设备 ID(可选,用于获取分辨率)
  114. * @returns {Promise<{success: boolean, error?: string, corners?: Object}>}
  115. */
  116. export async function matchImageRegionLocation(screenshotPath, regionPath, device = null) {
  117. try {
  118. if (!regionPath) {
  119. return { success: false, error: '缺少区域截图路径' };
  120. }
  121. // 如果 screenshotPath 为 '__AUTO_SCREENSHOT__' 或 null,自动从设备获取截图
  122. let absoluteScreenshotPath = screenshotPath;
  123. if (!screenshotPath || screenshotPath === '__AUTO_SCREENSHOT__' || screenshotPath === null) {
  124. if (!device) {
  125. return { success: false, error: '缺少完整截图路径,且无法自动获取设备截图(缺少设备ID)' };
  126. }
  127. // 自动获取设备截图
  128. const resolutionResult = await getDeviceResolution(device);
  129. if (!resolutionResult.success) {
  130. return { success: false, error: '获取设备分辨率失败' };
  131. }
  132. const screenshotResult = await captureScreenshot(device, { format: 'png' });
  133. if (!screenshotResult.success || !screenshotResult.data) {
  134. return { success: false, error: '自动获取设备截图失败' };
  135. }
  136. // 保存截图到临时文件
  137. const tempDir = join(__dirname, '..');
  138. const tempScreenshotPath = join(tempDir, 'temp_screenshot.png');
  139. const screenshotBuffer = Buffer.from(screenshotResult.data, 'base64');
  140. await writeFile(tempScreenshotPath, screenshotBuffer);
  141. absoluteScreenshotPath = tempScreenshotPath;
  142. } else {
  143. // 将相对路径转换为绝对路径
  144. if (!isAbsolute(screenshotPath)) {
  145. absoluteScreenshotPath = join(__dirname, '..', screenshotPath);
  146. }
  147. }
  148. let absoluteRegionPath = regionPath;
  149. if (!isAbsolute(regionPath)) {
  150. absoluteRegionPath = join(__dirname, '..', regionPath);
  151. }
  152. // 可选:如果提供了设备ID,获取设备分辨率用于缩放
  153. let width = null;
  154. let height = null;
  155. if (device) {
  156. const resolutionResult = await getDeviceResolution(device);
  157. if (resolutionResult.success) {
  158. width = resolutionResult.width;
  159. height = resolutionResult.height;
  160. }
  161. }
  162. // 调用图像匹配函数
  163. const matchResult = await matchImage(absoluteScreenshotPath, absoluteRegionPath, width, height);
  164. if (!matchResult.success) {
  165. return { success: false, error: matchResult.error || '图像匹配失败' };
  166. }
  167. // 获取匹配结果
  168. const { x, y, width: w, height: h } = matchResult;
  169. // 计算四个顶点坐标
  170. const corners = {
  171. topLeft: { x, y },
  172. topRight: { x: x + w, y },
  173. bottomLeft: { x, y: y + h },
  174. bottomRight: { x: x + w, y: y + h }
  175. };
  176. // 如果提供了保存路径,裁剪并保存区域图片
  177. // 这个功能会在 executeImageRegionLocation 中调用时传入 savePath
  178. // 但为了保持API简洁,我们在这里不处理,而是在 executeImageRegionLocation 中处理
  179. return {
  180. success: true,
  181. x,
  182. y,
  183. width: w,
  184. height: h,
  185. corners: corners,
  186. similarity: matchResult.similarity,
  187. screenshotPath: absoluteScreenshotPath // 返回截图路径,用于后续裁剪
  188. };
  189. } catch (error) {
  190. // 图像区域定位失败
  191. return { success: false, error: error.message };
  192. }
  193. }
  194. /**
  195. * 裁剪并保存图片区域
  196. * @param {string} imagePath - 原图片路径
  197. * @param {number} x - 裁剪区域左上角x坐标
  198. * @param {number} y - 裁剪区域左上角y坐标
  199. * @param {number} width - 裁剪区域宽度
  200. * @param {number} height - 裁剪区域高度
  201. * @param {string} savePath - 保存路径
  202. * @returns {Promise<{success: boolean, error?: string}>}
  203. */
  204. export async function cropAndSaveImage(imagePath, x, y, width, height, savePath) {
  205. try {
  206. // 处理相对路径,转换为绝对路径
  207. let absoluteImagePath = imagePath;
  208. if (!isAbsolute(imagePath)) {
  209. // 如果是相对路径,相对于项目根目录
  210. if (imagePath.startsWith('static/processing/')) {
  211. absoluteImagePath = join(__dirname, '..', imagePath);
  212. } else {
  213. absoluteImagePath = join(__dirname, '..', imagePath);
  214. }
  215. }
  216. let absoluteSavePath = savePath;
  217. if (!isAbsolute(savePath)) {
  218. // 如果是相对路径,需要判断是相对于工作流目录还是项目根目录
  219. // 优先检查 savePath 是否已经包含完整路径(以 static/processing/ 开头)
  220. if (savePath.startsWith('static/processing/')) {
  221. // savePath 已经是完整路径,如 "static/processing/微信聊天自动发送工作流/history/chat-area-cropped.png"
  222. // 直接拼接项目根目录即可,不需要再次提取工作流目录
  223. absoluteSavePath = join(__dirname, '..', savePath);
  224. } else if (imagePath.includes('static/processing/')) {
  225. // savePath 是相对于工作流目录的,如 "history/chat-area-cropped.png"
  226. // 从 imagePath 提取工作流目录路径
  227. const workflowMatch = imagePath.match(/static\/processing\/[^\/]+/);
  228. if (workflowMatch) {
  229. absoluteSavePath = join(__dirname, '..', workflowMatch[0], savePath);
  230. } else {
  231. // 如果无法匹配,尝试从 imagePath 提取工作流目录(去掉 /resources/ScreenShot.jpg)
  232. const workflowDir = imagePath.split('/resources/')[0];
  233. absoluteSavePath = join(__dirname, '..', workflowDir, savePath);
  234. }
  235. } else {
  236. // 默认相对于项目根目录
  237. absoluteSavePath = join(__dirname, '..', savePath);
  238. }
  239. }
  240. // 构建Python脚本代码
  241. // 使用 pathlib.Path 来处理路径,确保中文路径正确
  242. // 将 Windows 路径转换为 Python 可以处理的格式
  243. const normalizedImagePath = absoluteImagePath.replace(/\\/g, '/');
  244. const normalizedSavePath = absoluteSavePath.replace(/\\/g, '/');
  245. // 转义路径中的特殊字符,确保 Python 字符串正确
  246. const escapedImagePath = normalizedImagePath.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
  247. const escapedSavePath = normalizedSavePath.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
  248. const pythonCode = `
  249. # -*- coding: utf-8 -*-
  250. import sys
  251. import cv2
  252. import os
  253. from pathlib import Path
  254. # 立即输出,确认脚本开始执行
  255. print("Python脚本开始执行", file=sys.stderr)
  256. sys.stderr.flush()
  257. # 使用 Path 对象来处理路径,确保中文路径正确
  258. try:
  259. image_path_obj = Path(r"${escapedImagePath}")
  260. save_path_obj = Path(r"${escapedSavePath}")
  261. print(f"Path对象创建成功", file=sys.stderr)
  262. sys.stderr.flush()
  263. except Exception as e:
  264. print(f"创建Path对象失败: {e}", file=sys.stderr)
  265. sys.stderr.flush()
  266. sys.exit(1)
  267. # 转换为字符串(Path 会自动处理编码)
  268. image_path = str(image_path_obj)
  269. save_path = str(save_path_obj)
  270. x = ${x}
  271. y = ${y}
  272. w = ${width}
  273. h = ${height}
  274. print(f"图片路径: {image_path}", file=sys.stderr)
  275. sys.stderr.flush()
  276. print(f"保存路径: {save_path}", file=sys.stderr)
  277. sys.stderr.flush()
  278. print(f"裁剪参数: x={x}, y={y}, w={w}, h={h}", file=sys.stderr)
  279. sys.stderr.flush()
  280. try:
  281. # 检查图片文件是否存在
  282. print(f"检查图片文件: {image_path_obj}", file=sys.stderr)
  283. sys.stderr.flush()
  284. print(f"图片文件是否存在: {image_path_obj.exists()}", file=sys.stderr)
  285. sys.stderr.flush()
  286. if not image_path_obj.exists():
  287. print(f"错误: 图片文件不存在: {image_path}", file=sys.stderr)
  288. sys.stderr.flush()
  289. # 检查父目录是否存在
  290. parent = image_path_obj.parent
  291. print(f"父目录: {parent}", file=sys.stderr)
  292. sys.stderr.flush()
  293. print(f"父目录是否存在: {parent.exists()}", file=sys.stderr)
  294. sys.stderr.flush()
  295. if parent.exists():
  296. try:
  297. files = list(parent.iterdir())
  298. print(f"父目录中的文件: {[str(f.name) for f in files]}", file=sys.stderr)
  299. sys.stderr.flush()
  300. except Exception as e:
  301. print(f"列出父目录内容失败: {e}", file=sys.stderr)
  302. sys.stderr.flush()
  303. sys.exit(1)
  304. # 读取图片
  305. print(f"开始读取图片: {image_path}", file=sys.stderr)
  306. img = cv2.imread(image_path)
  307. if img is None:
  308. print(f"错误: 无法读取图片 {image_path}", file=sys.stderr)
  309. print(f"cv2.imread 返回 None,可能是文件格式不支持或文件损坏", file=sys.stderr)
  310. sys.exit(1)
  311. print(f"图片读取成功,尺寸: {img.shape}", file=sys.stderr)
  312. # 验证坐标是否在图片范围内
  313. img_height, img_width = img.shape[:2]
  314. if x < 0 or y < 0 or x + w > img_width or y + h > img_height:
  315. print(f"错误: 裁剪区域超出图片范围。图片尺寸: {img_width}x{img_height}, 裁剪区域: x={x}, y={y}, w={w}, h={h}", file=sys.stderr)
  316. sys.exit(1)
  317. # 裁剪区域
  318. cropped = img[y:y+h, x:x+w]
  319. print(f"裁剪成功,裁剪后尺寸: {cropped.shape}", file=sys.stderr)
  320. # 确保保存目录存在
  321. save_dir = save_path_obj.parent
  322. save_dir.mkdir(parents=True, exist_ok=True)
  323. print(f"保存目录已创建/存在: {save_dir}", file=sys.stderr)
  324. # 保存裁剪后的图片
  325. success = cv2.imwrite(str(save_path), cropped)
  326. print(f"cv2.imwrite 返回值: {success}", file=sys.stderr)
  327. if not success:
  328. print(f"错误: cv2.imwrite 返回 False,保存失败", file=sys.stderr)
  329. print(f"尝试保存到: {save_path}", file=sys.stderr)
  330. print(f"保存路径类型: {type(save_path)}", file=sys.stderr)
  331. sys.exit(1)
  332. # 验证文件是否真的保存了
  333. print(f"检查文件是否存在: {save_path_obj}", file=sys.stderr)
  334. print(f"文件是否存在: {save_path_obj.exists()}", file=sys.stderr)
  335. if save_path_obj.exists():
  336. file_size = save_path_obj.stat().st_size
  337. print(f"成功保存区域截图到: {save_path} (文件大小: {file_size} 字节)", file=sys.stderr)
  338. else:
  339. print(f"错误: 文件保存后不存在: {save_path}", file=sys.stderr)
  340. print(f"保存目录是否存在: {save_dir.exists()}", file=sys.stderr)
  341. print(f"保存目录路径: {save_dir}", file=sys.stderr)
  342. # 尝试列出目录内容
  343. try:
  344. if save_dir.exists():
  345. files = list(save_dir.iterdir())
  346. print(f"目录中的文件: {[str(f) for f in files]}", file=sys.stderr)
  347. except Exception as e:
  348. print(f"列出目录内容失败: {e}", file=sys.stderr)
  349. sys.exit(1)
  350. except Exception as e:
  351. print(f"裁剪图片失败: {e}", file=sys.stderr)
  352. import traceback
  353. traceback.print_exc(file=sys.stderr)
  354. sys.exit(1)
  355. `;
  356. // 获取Python可执行文件路径
  357. const pythonExePath = join(__dirname, '..', 'py', 'venv', 'Scripts', 'python.exe');
  358. // 执行Python脚本
  359. try {
  360. const { stdout, stderr } = await execAsync(`"${pythonExePath}" -c "${pythonCode.replace(/"/g, '\\"')}"`, {
  361. maxBuffer: 10 * 1024 * 1024,
  362. encoding: 'utf8',
  363. cwd: join(__dirname, '..')
  364. });
  365. // 检查是否有错误(Python脚本通过 sys.exit(1) 退出时,execAsync 会抛出错误)
  366. // 但如果脚本正常退出,stderr 可能包含我们的调试信息
  367. if (stderr && !stderr.includes('成功') && !stderr.includes('图片路径') && !stderr.includes('保存路径') && !stderr.includes('裁剪参数') && !stderr.includes('图片读取成功') && !stderr.includes('裁剪成功') && !stderr.includes('保存目录')) {
  368. // 如果 stderr 包含错误信息(不是我们的调试信息),可能是错误
  369. if (!stderr.includes('成功保存')) {
  370. // 不抛出错误,让后续验证文件是否存在
  371. }
  372. }
  373. // 验证文件是否真的保存了
  374. // 检查截图文件是否存在
  375. const screenshotExists = fs.existsSync(absoluteImagePath);
  376. if (fs.existsSync(absoluteSavePath)) {
  377. return { success: true };
  378. } else {
  379. // 检查父目录是否存在
  380. const parentDir = dirname(absoluteSavePath);
  381. const parentExists = fs.existsSync(parentDir);
  382. // 构建详细的错误信息
  383. let errorMsg = `文件保存失败,文件不存在: ${absoluteSavePath}`;
  384. if (!screenshotExists) {
  385. errorMsg += `\n截图文件不存在: ${absoluteImagePath}`;
  386. }
  387. if (!parentExists) {
  388. errorMsg += `\n保存目录不存在: ${parentDir}`;
  389. }
  390. if (stderr) {
  391. errorMsg += `\nPython stderr: ${stderr}`;
  392. } else {
  393. errorMsg += `\nPython stderr: (空) - 可能Python脚本未执行或执行失败`;
  394. }
  395. if (stdout) {
  396. errorMsg += `\nPython stdout: ${stdout}`;
  397. }
  398. return { success: false, error: errorMsg };
  399. }
  400. } catch (execError) {
  401. // execAsync 执行失败(Python脚本返回非0退出码)
  402. return { success: false, error: execError.stderr || execError.message || 'Python脚本执行失败' };
  403. }
  404. } catch (error) {
  405. return { success: false, error: error.message };
  406. }
  407. }
  408. /**
  409. * 执行文字识别:截图、调用 Python 脚本、返回坐标
  410. * @param {string} ipPort - 设备 ID/IP:Port
  411. * @param {string} targetText - 目标文字
  412. * @returns {Promise<{success: boolean, error?: string, coordinate?: Object, clickPosition?: Object}>}
  413. */
  414. export async function findTextAndGetCoordinate(ipPort, targetText) {
  415. try {
  416. if (!ipPort) {
  417. return { success: false, error: '缺少设备 ID' };
  418. }
  419. if (!targetText) {
  420. return { success: false, error: '缺少目标文字' };
  421. }
  422. // 1. 获取设备分辨率
  423. const resolutionResult = await getDeviceResolution(ipPort);
  424. if (!resolutionResult.success) {
  425. return { success: false, error: '获取设备分辨率失败' };
  426. }
  427. const { width, height } = resolutionResult;
  428. // 2. 获取屏幕截图
  429. const screenshotResult = await captureScreenshot(ipPort, { format: 'png' });
  430. if (!screenshotResult.success || !screenshotResult.data) {
  431. return { success: false, error: '获取屏幕截图失败' };
  432. }
  433. // 3. 保存截图到临时文件
  434. const tempDir = join(__dirname, '..');
  435. const screenshotPath = join(tempDir, 'temp_screenshot.png');
  436. const screenshotBuffer = Buffer.from(screenshotResult.data, 'base64');
  437. await writeFile(screenshotPath, screenshotBuffer);
  438. // 4. 调用 JS 函数进行文字识别
  439. const textResult = await findTextLocation(screenshotPath, targetText, width, height);
  440. if (!textResult.success || !textResult.found) {
  441. return { success: false, error: textResult.error || `未找到文字: ${targetText}` };
  442. }
  443. // 5. 返回识别结果
  444. const { x, y, width: w, height: h } = textResult;
  445. // 计算点击位置(中心点)
  446. const clickX = Math.round(x + w / 2);
  447. const clickY = Math.round(y + h / 2);
  448. return {
  449. success: true,
  450. coordinate: { x, y, width: w, height: h },
  451. clickPosition: { x: clickX, y: clickY }
  452. };
  453. } catch (error) {
  454. // 如果是超时错误,提供更友好的提示
  455. if (error.message && error.message.includes('timeout')) {
  456. return { success: false, error: '文字识别超时,请检查网络连接或稍后重试' };
  457. }
  458. return { success: false, error: error.message };
  459. }
  460. }
  461. /**
  462. * 全屏OCR识别(通用功能)
  463. * @param {string} ipPort - 设备 ID/IP:Port
  464. * @param {string} folderPath - 工作流文件夹路径(可选,用于保存临时文件)
  465. * @returns {Promise<{success: boolean, error?: string, text?: string}>}
  466. */
  467. export async function ocrFullScreen(ipPort, folderPath = null) {
  468. try {
  469. if (!ipPort) {
  470. return { success: false, error: '缺少设备 ID' };
  471. }
  472. // 1. 获取设备分辨率
  473. const resolutionResult = await getDeviceResolution(ipPort);
  474. if (!resolutionResult.success) {
  475. return { success: false, error: '获取设备分辨率失败' };
  476. }
  477. const { width, height } = resolutionResult;
  478. // 2. 获取屏幕截图
  479. const screenshotResult = await captureScreenshot(ipPort, { format: 'png' });
  480. if (!screenshotResult.success || !screenshotResult.data) {
  481. return { success: false, error: '获取屏幕截图失败' };
  482. }
  483. // 3. 保存截图到临时文件(如果提供了工作流文件夹,保存到 tmp/时间戳 目录)
  484. let screenshotPath;
  485. let tmpDir = null; // 用于跟踪需要删除的临时目录
  486. if (folderPath) {
  487. // 确保 folderPath 是绝对路径
  488. let absoluteFolderPath = folderPath;
  489. if (!isAbsolute(folderPath)) {
  490. // 如果已经是 static/processing/xxx 格式,去掉开头的 static/processing 再拼接
  491. if (folderPath.startsWith('static/processing/')) {
  492. const folderName = folderPath.replace('static/processing/', '');
  493. absoluteFolderPath = join(__dirname, '..', 'static', 'processing', folderName);
  494. } else if (folderPath.startsWith('static\\processing\\')) {
  495. const folderName = folderPath.replace('static\\processing\\', '');
  496. absoluteFolderPath = join(__dirname, '..', 'static', 'processing', folderName);
  497. } else {
  498. // 如果只是文件夹名,需要加上 static/processing
  499. absoluteFolderPath = join(__dirname, '..', 'static', 'processing', folderPath);
  500. }
  501. }
  502. const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19).replace('T', '_');
  503. tmpDir = join(absoluteFolderPath, 'tmp', timestamp);
  504. await mkdir(tmpDir, { recursive: true });
  505. screenshotPath = join(tmpDir, 'screenshot_ocr.png');
  506. } else {
  507. const tempDir = join(__dirname, '..');
  508. screenshotPath = join(tempDir, 'temp_screenshot_ocr.png');
  509. }
  510. const screenshotBuffer = Buffer.from(screenshotResult.data, 'base64');
  511. await writeFile(screenshotPath, screenshotBuffer);
  512. try {
  513. // 4. 调用 JS 实现进行全屏OCR识别
  514. const normalizedScreenshotPath = screenshotPath.replace(/\\/g, '/');
  515. const result = await ocrFullScreenFromFunc(normalizedScreenshotPath, width, height);
  516. if (result.success) {
  517. return {
  518. success: true,
  519. text: result.text || ''
  520. };
  521. } else {
  522. return { success: false, error: result.error || 'OCR识别失败' };
  523. }
  524. } finally {
  525. // 5. 使用完后,先判断 tmp 目录总大小是否超过 20MB,再决定是否删除临时目录
  526. if (tmpDir) {
  527. try {
  528. // 获取 tmp 目录的父目录(工作流目录下的 tmp 文件夹)
  529. const tmpParentDir = dirname(tmpDir);
  530. // 检查 tmp 目录下所有文件和子目录的总大小
  531. let totalSize = 0;
  532. const dirsToDelete = [];
  533. try {
  534. const entries = await readdir(tmpParentDir, { withFileTypes: true });
  535. for (const entry of entries) {
  536. const entryPath = join(tmpParentDir, entry.name);
  537. try {
  538. if (entry.isFile()) {
  539. const stats = await stat(entryPath);
  540. totalSize += stats.size;
  541. } else if (entry.isDirectory()) {
  542. // 递归计算子目录大小
  543. const dirSize = await calculateDirSize(entryPath);
  544. totalSize += dirSize;
  545. const dirStats = await stat(entryPath);
  546. dirsToDelete.push({ path: entryPath, size: dirSize, mtime: dirStats.mtime });
  547. }
  548. } catch (e) {
  549. // 忽略无法访问的文件
  550. }
  551. }
  552. } catch (e) {
  553. // 如果无法读取目录,直接删除临时目录
  554. await rm(tmpDir, { recursive: true, force: true });
  555. return;
  556. }
  557. const maxSize = 20 * 1024 * 1024; // 20MB
  558. // 如果总大小超过 20MB,删除时间最早的目录
  559. if (totalSize > maxSize) {
  560. // 按修改时间排序(最早的在前)
  561. dirsToDelete.sort((a, b) => a.mtime - b.mtime);
  562. // 删除最早的目录直到总大小小于 20MB
  563. for (const dirInfo of dirsToDelete) {
  564. if (totalSize <= maxSize) {
  565. break;
  566. }
  567. try {
  568. await rm(dirInfo.path, { recursive: true, force: true });
  569. totalSize -= dirInfo.size;
  570. } catch (e) {
  571. // 忽略删除失败
  572. }
  573. }
  574. }
  575. // 如果临时目录仍然存在且总大小未超过限制,也删除它(保持原有行为)
  576. try {
  577. const tmpDirExists = await stat(tmpDir).then(() => true).catch(() => false);
  578. if (tmpDirExists) {
  579. await rm(tmpDir, { recursive: true, force: true });
  580. }
  581. } catch (rmError) {
  582. // 忽略删除失败
  583. }
  584. } catch (error) {
  585. // 清理失败不影响主流程
  586. }
  587. }
  588. }
  589. } catch (error) {
  590. if (error.message && error.message.includes('timeout')) {
  591. return { success: false, error: 'OCR识别超时,请检查网络连接或稍后重试' };
  592. }
  593. return { success: false, error: error.message };
  594. }
  595. }
  596. /**
  597. * OCR识别最后一条消息(兼容旧API)
  598. * @param {string} ipPort - 设备 ID/IP:Port
  599. * @param {string} method - 识别方法 ('full-screen' | 'by-avatar')
  600. * @param {string} avatarPath - 头像路径(by-avatar 时使用)
  601. * @param {string} area - 区域(未使用,保留兼容性)
  602. * @param {string} folderPath - 工作流文件夹路径(可选)
  603. * @returns {Promise<{success: boolean, error?: string, text?: string, position?: Object}>}
  604. */
  605. export async function ocrLastMessage(ipPort, method, avatarPath, area, folderPath = null) {
  606. try {
  607. if (!ipPort) {
  608. return { success: false, error: '缺少设备 ID' };
  609. }
  610. // 1. 获取设备分辨率
  611. const resolutionResult = await getDeviceResolution(ipPort);
  612. if (!resolutionResult.success) {
  613. return { success: false, error: '获取设备分辨率失败' };
  614. }
  615. const { width, height } = resolutionResult;
  616. // 2. 获取屏幕截图
  617. const screenshotResult = await captureScreenshot(ipPort, { format: 'png' });
  618. if (!screenshotResult.success || !screenshotResult.data) {
  619. return { success: false, error: '获取屏幕截图失败' };
  620. }
  621. // 3. 保存截图到临时文件(如果提供了工作流文件夹,保存到 tmp/时间戳 目录)
  622. let screenshotPath;
  623. let tmpDir = null; // 用于跟踪需要删除的临时目录
  624. if (folderPath) {
  625. // 确保 folderPath 是绝对路径
  626. let absoluteFolderPath = folderPath;
  627. if (!isAbsolute(folderPath)) {
  628. // 如果已经是 static/processing/xxx 格式,去掉开头的 static/processing 再拼接
  629. if (folderPath.startsWith('static/processing/')) {
  630. const folderName = folderPath.replace('static/processing/', '');
  631. absoluteFolderPath = join(__dirname, '..', 'static', 'processing', folderName);
  632. } else if (folderPath.startsWith('static\\processing\\')) {
  633. const folderName = folderPath.replace('static\\processing\\', '');
  634. absoluteFolderPath = join(__dirname, '..', 'static', 'processing', folderName);
  635. } else {
  636. // 如果只是文件夹名,需要加上 static/processing
  637. absoluteFolderPath = join(__dirname, '..', 'static', 'processing', folderPath);
  638. }
  639. }
  640. const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19).replace('T', '_');
  641. tmpDir = join(absoluteFolderPath, 'tmp', timestamp);
  642. await mkdir(tmpDir, { recursive: true });
  643. screenshotPath = join(tmpDir, 'screenshot_ocr.png');
  644. } else {
  645. const tempDir = join(__dirname, '..');
  646. screenshotPath = join(tempDir, 'temp_screenshot_ocr.png');
  647. }
  648. const screenshotBuffer = Buffer.from(screenshotResult.data, 'base64');
  649. await writeFile(screenshotPath, screenshotBuffer);
  650. try {
  651. // 4. 调用 JS 实现进行OCR识别
  652. const normalizedScreenshotPath = screenshotPath.replace(/\\/g, '/');
  653. let result;
  654. if (method === 'full-screen') {
  655. // 全屏OCR识别
  656. result = await ocrFullScreenFromFunc(normalizedScreenshotPath, width, height);
  657. } else if (method === 'by-avatar' && avatarPath) {
  658. // 通过头像定位最后一条消息
  659. let friendAvatarArg = null;
  660. let myAvatarArg = null;
  661. if (isAbsolute(avatarPath)) {
  662. friendAvatarArg = avatarPath;
  663. myAvatarArg = avatarPath;
  664. } else {
  665. const folderName = avatarPath.split(/[/\\]/)[0];
  666. const avatarName = avatarPath.split(/[/\\]/).slice(1).join('/');
  667. friendAvatarArg = join(__dirname, '..', 'static', 'processing', folderName, avatarName);
  668. myAvatarArg = friendAvatarArg;
  669. }
  670. const normalizedFriendAvatar = friendAvatarArg.replace(/\\/g, '/');
  671. const normalizedMyAvatar = myAvatarArg.replace(/\\/g, '/');
  672. result = await getLastMessageFromFunc(normalizedScreenshotPath, normalizedFriendAvatar, normalizedMyAvatar, width, height);
  673. } else {
  674. // 默认使用全屏OCR
  675. result = await ocrFullScreenFromFunc(normalizedScreenshotPath, width, height);
  676. }
  677. if (result.success) {
  678. // 返回兼容旧API的格式
  679. return {
  680. success: true,
  681. text: result.text || '',
  682. position: result.position || null
  683. };
  684. } else {
  685. return { success: false, error: result.error || 'OCR识别失败' };
  686. }
  687. } finally {
  688. // 5. 使用完后,先判断 tmp 目录总大小是否超过 20MB,再决定是否删除临时目录
  689. if (tmpDir) {
  690. try {
  691. // 获取 tmp 目录的父目录(工作流目录下的 tmp 文件夹)
  692. const tmpParentDir = dirname(tmpDir);
  693. // 检查 tmp 目录下所有文件和子目录的总大小
  694. let totalSize = 0;
  695. const dirsToDelete = [];
  696. try {
  697. const entries = await readdir(tmpParentDir, { withFileTypes: true });
  698. for (const entry of entries) {
  699. const entryPath = join(tmpParentDir, entry.name);
  700. try {
  701. if (entry.isFile()) {
  702. const stats = await stat(entryPath);
  703. totalSize += stats.size;
  704. } else if (entry.isDirectory()) {
  705. // 递归计算子目录大小
  706. const dirSize = await calculateDirSize(entryPath);
  707. totalSize += dirSize;
  708. const dirStats = await stat(entryPath);
  709. dirsToDelete.push({ path: entryPath, size: dirSize, mtime: dirStats.mtime });
  710. }
  711. } catch (e) {
  712. // 忽略无法访问的文件
  713. }
  714. }
  715. } catch (e) {
  716. // 如果无法读取目录,直接删除临时目录
  717. await rm(tmpDir, { recursive: true, force: true });
  718. return;
  719. }
  720. const maxSize = 20 * 1024 * 1024; // 20MB
  721. // 如果总大小超过 20MB,删除时间最早的目录
  722. if (totalSize > maxSize) {
  723. // 按修改时间排序(最早的在前)
  724. dirsToDelete.sort((a, b) => a.mtime - b.mtime);
  725. // 删除最早的目录直到总大小小于 20MB
  726. for (const dirInfo of dirsToDelete) {
  727. if (totalSize <= maxSize) {
  728. break;
  729. }
  730. try {
  731. await rm(dirInfo.path, { recursive: true, force: true });
  732. totalSize -= dirInfo.size;
  733. } catch (e) {
  734. // 忽略删除失败
  735. }
  736. }
  737. }
  738. // 如果临时目录仍然存在且总大小未超过限制,也删除它(保持原有行为)
  739. try {
  740. const tmpDirExists = await stat(tmpDir).then(() => true).catch(() => false);
  741. if (tmpDirExists) {
  742. await rm(tmpDir, { recursive: true, force: true });
  743. }
  744. } catch (rmError) {
  745. // 忽略删除失败
  746. }
  747. } catch (error) {
  748. // 清理失败不影响主流程
  749. }
  750. }
  751. }
  752. } catch (error) {
  753. if (error.message && error.message.includes('timeout')) {
  754. return { success: false, error: 'OCR识别超时,请检查网络连接或稍后重试' };
  755. }
  756. return { success: false, error: error.message };
  757. }
  758. }
  759. /**
  760. * 注册 IPC 处理器(Python 执行相关)
  761. */
  762. export function registerIpcHandlers() {
  763. // 图像匹配
  764. ipcMain.handle('match-image-and-get-coordinate', async (event, ipPort, templateImagePath) => {
  765. return await matchImageAndGetCoordinate(ipPort, templateImagePath);
  766. });
  767. // 图像区域定位
  768. ipcMain.handle('match-image-region-location', async (event, screenshotPath, regionPath, device) => {
  769. return await matchImageRegionLocation(screenshotPath, regionPath, device);
  770. });
  771. ipcMain.handle('crop-and-save-image', async (event, imagePath, x, y, width, height, savePath) => {
  772. const result = await cropAndSaveImage(imagePath, x, y, width, height, savePath);
  773. return result;
  774. });
  775. // 文字识别
  776. ipcMain.handle('find-text-and-get-coordinate', async (event, ipPort, targetText) => {
  777. return await findTextAndGetCoordinate(ipPort, targetText);
  778. });
  779. // OCR识别最后一条消息(兼容旧API)
  780. ipcMain.handle('ocr-last-message', async (event, ipPort, method, avatarPath, area, folderPath) => {
  781. return await ocrLastMessage(ipPort, method, avatarPath, area, folderPath);
  782. });
  783. }