|
|
@@ -1,11 +1,14 @@
|
|
|
import { ipcMain } from 'electron';
|
|
|
-import { readdir, writeFile, readFile } from 'fs/promises';
|
|
|
+import { readdir, writeFile, readFile, mkdir, rm, stat } from 'fs/promises';
|
|
|
import { join, dirname, isAbsolute } from 'path';
|
|
|
import { fileURLToPath } from 'url';
|
|
|
import { exec } from 'child_process';
|
|
|
import { promisify } from 'util';
|
|
|
import { captureScreenshot } from './adb/screenshot.js';
|
|
|
import { getDeviceResolution } from './adb/device-info.js';
|
|
|
+import { matchImage } from './func/img-reg.js';
|
|
|
+import { findTextLocation } from './func/string-reg-location.js';
|
|
|
+import { extractChatHistory as extractChatHistoryFromFunc, getLastMessage as getLastMessageFromFunc, ocrFullScreen as ocrFullScreenFromFunc } from './func/extract-chat-history.js';
|
|
|
|
|
|
const execAsync = promisify(exec);
|
|
|
const __filename = fileURLToPath(import.meta.url);
|
|
|
@@ -20,11 +23,17 @@ export async function getStaticFolders() {
|
|
|
const folders = [];
|
|
|
for (const entry of entries) {
|
|
|
if (entry.isDirectory()) {
|
|
|
- folders.push(entry.name);
|
|
|
+ const folderPath = join(staticPath, entry.name);
|
|
|
+ const stats = await stat(folderPath);
|
|
|
+ folders.push({
|
|
|
+ name: entry.name,
|
|
|
+ createdAt: stats.birthtime || stats.mtime, // 使用创建时间，如果没有则使用修改时间
|
|
|
+ });
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- return folders.sort(); // 按名称排序
|
|
|
+ // 按创建时间排序，最新的在前
|
|
|
+ return folders.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
|
|
|
} catch (error) {
|
|
|
console.error('Failed to read static/processing folders:', error);
|
|
|
return [];
|
|
|
@@ -66,86 +75,77 @@ export async function matchImageAndGetCoordinate(ipPort, templateImagePath) {
|
|
|
const screenshotBuffer = Buffer.from(screenshotResult.data, 'base64');
|
|
|
await writeFile(screenshotPath, screenshotBuffer);
|
|
|
|
|
|
- // 4. 调用 Python 脚本进行图像匹配
|
|
|
- // 使用虚拟环境中的 Python 解释器
|
|
|
- const pythonExePath = join(__dirname, '..', 'py', 'venv', 'Scripts', 'python.exe');
|
|
|
- const pythonScriptPath = join(__dirname, '..', 'py', 'img-reg.py');
|
|
|
+ // 4. 调用 JS 函数进行图像匹配
|
|
|
+ const matchResult = await matchImage(screenshotPath, absoluteTemplatePath, width, height);
|
|
|
|
|
|
- // 确保路径使用正斜杠，避免 Windows 路径问题
|
|
|
- const normalizedScreenshotPath = screenshotPath.replace(/\\/g, '/');
|
|
|
- const normalizedTemplatePath = absoluteTemplatePath.replace(/\\/g, '/');
|
|
|
-
|
|
|
- // 使用 -u 参数确保输出不被缓冲，并设置 UTF-8 编码
|
|
|
- const command = `"${pythonExePath}" -u "${pythonScriptPath}" "${normalizedScreenshotPath}" "${normalizedTemplatePath}" ${width} ${height}`;
|
|
|
-
|
|
|
- const { stdout, stderr } = await execAsync(command, {
|
|
|
- timeout: 10000,
|
|
|
- maxBuffer: 10 * 1024 * 1024,
|
|
|
- cwd: join(__dirname, '..'),
|
|
|
- encoding: 'utf8',
|
|
|
- env: { ...process.env, PYTHONIOENCODING: 'utf-8', PYTHONUTF8: '1' }
|
|
|
- });
|
|
|
-
|
|
|
- // 5. 解析 Python 脚本输出
|
|
|
- // 输出格式可能是:
|
|
|
- // - "找到匹配！坐标: x=100, y=200, 宽度=50, 高度=50"
|
|
|
- // - "JSON格式: {"x": 100, "y": 200, "width": 50, "height": 50}"
|
|
|
- // - "未找到匹配"
|
|
|
- // - "错误: xxx"
|
|
|
-
|
|
|
- // 检查是否有错误或未找到匹配
|
|
|
- if (stdout.includes('未找到匹配') || stdout.toLowerCase().includes('错误') || stderr) {
|
|
|
- const errorMsg = stdout.includes('未找到匹配')
|
|
|
- ? '未找到匹配的图像'
|
|
|
- : (stderr || stdout.match(/错误[：:]\s*(.+)/)?.[1] || '图像匹配失败');
|
|
|
- return { success: false, error: errorMsg };
|
|
|
+ if (!matchResult.success) {
|
|
|
+ return { success: false, error: matchResult.error || '图像匹配失败' };
|
|
|
}
|
|
|
|
|
|
- // 尝试从输出中提取 JSON
|
|
|
- // 先尝试匹配 JSON 格式的行
|
|
|
- let jsonMatch = stdout.match(/JSON格式:\s*(\{[^}]+\})/);
|
|
|
- if (!jsonMatch) {
|
|
|
- // 如果没有找到 JSON 格式，尝试直接匹配 JSON 对象
|
|
|
- jsonMatch = stdout.match(/\{\s*"x"\s*:\s*\d+\s*,\s*"y"\s*:\s*\d+\s*,\s*"width"\s*:\s*\d+\s*,\s*"height"\s*:\s*\d+\s*\}/);
|
|
|
- }
|
|
|
-
|
|
|
- if (!jsonMatch) {
|
|
|
- // 如果还是没有找到，尝试从文本中提取坐标
|
|
|
- const coordMatch = stdout.match(/坐标:\s*x=(\d+),\s*y=(\d+),\s*宽度=(\d+),\s*高度=(\d+)/);
|
|
|
- if (coordMatch) {
|
|
|
- const x = parseInt(coordMatch[1], 10);
|
|
|
- const y = parseInt(coordMatch[2], 10);
|
|
|
- const w = parseInt(coordMatch[3], 10);
|
|
|
- const h = parseInt(coordMatch[4], 10);
|
|
|
-
|
|
|
- // 计算点击位置（中心点）
|
|
|
- const clickX = Math.round(x + w / 2);
|
|
|
- const clickY = Math.round(y + h / 2);
|
|
|
-
|
|
|
- return {
|
|
|
- success: true,
|
|
|
- coordinate: { x, y, width: w, height: h },
|
|
|
- clickPosition: { x: clickX, y: clickY }
|
|
|
- };
|
|
|
- }
|
|
|
+ // 5. 返回匹配结果
|
|
|
+ if (matchResult.success && matchResult.x !== undefined) {
|
|
|
+ const { x, y, width: w, height: h } = matchResult;
|
|
|
|
|
|
- // 如果都找不到，返回错误并输出原始输出用于调试
|
|
|
- console.error('Python 脚本输出:', cleanStdout);
|
|
|
- console.error('Python 脚本错误输出:', cleanStderr);
|
|
|
- return { success: false, error: `无法解析 Python 脚本输出。输出: ${cleanStdout.substring(0, 200)}` };
|
|
|
+ // 计算点击位置（中心点）
|
|
|
+ const clickX = Math.round(x + w / 2);
|
|
|
+ const clickY = Math.round(y + h / 2);
|
|
|
+
|
|
|
+ return {
|
|
|
+ success: true,
|
|
|
+ coordinate: { x, y, width: w, height: h },
|
|
|
+ clickPosition: { x: clickX, y: clickY }
|
|
|
+ };
|
|
|
+ } else {
|
|
|
+ return {
|
|
|
+ success: false,
|
|
|
+ error: matchResult.error || '图像匹配失败'
|
|
|
+ };
|
|
|
}
|
|
|
+ } catch (error) {
|
|
|
+ console.error('图像匹配失败:', error);
|
|
|
+ return { success: false, error: error.message };
|
|
|
+ }
|
|
|
+}
|
|
|
|
|
|
- let coordinate;
|
|
|
- try {
|
|
|
- coordinate = JSON.parse(jsonMatch[1] || jsonMatch[0]);
|
|
|
- } catch (parseError) {
|
|
|
- console.error('JSON 解析失败:', parseError);
|
|
|
- console.error('原始输出:', cleanStdout);
|
|
|
- return { success: false, error: `JSON 解析失败: ${parseError.message}` };
|
|
|
+// 执行文字识别：截图、调用 Python 脚本、返回坐标
|
|
|
+export async function findTextAndGetCoordinate(ipPort, targetText) {
|
|
|
+ try {
|
|
|
+ if (!ipPort) {
|
|
|
+ return { success: false, error: '缺少设备 ID' };
|
|
|
}
|
|
|
+ if (!targetText) {
|
|
|
+ return { success: false, error: '缺少目标文字' };
|
|
|
+ }
|
|
|
+
|
|
|
+ // 1. 获取设备分辨率
|
|
|
+ const resolutionResult = await getDeviceResolution(ipPort);
|
|
|
+ if (!resolutionResult.success) {
|
|
|
+ return { success: false, error: '获取设备分辨率失败' };
|
|
|
+ }
|
|
|
+ const { width, height } = resolutionResult;
|
|
|
+
|
|
|
+ // 2. 获取屏幕截图
|
|
|
+ const screenshotResult = await captureScreenshot(ipPort, { format: 'png' });
|
|
|
+ if (!screenshotResult.success || !screenshotResult.data) {
|
|
|
+ return { success: false, error: '获取屏幕截图失败' };
|
|
|
+ }
|
|
|
+
|
|
|
+ // 3. 保存截图到临时文件
|
|
|
+ const tempDir = join(__dirname, '..');
|
|
|
+ const screenshotPath = join(tempDir, 'temp_screenshot.png');
|
|
|
+ const screenshotBuffer = Buffer.from(screenshotResult.data, 'base64');
|
|
|
+ await writeFile(screenshotPath, screenshotBuffer);
|
|
|
+
|
|
|
+ // 4. 调用 JS 函数进行文字识别
|
|
|
+ const textResult = await findTextLocation(screenshotPath, targetText, width, height);
|
|
|
|
|
|
- const { x, y, width: w, height: h } = coordinate;
|
|
|
+ if (!textResult.success || !textResult.found) {
|
|
|
+ return { success: false, error: textResult.error || `未找到文字: ${targetText}` };
|
|
|
+ }
|
|
|
|
|
|
+ // 5. 返回识别结果
|
|
|
+ const { x, y, width: w, height: h } = textResult;
|
|
|
+
|
|
|
// 计算点击位置（中心点）
|
|
|
const clickX = Math.round(x + w / 2);
|
|
|
const clickY = Math.round(y + h / 2);
|
|
|
@@ -156,20 +156,21 @@ export async function matchImageAndGetCoordinate(ipPort, templateImagePath) {
|
|
|
clickPosition: { x: clickX, y: clickY }
|
|
|
};
|
|
|
} catch (error) {
|
|
|
- console.error('图像匹配失败:', error);
|
|
|
+ console.error('文字识别失败:', error);
|
|
|
+ // 如果是超时错误，提供更友好的提示
|
|
|
+ if (error.message && error.message.includes('timeout')) {
|
|
|
+ return { success: false, error: '文字识别超时，请检查网络连接或稍后重试' };
|
|
|
+ }
|
|
|
return { success: false, error: error.message };
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-// 执行文字识别：截图、调用 Python 脚本、返回坐标
|
|
|
-export async function findTextAndGetCoordinate(ipPort, targetText) {
|
|
|
+// OCR识别最后一条消息（兼容旧API）
|
|
|
+export async function ocrLastMessage(ipPort, method, avatarPath, area, folderPath = null) {
|
|
|
try {
|
|
|
if (!ipPort) {
|
|
|
return { success: false, error: '缺少设备 ID' };
|
|
|
}
|
|
|
- if (!targetText) {
|
|
|
- return { success: false, error: '缺少目标文字' };
|
|
|
- }
|
|
|
|
|
|
// 1. 获取设备分辨率
|
|
|
const resolutionResult = await getDeviceResolution(ipPort);
|
|
|
@@ -184,104 +185,266 @@ export async function findTextAndGetCoordinate(ipPort, targetText) {
|
|
|
return { success: false, error: '获取屏幕截图失败' };
|
|
|
}
|
|
|
|
|
|
- // 3. 保存截图到临时文件
|
|
|
- const tempDir = join(__dirname, '..');
|
|
|
- const screenshotPath = join(tempDir, 'temp_screenshot.png');
|
|
|
+ // 3. 保存截图到临时文件（如果提供了工作流文件夹，保存到 tmp/时间戳 目录）
|
|
|
+ let screenshotPath;
|
|
|
+ if (folderPath) {
|
|
|
+ const { mkdir } = await import('fs/promises');
|
|
|
+ // 确保 folderPath 是绝对路径
|
|
|
+ let absoluteFolderPath = folderPath;
|
|
|
+ if (!isAbsolute(folderPath)) {
|
|
|
+ // 如果已经是 static/processing/xxx 格式，去掉开头的 static/processing 再拼接
|
|
|
+ if (folderPath.startsWith('static/processing/')) {
|
|
|
+ const folderName = folderPath.replace('static/processing/', '');
|
|
|
+ absoluteFolderPath = join(__dirname, '..', 'static', 'processing', folderName);
|
|
|
+ } else if (folderPath.startsWith('static\\processing\\')) {
|
|
|
+ const folderName = folderPath.replace('static\\processing\\', '');
|
|
|
+ absoluteFolderPath = join(__dirname, '..', 'static', 'processing', folderName);
|
|
|
+ } else {
|
|
|
+ // 如果只是文件夹名，需要加上 static/processing
|
|
|
+ absoluteFolderPath = join(__dirname, '..', 'static', 'processing', folderPath);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19).replace('T', '_');
|
|
|
+ const tmpDir = join(absoluteFolderPath, 'tmp', timestamp);
|
|
|
+ await mkdir(tmpDir, { recursive: true });
|
|
|
+ screenshotPath = join(tmpDir, 'screenshot_ocr.png');
|
|
|
+ } else {
|
|
|
+ const tempDir = join(__dirname, '..');
|
|
|
+ screenshotPath = join(tempDir, 'temp_screenshot_ocr.png');
|
|
|
+ }
|
|
|
const screenshotBuffer = Buffer.from(screenshotResult.data, 'base64');
|
|
|
await writeFile(screenshotPath, screenshotBuffer);
|
|
|
|
|
|
- // 4. 调用 Python 脚本进行文字识别
|
|
|
- // 使用虚拟环境中的 Python 解释器
|
|
|
- const pythonExePath = join(__dirname, '..', 'py', 'venv', 'Scripts', 'python.exe');
|
|
|
- const pythonScriptPath = join(__dirname, '..', 'py', 'string-reg-location.py');
|
|
|
- // 转义目标文字中的双引号，避免命令解析错误
|
|
|
- const escapedText = targetText.replace(/"/g, '\\"');
|
|
|
- const command = `"${pythonExePath}" "${pythonScriptPath}" "${screenshotPath}" "${escapedText}" ${width} ${height}`;
|
|
|
-
|
|
|
- // 设置环境变量，跳过模型源连接检查
|
|
|
- const env = {
|
|
|
- ...process.env,
|
|
|
- DISABLE_MODEL_SOURCE_CHECK: 'True'
|
|
|
- };
|
|
|
+ // 4. 调用 JS 实现进行OCR识别
|
|
|
+ const normalizedScreenshotPath = screenshotPath.replace(/\\/g, '/');
|
|
|
+ let result;
|
|
|
|
|
|
- const { stdout, stderr } = await execAsync(command, {
|
|
|
- timeout: 60000, // OCR 首次运行可能需要下载模型，设置60秒超时
|
|
|
- maxBuffer: 10 * 1024 * 1024,
|
|
|
- cwd: join(__dirname, '..'),
|
|
|
- env: env
|
|
|
- });
|
|
|
-
|
|
|
- // 5. 解析 Python 脚本输出
|
|
|
- // 检查是否有错误或未找到匹配
|
|
|
- // 忽略警告信息（DeprecationWarning 等）
|
|
|
- const cleanStdout = stdout.replace(/\[33m.*?\[0m/g, '').replace(/DeprecationWarning.*?\n/g, '');
|
|
|
- const cleanStderr = stderr ? stderr.replace(/\[33m.*?\[0m/g, '').replace(/DeprecationWarning.*?\n/g, '') : '';
|
|
|
+ if (method === 'full-screen') {
|
|
|
+ // 全屏OCR识别
|
|
|
+ result = await ocrFullScreenFromFunc(normalizedScreenshotPath, width, height);
|
|
|
+ } else if (method === 'by-avatar' && avatarPath) {
|
|
|
+ // 通过头像定位最后一条消息
|
|
|
+ let friendAvatarArg = null;
|
|
|
+ let myAvatarArg = null;
|
|
|
+
|
|
|
+ if (isAbsolute(avatarPath)) {
|
|
|
+ friendAvatarArg = avatarPath;
|
|
|
+ myAvatarArg = avatarPath;
|
|
|
+ } else {
|
|
|
+ const folderName = avatarPath.split(/[/\\]/)[0];
|
|
|
+ const avatarName = avatarPath.split(/[/\\]/).slice(1).join('/');
|
|
|
+ friendAvatarArg = join(__dirname, '..', 'static', 'processing', folderName, avatarName);
|
|
|
+ myAvatarArg = friendAvatarArg;
|
|
|
+ }
|
|
|
+
|
|
|
+ const normalizedFriendAvatar = friendAvatarArg.replace(/\\/g, '/');
|
|
|
+ const normalizedMyAvatar = myAvatarArg.replace(/\\/g, '/');
|
|
|
+ result = await getLastMessageFromFunc(normalizedScreenshotPath, normalizedFriendAvatar, normalizedMyAvatar, width, height);
|
|
|
+ } else {
|
|
|
+ // 默认使用全屏OCR
|
|
|
+ result = await ocrFullScreenFromFunc(normalizedScreenshotPath, width, height);
|
|
|
+ }
|
|
|
|
|
|
- if (cleanStdout.includes('未找到匹配的文字') || (cleanStdout.toLowerCase().includes('错误') && !cleanStdout.includes('找到文字')) || (cleanStderr && !cleanStderr.includes('Checking connectivity'))) {
|
|
|
- const errorMsg = cleanStdout.includes('未找到匹配的文字')
|
|
|
- ? `未找到文字: ${targetText}`
|
|
|
- : (cleanStderr || cleanStdout.match(/错误[：:]\s*(.+)/)?.[1] || '文字识别失败');
|
|
|
- return { success: false, error: errorMsg };
|
|
|
+ if (result.success) {
|
|
|
+ // 返回兼容旧API的格式
|
|
|
+ return {
|
|
|
+ success: true,
|
|
|
+ text: result.text || '',
|
|
|
+ position: result.position || null
|
|
|
+ };
|
|
|
+ } else {
|
|
|
+ return { success: false, error: result.error || 'OCR识别失败' };
|
|
|
}
|
|
|
+ } catch (error) {
|
|
|
+ console.error('OCR识别失败:', error);
|
|
|
+ if (error.message && error.message.includes('timeout')) {
|
|
|
+ return { success: false, error: 'OCR识别超时，请检查网络连接或稍后重试' };
|
|
|
+ }
|
|
|
+ return { success: false, error: error.message };
|
|
|
+ }
|
|
|
+}
|
|
|
|
|
|
- // 尝试从输出中提取 JSON（使用清理后的输出）
|
|
|
- let jsonMatch = cleanStdout.match(/JSON格式:\s*(\{[^}]+\})/);
|
|
|
- if (!jsonMatch) {
|
|
|
- // 如果没有找到 JSON 格式，尝试直接匹配 JSON 对象
|
|
|
- jsonMatch = cleanStdout.match(/\{\s*"x"\s*:\s*\d+\s*,\s*"y"\s*:\s*\d+\s*,\s*"width"\s*:\s*\d+\s*,\s*"height"\s*:\s*\d+\s*\}/);
|
|
|
+// 提取聊天记录
|
|
|
+export async function extractChatHistory(ipPort, friendAvatarPath, myAvatarPath, workflowFolderPath = null) {
|
|
|
+ try {
|
|
|
+ if (!ipPort) {
|
|
|
+ return { success: false, error: '缺少设备 ID' };
|
|
|
}
|
|
|
-
|
|
|
- if (!jsonMatch) {
|
|
|
- // 如果还是没有找到，尝试从文本中提取坐标
|
|
|
- const coordMatch = cleanStdout.match(/坐标:\s*x=(\d+),\s*y=(\d+),\s*宽度=(\d+),\s*高度=(\d+)/);
|
|
|
- if (coordMatch) {
|
|
|
- const x = parseInt(coordMatch[1], 10);
|
|
|
- const y = parseInt(coordMatch[2], 10);
|
|
|
- const w = parseInt(coordMatch[3], 10);
|
|
|
- const h = parseInt(coordMatch[4], 10);
|
|
|
-
|
|
|
- // 计算点击位置（中心点）
|
|
|
- const clickX = Math.round(x + w / 2);
|
|
|
- const clickY = Math.round(y + h / 2);
|
|
|
-
|
|
|
- return {
|
|
|
- success: true,
|
|
|
- coordinate: { x, y, width: w, height: h },
|
|
|
- clickPosition: { x: clickX, y: clickY }
|
|
|
- };
|
|
|
+
|
|
|
+ // 1. 获取设备分辨率
|
|
|
+ const resolutionResult = await getDeviceResolution(ipPort);
|
|
|
+ if (!resolutionResult.success) {
|
|
|
+ return { success: false, error: '获取设备分辨率失败' };
|
|
|
+ }
|
|
|
+ const { width, height } = resolutionResult;
|
|
|
+
|
|
|
+ // 2. 获取屏幕截图
|
|
|
+ const screenshotResult = await captureScreenshot(ipPort, { format: 'png' });
|
|
|
+ if (!screenshotResult.success || !screenshotResult.data) {
|
|
|
+ return { success: false, error: '获取屏幕截图失败' };
|
|
|
+ }
|
|
|
+
|
|
|
+ // 3. 保存截图到临时文件（如果提供了工作流文件夹，保存到 tmp/时间戳 目录）
|
|
|
+ let screenshotPath;
|
|
|
+ if (workflowFolderPath) {
|
|
|
+ const { mkdir } = await import('fs/promises');
|
|
|
+ // 确保 workflowFolderPath 是绝对路径
|
|
|
+ let absoluteWorkflowPath = workflowFolderPath;
|
|
|
+ if (!isAbsolute(workflowFolderPath)) {
|
|
|
+ // 如果已经是 static/processing/xxx 格式，直接拼接（去掉开头的 static/processing）
|
|
|
+ if (workflowFolderPath.startsWith('static/processing/')) {
|
|
|
+ const folderName = workflowFolderPath.replace('static/processing/', '');
|
|
|
+ absoluteWorkflowPath = join(__dirname, '..', 'static', 'processing', folderName);
|
|
|
+ } else if (workflowFolderPath.startsWith('static\\processing\\')) {
|
|
|
+ const folderName = workflowFolderPath.replace('static\\processing\\', '');
|
|
|
+ absoluteWorkflowPath = join(__dirname, '..', 'static', 'processing', folderName);
|
|
|
+ } else {
|
|
|
+ // 如果只是文件夹名，需要加上 static/processing
|
|
|
+ absoluteWorkflowPath = join(__dirname, '..', 'static', 'processing', workflowFolderPath);
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
- // 如果都找不到，返回错误并输出原始输出用于调试
|
|
|
- console.error('Python 脚本输出:', cleanStdout);
|
|
|
- console.error('Python 脚本错误输出:', cleanStderr);
|
|
|
- return { success: false, error: `无法解析 Python 脚本输出。输出: ${cleanStdout.substring(0, 200)}` };
|
|
|
+ const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19).replace('T', '_');
|
|
|
+ const tmpDir = join(absoluteWorkflowPath, 'tmp', timestamp);
|
|
|
+ await mkdir(tmpDir, { recursive: true });
|
|
|
+ screenshotPath = join(tmpDir, 'screenshot.png');
|
|
|
+ } else {
|
|
|
+ const tempDir = join(__dirname, '..');
|
|
|
+ screenshotPath = join(tempDir, 'temp_screenshot_chat.png');
|
|
|
}
|
|
|
-
|
|
|
- let coordinate;
|
|
|
+ const screenshotBuffer = Buffer.from(screenshotResult.data, 'base64');
|
|
|
+ await writeFile(screenshotPath, screenshotBuffer);
|
|
|
+
|
|
|
+ // 验证文件是否成功写入
|
|
|
try {
|
|
|
- coordinate = JSON.parse(jsonMatch[1] || jsonMatch[0]);
|
|
|
- } catch (parseError) {
|
|
|
- console.error('JSON 解析失败:', parseError);
|
|
|
- console.error('原始输出:', cleanStdout);
|
|
|
- return { success: false, error: `JSON 解析失败: ${parseError.message}` };
|
|
|
+ const { access, constants } = await import('fs/promises');
|
|
|
+ await access(screenshotPath, constants.F_OK);
|
|
|
+ console.log(`截图已保存到: ${screenshotPath}`);
|
|
|
+ } catch (err) {
|
|
|
+ console.error(`截图文件写入验证失败: ${screenshotPath}`, err);
|
|
|
+ return { success: false, error: `截图文件写入失败: ${err.message}` };
|
|
|
+ }
|
|
|
+
|
|
|
+ // 4. 调用 JS 函数提取聊天记录
|
|
|
+ // 转换头像路径为绝对路径
|
|
|
+ let friendAvatarArg = null;
|
|
|
+ if (friendAvatarPath) {
|
|
|
+ if (isAbsolute(friendAvatarPath)) {
|
|
|
+ friendAvatarArg = friendAvatarPath;
|
|
|
+ } else {
|
|
|
+ friendAvatarArg = join(__dirname, '..', 'static', 'processing', friendAvatarPath);
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
- const { x, y, width: w, height: h } = coordinate;
|
|
|
+ let myAvatarArg = null;
|
|
|
+ if (myAvatarPath) {
|
|
|
+ if (isAbsolute(myAvatarPath)) {
|
|
|
+ myAvatarArg = myAvatarPath;
|
|
|
+ } else {
|
|
|
+ myAvatarArg = join(__dirname, '..', 'static', 'processing', myAvatarPath);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 如果提供了工作流文件夹路径，转换为绝对路径
|
|
|
+ let workflowFolderArg = null;
|
|
|
+ if (workflowFolderPath) {
|
|
|
+ if (isAbsolute(workflowFolderPath)) {
|
|
|
+ workflowFolderArg = workflowFolderPath;
|
|
|
+ } else {
|
|
|
+ if (workflowFolderPath.startsWith('static/processing/')) {
|
|
|
+ const folderName = workflowFolderPath.replace('static/processing/', '');
|
|
|
+ workflowFolderArg = join(__dirname, '..', 'static', 'processing', folderName);
|
|
|
+ } else if (workflowFolderPath.startsWith('static\\processing\\')) {
|
|
|
+ const folderName = workflowFolderPath.replace('static\\processing\\', '');
|
|
|
+ workflowFolderArg = join(__dirname, '..', 'static', 'processing', folderName);
|
|
|
+ } else {
|
|
|
+ workflowFolderArg = join(__dirname, '..', 'static', 'processing', workflowFolderPath);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ const result = await extractChatHistoryFromFunc(screenshotPath, friendAvatarArg, myAvatarArg, width, height, workflowFolderArg);
|
|
|
+ return result;
|
|
|
+ } catch (error) {
|
|
|
+ console.error('提取聊天记录失败:', error);
|
|
|
+ if (error.message && error.message.includes('timeout')) {
|
|
|
+ return { success: false, error: '提取聊天记录超时，请检查网络连接或稍后重试' };
|
|
|
+ }
|
|
|
+ return { success: false, error: error.message };
|
|
|
+ }
|
|
|
+}
|
|
|
|
|
|
- // 计算点击位置（中心点）
|
|
|
- const clickX = Math.round(x + w / 2);
|
|
|
- const clickY = Math.round(y + h / 2);
|
|
|
+// 获取最后一条消息（带发送者信息）
|
|
|
+export async function getLastChatMessage(ipPort, friendAvatarPath, myAvatarPath) {
|
|
|
+ try {
|
|
|
+ if (!ipPort) {
|
|
|
+ return { success: false, error: '缺少设备 ID' };
|
|
|
+ }
|
|
|
|
|
|
- return {
|
|
|
- success: true,
|
|
|
- coordinate: { x, y, width: w, height: h },
|
|
|
- clickPosition: { x: clickX, y: clickY }
|
|
|
- };
|
|
|
+ // 1. 获取设备分辨率
|
|
|
+ const resolutionResult = await getDeviceResolution(ipPort);
|
|
|
+ if (!resolutionResult.success) {
|
|
|
+ return { success: false, error: '获取设备分辨率失败' };
|
|
|
+ }
|
|
|
+ const { width, height } = resolutionResult;
|
|
|
+
|
|
|
+ // 2. 获取屏幕截图
|
|
|
+ const screenshotResult = await captureScreenshot(ipPort, { format: 'png' });
|
|
|
+ if (!screenshotResult.success || !screenshotResult.data) {
|
|
|
+ return { success: false, error: '获取屏幕截图失败' };
|
|
|
+ }
|
|
|
+
|
|
|
+ // 3. 保存截图到临时文件
|
|
|
+ const tempDir = join(__dirname, '..');
|
|
|
+ const screenshotPath = join(tempDir, 'temp_screenshot_chat.png');
|
|
|
+ const screenshotBuffer = Buffer.from(screenshotResult.data, 'base64');
|
|
|
+ await writeFile(screenshotPath, screenshotBuffer);
|
|
|
+
|
|
|
+ // 4. 调用 JS 实现获取最后一条消息
|
|
|
+ // 转换头像路径为绝对路径
|
|
|
+ let friendAvatarArg = null;
|
|
|
+ if (friendAvatarPath) {
|
|
|
+ if (isAbsolute(friendAvatarPath)) {
|
|
|
+ friendAvatarArg = friendAvatarPath;
|
|
|
+ } else {
|
|
|
+ friendAvatarArg = join(__dirname, '..', 'static', 'processing', friendAvatarPath);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ let myAvatarArg = null;
|
|
|
+ if (myAvatarPath) {
|
|
|
+ if (isAbsolute(myAvatarPath)) {
|
|
|
+ myAvatarArg = myAvatarPath;
|
|
|
+ } else {
|
|
|
+ myAvatarArg = join(__dirname, '..', 'static', 'processing', myAvatarPath);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ const normalizedScreenshotPath = screenshotPath.replace(/\\/g, '/');
|
|
|
+ const normalizedFriendAvatar = friendAvatarArg ? friendAvatarArg.replace(/\\/g, '/') : null;
|
|
|
+ const normalizedMyAvatar = myAvatarArg ? myAvatarArg.replace(/\\/g, '/') : null;
|
|
|
+
|
|
|
+ const result = await getLastMessageFromFunc(normalizedScreenshotPath, normalizedFriendAvatar, normalizedMyAvatar, width, height);
|
|
|
+
|
|
|
+ if (result.success) {
|
|
|
+ // 确保正确显示UTF-8编码的中文
|
|
|
+ const displayText = result.text || '';
|
|
|
+ try {
|
|
|
+ const textStr = Buffer.isBuffer(displayText)
|
|
|
+ ? displayText.toString('utf8')
|
|
|
+ : String(displayText);
|
|
|
+ console.log(`最后一条消息 [${result.sender || 'unknown'}]:`, textStr);
|
|
|
+ } catch (e) {
|
|
|
+ console.log(`最后一条消息 [${result.sender || 'unknown'}]:`, displayText);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ return result;
|
|
|
} catch (error) {
|
|
|
- console.error('文字识别失败:', error);
|
|
|
- // 如果是超时错误，提供更友好的提示
|
|
|
+ console.error('获取最后一条消息失败:', error);
|
|
|
if (error.message && error.message.includes('timeout')) {
|
|
|
- return { success: false, error: '文字识别超时，请检查网络连接或稍后重试' };
|
|
|
+ return { success: false, error: '获取最后一条消息超时，请检查网络连接或稍后重试' };
|
|
|
}
|
|
|
return { success: false, error: error.message };
|
|
|
}
|
|
|
@@ -364,4 +527,227 @@ export function registerIpcHandlers() {
|
|
|
ipcMain.handle('find-text-and-get-coordinate', async (event, ipPort, targetText) => {
|
|
|
return await findTextAndGetCoordinate(ipPort, targetText);
|
|
|
});
|
|
|
+
|
|
|
+ ipcMain.handle('ocr-last-message', async (event, ipPort, method, avatarPath, area, folderPath) => {
|
|
|
+ return await ocrLastMessage(ipPort, method, avatarPath, area, folderPath);
|
|
|
+ });
|
|
|
+
|
|
|
+ ipcMain.handle('extract-chat-history', async (event, ipPort, friendAvatarPath, myAvatarPath, workflowFolderPath) => {
|
|
|
+ return await extractChatHistory(ipPort, friendAvatarPath, myAvatarPath, workflowFolderPath);
|
|
|
+ });
|
|
|
+
|
|
|
+ ipcMain.handle('get-last-chat-message', async (event, ipPort, friendAvatarPath, myAvatarPath) => {
|
|
|
+ return await getLastChatMessage(ipPort, friendAvatarPath, myAvatarPath);
|
|
|
+ });
|
|
|
+
|
|
|
+ ipcMain.handle('save-workflow', async (event, workflowJson, imagesData) => {
|
|
|
+ return await saveWorkflow(workflowJson, imagesData);
|
|
|
+ });
|
|
|
+
|
|
|
+ ipcMain.handle('delete-workflow', async (event, folderName) => {
|
|
|
+ return await deleteWorkflow(folderName);
|
|
|
+ });
|
|
|
+
|
|
|
+ ipcMain.handle('ensure-directory', async (event, dirPath) => {
|
|
|
+ try {
|
|
|
+ await mkdir(dirPath, { recursive: true });
|
|
|
+ return { success: true };
|
|
|
+ } catch (error) {
|
|
|
+ return { success: false, error: error.message };
|
|
|
+ }
|
|
|
+ });
|
|
|
+
|
|
|
+ ipcMain.handle('write-text-file', async (event, filePath, content) => {
|
|
|
+ try {
|
|
|
+ await writeFile(filePath, content, 'utf-8');
|
|
|
+ return { success: true };
|
|
|
+ } catch (error) {
|
|
|
+ return { success: false, error: error.message };
|
|
|
+ }
|
|
|
+ });
|
|
|
+
|
|
|
+ ipcMain.handle('read-text-file', async (event, filePath) => {
|
|
|
+ try {
|
|
|
+ const content = await readFile(filePath, 'utf-8');
|
|
|
+ return { success: true, content };
|
|
|
+ } catch (error) {
|
|
|
+ return { success: false, error: error.message };
|
|
|
+ }
|
|
|
+ });
|
|
|
+
|
|
|
+ // 保存聊天记录到 history 文件夹
|
|
|
+ ipcMain.handle('save-chat-history', async (event, workflowFolderPath, historyData) => {
|
|
|
+ try {
|
|
|
+ // 将相对路径转换为绝对路径
|
|
|
+ let absoluteWorkflowPath = workflowFolderPath;
|
|
|
+ if (!isAbsolute(workflowFolderPath)) {
|
|
|
+ if (workflowFolderPath.startsWith('static/processing/')) {
|
|
|
+ const folderName = workflowFolderPath.replace('static/processing/', '');
|
|
|
+ absoluteWorkflowPath = join(__dirname, '..', 'static', 'processing', folderName);
|
|
|
+ } else if (workflowFolderPath.startsWith('static\\processing\\')) {
|
|
|
+ const folderName = workflowFolderPath.replace('static\\processing\\', '');
|
|
|
+ absoluteWorkflowPath = join(__dirname, '..', 'static', 'processing', folderName);
|
|
|
+ } else {
|
|
|
+ absoluteWorkflowPath = join(__dirname, '..', 'static', 'processing', workflowFolderPath);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ const historyDir = join(absoluteWorkflowPath, 'history');
|
|
|
+ await mkdir(historyDir, { recursive: true });
|
|
|
+
|
|
|
+ // 生成文件名（使用时间戳）
|
|
|
+ const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, -5);
|
|
|
+ const fileName = `chat_${timestamp}.json`;
|
|
|
+ const filePath = join(historyDir, fileName);
|
|
|
+
|
|
|
+ // 保存到文件
|
|
|
+ await writeFile(filePath, JSON.stringify(historyData, null, 2), 'utf-8');
|
|
|
+
|
|
|
+ console.log(`聊天记录已保存到: ${filePath}`);
|
|
|
+ return { success: true, filePath };
|
|
|
+ } catch (error) {
|
|
|
+ console.error('保存聊天记录失败:', error);
|
|
|
+ return { success: false, error: error.message };
|
|
|
+ }
|
|
|
+ });
|
|
|
+
|
|
|
+ // 保存聊天记录总结
|
|
|
+ ipcMain.handle('save-chat-history-summary', async (event, workflowFolderPath, summary) => {
|
|
|
+ try {
|
|
|
+ // 将相对路径转换为绝对路径
|
|
|
+ let absoluteWorkflowPath = workflowFolderPath;
|
|
|
+ if (!isAbsolute(workflowFolderPath)) {
|
|
|
+ if (workflowFolderPath.startsWith('static/processing/')) {
|
|
|
+ const folderName = workflowFolderPath.replace('static/processing/', '');
|
|
|
+ absoluteWorkflowPath = join(__dirname, '..', 'static', 'processing', folderName);
|
|
|
+ } else if (workflowFolderPath.startsWith('static\\processing\\')) {
|
|
|
+ const folderName = workflowFolderPath.replace('static\\processing\\', '');
|
|
|
+ absoluteWorkflowPath = join(__dirname, '..', 'static', 'processing', folderName);
|
|
|
+ } else {
|
|
|
+ absoluteWorkflowPath = join(__dirname, '..', 'static', 'processing', workflowFolderPath);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ const historyDir = join(absoluteWorkflowPath, 'history');
|
|
|
+ await mkdir(historyDir, { recursive: true });
|
|
|
+
|
|
|
+ const summaryFilePath = join(historyDir, 'summary.txt');
|
|
|
+ await writeFile(summaryFilePath, summary, 'utf-8');
|
|
|
+
|
|
|
+ console.log(`聊天记录总结已保存到: ${summaryFilePath}`);
|
|
|
+ return { success: true };
|
|
|
+ } catch (error) {
|
|
|
+ console.error('保存聊天记录总结失败:', error);
|
|
|
+ return { success: false, error: error.message };
|
|
|
+ }
|
|
|
+ });
|
|
|
+
|
|
|
+ // 读取聊天记录总结
|
|
|
+ ipcMain.handle('get-chat-history-summary', async (event, workflowFolderPath) => {
|
|
|
+ try {
|
|
|
+ // 将相对路径转换为绝对路径
|
|
|
+ let absoluteWorkflowPath = workflowFolderPath;
|
|
|
+ if (!isAbsolute(workflowFolderPath)) {
|
|
|
+ if (workflowFolderPath.startsWith('static/processing/')) {
|
|
|
+ const folderName = workflowFolderPath.replace('static/processing/', '');
|
|
|
+ absoluteWorkflowPath = join(__dirname, '..', 'static', 'processing', folderName);
|
|
|
+ } else if (workflowFolderPath.startsWith('static\\processing\\')) {
|
|
|
+ const folderName = workflowFolderPath.replace('static\\processing\\', '');
|
|
|
+ absoluteWorkflowPath = join(__dirname, '..', 'static', 'processing', folderName);
|
|
|
+ } else {
|
|
|
+ absoluteWorkflowPath = join(__dirname, '..', 'static', 'processing', workflowFolderPath);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ const summaryFilePath = join(absoluteWorkflowPath, 'history', 'summary.txt');
|
|
|
+
|
|
|
+ // 检查文件是否存在
|
|
|
+ try {
|
|
|
+ const { access, constants } = await import('fs/promises');
|
|
|
+ await access(summaryFilePath, constants.F_OK);
|
|
|
+ const summary = await readFile(summaryFilePath, 'utf-8');
|
|
|
+ return { success: true, summary: summary.trim() };
|
|
|
+ } catch (error) {
|
|
|
+ // 文件不存在，返回空字符串
|
|
|
+ return { success: true, summary: '' };
|
|
|
+ }
|
|
|
+ } catch (error) {
|
|
|
+ console.error('读取聊天记录总结失败:', error);
|
|
|
+ return { success: false, error: error.message };
|
|
|
+ }
|
|
|
+ });
|
|
|
+}
|
|
|
+
|
|
|
+// 保存工作流到 static/processing 目录
|
|
|
+export async function saveWorkflow(workflowJson, imagesData = []) {
|
|
|
+ try {
|
|
|
+ // 支持新旧格式
|
|
|
+ const hasActions = Array.isArray(workflowJson.actions) || Array.isArray(workflowJson);
|
|
|
+ if (!workflowJson || typeof workflowJson !== 'object' || !hasActions) {
|
|
|
+ return { success: false, error: '工作流格式错误：缺少 actions 数组' };
|
|
|
+ }
|
|
|
+
|
|
|
+ // 生成文件夹名称（使用时间戳）
|
|
|
+ const now = new Date();
|
|
|
+ const timestamp = now.getFullYear() +
|
|
|
+ String(now.getMonth() + 1).padStart(2, '0') +
|
|
|
+ String(now.getDate()).padStart(2, '0') + '_' +
|
|
|
+ String(now.getHours()).padStart(2, '0') +
|
|
|
+ String(now.getMinutes()).padStart(2, '0') +
|
|
|
+ String(now.getSeconds()).padStart(2, '0');
|
|
|
+ const folderName = workflowJson.name || `工作流_${timestamp}`;
|
|
|
+
|
|
|
+ // 创建工作流文件夹
|
|
|
+ const workflowPath = join(__dirname, '..', 'static', 'processing', folderName);
|
|
|
+ await mkdir(workflowPath, { recursive: true });
|
|
|
+
|
|
|
+ // 保存 processing.json
|
|
|
+ const jsonPath = join(workflowPath, 'processing.json');
|
|
|
+ const jsonContent = JSON.stringify(workflowJson, null, '\t');
|
|
|
+ await writeFile(jsonPath, jsonContent, 'utf-8');
|
|
|
+
|
|
|
+ // 保存图片
|
|
|
+ if (imagesData && Array.isArray(imagesData) && imagesData.length > 0) {
|
|
|
+ for (const imageData of imagesData) {
|
|
|
+ if (imageData.base64 && imageData.name) {
|
|
|
+ try {
|
|
|
+ // 将base64转换为Buffer
|
|
|
+ const imageBuffer = Buffer.from(imageData.base64, 'base64');
|
|
|
+ const imagePath = join(workflowPath, imageData.name);
|
|
|
+ await writeFile(imagePath, imageBuffer);
|
|
|
+ console.log(`图片已保存: ${imageData.name}`);
|
|
|
+ } catch (imageError) {
|
|
|
+ console.error(`保存图片失败 ${imageData.name}:`, imageError);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ console.log(`工作流已保存: ${folderName}`);
|
|
|
+ return { success: true, folderName, path: workflowPath };
|
|
|
+ } catch (error) {
|
|
|
+ console.error('保存工作流失败:', error);
|
|
|
+ return { success: false, error: error.message };
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+// 删除工作流文件夹
|
|
|
+export async function deleteWorkflow(folderName) {
|
|
|
+ try {
|
|
|
+ if (!folderName || typeof folderName !== 'string') {
|
|
|
+ return { success: false, error: '文件夹名称无效' };
|
|
|
+ }
|
|
|
+
|
|
|
+ // 构建文件夹路径
|
|
|
+ const workflowPath = join(__dirname, '..', 'static', 'processing', folderName);
|
|
|
+
|
|
|
+ // 删除整个文件夹（包括所有内容）
|
|
|
+ await rm(workflowPath, { recursive: true, force: true });
|
|
|
+
|
|
|
+ console.log(`工作流已删除: ${folderName}`);
|
|
|
+ return { success: true, folderName };
|
|
|
+ } catch (error) {
|
|
|
+ console.error('删除工作流失败:', error);
|
|
|
+ return { success: false, error: error.message };
|
|
|
+ }
|
|
|
}
|
