日志优化

src/utils/file.js

@@ -156,7 +156,7 @@ async function readLastNLines(dirPath, filePathList, n) {
         const lastNLines = allFile.slice(-n).reverse();
         return lastNLines;
     } catch (e) {
-        logger.info('获取日志异常了~~1111111111111111', e);
+        logger.info('读取文件发生异常咯~~2', e);
     }
 }
 
@@ -195,46 +195,138 @@ async function getRecentLogs(dirPath, logFiles, requiredLogs = 100) {
 }
 
 
-async function readLinesFromEnd(filePath, maxLines) {
-    const lines = [];
-    const fileSize = fs.statSync(filePath).size;
-    const chunkSize = 1024 * 10 * 5; // 每次读取的块大小
-
-    if (fileSize < chunkSize) {
-        const fileStream = fs.createReadStream(filePath, {
-            start: Math.max(0, 0),
-            end: fileSize
-        });
+async function getLatestLogEntries(dirPath, logFiles, requiredLogs = 100) {
+    const MAX_LOGS = requiredLogs;
+    let logs = [];
 
-        let buffer = '';
-        for await (const chunk of fileStream) {
-            buffer = chunk + buffer;
-            let lineEndIndex = buffer.length;
+    try {
+        for (const file of logFiles) {
+            // 创建文件流
+            const filepath = `${dirPath}/${file}`;
+            logger.info('文件~~~', filepath);
+
+            // const readStream = fs.createReadStream(filepath, { highWaterMark: 2 * 1024 * 1024 }); // 每次读取10MB
+            // let buffer = '';
+            //
+            // logs = await new Promise((resolve,reject) =>{
+            //     // 逐块读取并处理数据
+            //     readStream.on('data', (chunk) => {
+            //         buffer += chunk.toString(); // 将块数据转成字符串
+            //         const lines = buffer.split('\n'); // 按行拆分
+            //
+            //         // 处理新行
+            //         for (let i = 0; i < lines.length - 1; i++) {
+            //             logs.unshift(lines[i]); // 添加到数组前面以实现倒序
+            //             // logs.push(lines[i]); // 添加到数组前面以实现倒序
+            //             logger.info( lines[i]);
+            //
+            //             if (logs.length >= MAX_LOGS) break; // 达到所需条数则停止
+            //         }
+            //
+            //         // 保留未处理的最后一行（如果是半行，继续留在缓冲区）
+            //         buffer = lines[lines.length - 1];
+            //     });
+            //
+            //     // 读取结束后处理剩余缓冲区
+            //     readStream.on('end', () => {
+            //         if (buffer) {
+            //             logs.unshift(buffer); // 加入最后的未处理行
+            //         }
+            //         resolve(logs); // 完成时解析 Promise
+            //     });
+            //
+            //     readStream.on('error', (err) => {
+            //         reject(err); // 如果出错，拒绝 Promise
+            //     });
+            //
+            //     // 确保在获取到足够记录时停止读取
+            //     readStream.on('close', () => {
+            //         if (logs.length >= MAX_LOGS) {
+            //             readStream.destroy(); // 如果达到了最大记录数，停止读取流
+            //         }else{
+            //             logger.info('----继续读取~~',logs.length);
+            //         }
+            //     });
+            // })
+            // // logger.info( logs);
+            // // 如果已达到所需行数，停止继续读取文件
+            // if (logs.length >= MAX_LOGS) {
+            //     break; // 退出循环
+            // }
 
-            while (lineEndIndex >= 0 && lines.length < maxLines) {
-                const lineStartIndex = buffer.lastIndexOf('\n', lineEndIndex - 1);
-                if (lineStartIndex === -1) {
-                    break;
-                }
-                const line = buffer.substring(lineStartIndex + 1, lineEndIndex).trim();
-                if (line) {
-                    lines.push(line);
-                }
-                lineEndIndex = lineStartIndex;
+            // 文件信息
+            const stats = fs.statSync(filepath); // 获取文件信息
+            let position = stats.size; // 从文件末尾开始
+            const chunkSize = 1 * 1024 * 1024; // 每次读取1MB
+
+            // 文件读取方式
+            while (position > 0 && logs.length < MAX_LOGS) {
+                const readSize = Math.min(chunkSize, position); // 确保不读取超过文件的大小
+                const readStream = fs.createReadStream(filepath, {
+                    start: position - readSize, // 从当前位置向上读取
+                    end: position - 1, // 读取到当前位置
+                    highWaterMark: readSize // 一次性读取指定大小
+                });
+
+                // 缓存
+                let buffer = '';
+                await new Promise((resolve, reject) => {
+                    readStream.on('data', (chunk) => {
+                        buffer = chunk.toString() + buffer; // 新数据放在前面
+                    });
+
+                    readStream.on('end', () => {
+                        // 按行处理数据
+                        const lines = buffer.split('\n');
+                        while (lines.length > 0) {
+                            const line = lines.pop(); // 取出最新的一行
+                            if (line) {
+                                logs.unshift(line); // 添加到日志数组开头
+                                // logger.info(line);
+                                if (logs.length >= requiredLogs) break; // 达到要求的条数
+                            }
+                        }
+                        resolve();
+                    });
+
+                    readStream.on('error', (err) => {
+                        console.error('读取文件发生错误:', err);
+                        reject(err); // 确保不会因错误而卡住
+                    });
+
+                    readStream.on('close', () => {
+                        if (logs.length >= MAX_LOGS) {
+                            readStream.destroy(); // 如果达到了最大记录数，停止读取流
+                        } else {
+                            logger.info('----继续读取~~', logs.length);
+                        }
+                    });
+                });
+                position -= readSize; // 递减位置
             }
 
-            if (lines.length >= maxLines) {
-                break;
+            // 如果已达到所需行数，停止继续读取文件
+            if (logs.length >= MAX_LOGS) {
+                break; // 退出循环
             }
         }
-    } else {
-        let forNum = (fileSize / chunkSize) + 1;
-        for (let i = 1; i <= forNum; i++) {
-            let position = fileSize - chunkSize * i;
-            let z = position + chunkSize
+    } catch (e) {
+        logger.info('读取文件发生异常咯~~~', e.message);
+    }
+    // logger.info( logs);
+    return logs.reverse(); // 如果日志不够500条，返回现有日志
+}
+
+async function readLinesFromEnd(filePath, maxLines) {
+    const lines = [];
+    const fileSize = fs.statSync(filePath).size;
+    const chunkSize = 1024 * 10 * 5; // 每次读取的块大小
+
+    try {
+        if (fileSize < chunkSize) {
             const fileStream = fs.createReadStream(filePath, {
-                start: Math.max(position, 0),
-                end: z
+                start: Math.max(0, 0),
+                end: fileSize
             });
 
             let buffer = '';
@@ -257,13 +349,47 @@ async function readLinesFromEnd(filePath, maxLines) {
                 if (lines.length >= maxLines) {
                     break;
                 }
+            }
+        } else {
+            let forNum = (fileSize / chunkSize) + 1;
+            for (let i = 1; i <= forNum; i++) {
+                let position = fileSize - chunkSize * i;
+                let z = position + chunkSize
+                const fileStream = fs.createReadStream(filePath, {
+                    start: Math.max(position, 0),
+                    end: z
+                });
+
+                let buffer = '';
+                for await (const chunk of fileStream) {
+                    buffer = chunk + buffer;
+                    let lineEndIndex = buffer.length;
+
+                    while (lineEndIndex >= 0 && lines.length < maxLines) {
+                        const lineStartIndex = buffer.lastIndexOf('\n', lineEndIndex - 1);
+                        if (lineStartIndex === -1) {
+                            break;
+                        }
+                        const line = buffer.substring(lineStartIndex + 1, lineEndIndex).trim();
+                        if (line) {
+                            lines.push(line);
+                        }
+                        lineEndIndex = lineStartIndex;
+                    }
+
+                    if (lines.length >= maxLines) {
+                        break;
+                    }
 
-                position -= chunkSize;
-                if (position < 0) {
-                    position = 0;
+                    position -= chunkSize;
+                    if (position < 0) {
+                        position = 0;
+                    }
                 }
             }
         }
+    } catch (e) {
+        logger.info('读取文件发生异常咯~~~1', e.message);
     }
 
     return lines.reverse();
@@ -297,6 +423,7 @@ module.exports = {
     writeFile,
     copyFileSync,
     readLastNLines,
+    getLatestLogEntries,
     getRecentLogs,
     getLastFile
 }

src/web.js

@@ -76,7 +76,7 @@ function init() {
         const n = param.n
         const id = param.id
 
-        var port = 1111;
+        var port = -1;
         robot.appMap.forEach((value, key) => {
             // logger.info(JSON.stringify(value))
             if (value.id + "" === id + "") {
@@ -97,8 +97,11 @@ function init() {
                 file.getLastFile(logPath, 5, async (fileNameList, _) => {
                     // logger.info(`---------------------5`)
                     // logger.info('cccccccccccccc文件：', fileNameList);
+
                     // result = await file.readLastNLines(logPath, fileNameList, n);
-                    result =  await  file.getRecentLogs(logPath,fileNameList.reverse(),n);
+                    // result =  await  file.getRecentLogs(logPath,fileNameList.reverse(),n);
+                    result =  await  file.getLatestLogEntries(logPath,fileNameList.reverse(),n);
+
 
                     // logger.info('?????:', result);
                     // logger.info(`---------------------6`)