优化日志获取

src/utils/file.js

@@ -3,6 +3,7 @@ const path = require('path');
 const {logger} = require("./logger");
 const {rustConfig} = require("../config");
 const http = require("./http");
+const readline = require('readline');
 
 
 // 检查目录是否存在，不存在新建
@@ -103,7 +104,7 @@ function getLastFile(dirPath, number, callback) {
             for (file of files) {
                 const filePath = path.join(dirPath, file);
                 const stats = await fs.promises.stat(filePath);
-                if(stats.isFile()){
+                if (stats.isFile()) {
                     fileList.push({name: file, time: stats.birthtime.getTime()});
                     // logger.info("文件:" + filePath + "时间：" + stats.birthtime.getTime())
                 }
@@ -122,7 +123,7 @@ function getLastFile(dirPath, number, callback) {
 }
 
 // 读取日志（倒叙）指定行数
-function readLastNLines(dirPath, filePathList, n) {
+async function readLastNLines(dirPath, filePathList, n) {
     try {
         // for (var i = 0; i < filePathList.length; i++) {
         //     var item = filePathList[i];
@@ -133,12 +134,16 @@ function readLastNLines(dirPath, filePathList, n) {
         var count = 0;
         for (var i = filePathList.length - 1; i >= 0; i--) {
             var item = filePathList[i];
-            var filepath = dirPath + "/" + item
-
+            const filepath = `${dirPath}/${item}`;
             logger.info("日志文件:" + filepath)
-            const content = fs.readFileSync(`${filepath}`, "utf8");
-            let lines = content.trim().split("\n");
 
+            //老版本 加载整个文件
+            // const content = fs.readFileSync(`${filepath}`, "utf8");
+            // let lines = content.trim().split("\n");
+
+            //新版本：流式读取和逐行处理
+            const lines = await readLines(filepath, n - count);
+            // logger.info("日:" + lines)
             fileList.push({
                 filePath: item,
                 lines: lines,
@@ -150,14 +155,38 @@ function readLastNLines(dirPath, filePathList, n) {
         }
         fileList.reverse();
 
+        // const allFile = [].concat(...fileList.map((item) => item.lines));
+        // let lastNLines = allFile.slice(-n).join("\n");
+        // return lastNLines.split("\n").reverse();
+
         const allFile = [].concat(...fileList.map((item) => item.lines));
-        let lastNLines = allFile.slice(-n).join("\n");
-        return lastNLines.split("\n").reverse();
+        const lastNLines = allFile.slice(-n).reverse();
+        return lastNLines;
     } catch (e) {
         logger.info('获取日志异常了~~1111111111111111', e);
     }
 }
 
+ async function readLines(filePath, maxLines) {
+     const fileStream = fs.createReadStream(filePath);
+     const rl = readline.createInterface({
+         input: fileStream,
+         crlfDelay: Infinity
+     });
+
+     const lines = [];
+     for await   (const line of rl) {
+         lines.push(line);
+         if (lines.length >= maxLines) {
+             break;
+         }
+     }
+
+     rl.close();
+     fileStream.close();
+     return lines;
+ }
+
 
 module.exports = {
     dowFile,

src/web.js

@@ -90,13 +90,13 @@ function init() {
             try {
                 file.checkPathSync(directoryPath)
 
-                file.getLastFile(logPath, 5, (fileNameList,_) => {
-                    // logger.info('文件：', fileNameList);
-                    result = file.readLastNLines(logPath, fileNameList, n);
-                    // logger.info('?????:', result);
-                    res.send({'code': 200, 'data': result, "message": "SUCCESS"});
-                });
-            }catch (e) {
+                   file.getLastFile(logPath, 5,  async (fileNameList, _) => {
+                       // logger.info('文件：', fileNameList);
+                       result = await file.readLastNLines(logPath, fileNameList, n);
+                       // logger.info('?????:', result);
+                       res.send({'code': 200, 'data': result, "message": "SUCCESS"});
+                   });
+            } catch (e) {
                 logger.info('获取日志异常了~~', e);
                 res.send({'code': 200, 'data': [], "message": "SUCCESS"});
             }
@@ -123,7 +123,7 @@ function init() {
         res.send({'code': 200, 'data': "null", "message": "SUCCESS"});
     })
     //启动-仓位检查
-    app.get('/searchPositions', (req, res) =>{
+    app.get('/searchPositions', (req, res) => {
         const param = req.query;
         logger.info(JSON.stringify(param));
         robot.searchPositions(param)