|
@@ -142,7 +142,7 @@ async function readLastNLines(dirPath, filePathList, n) {
|
|
// let lines = content.trim().split("\n");
|
|
// let lines = content.trim().split("\n");
|
|
|
|
|
|
//新版本:流式读取和逐行处理
|
|
//新版本:流式读取和逐行处理
|
|
- const lines = await readLines(filepath, n - count);
|
|
|
|
|
|
+ const lines = await readLinesFromEnd(filepath, n - count);
|
|
// logger.info("日:" + lines)
|
|
// logger.info("日:" + lines)
|
|
fileList.push({
|
|
fileList.push({
|
|
filePath: item,
|
|
filePath: item,
|
|
@@ -167,6 +167,47 @@ async function readLastNLines(dirPath, filePathList, n) {
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+
|
|
|
|
+async function readLinesFromEnd(filePath, maxLines) {
|
|
|
|
+ const lines = [];
|
|
|
|
+ const fileSize = fs.statSync(filePath).size;
|
|
|
|
+ const chunkSize = 1024; // 每次读取的块大小
|
|
|
|
+ let buffer = '';
|
|
|
|
+ let position = fileSize - chunkSize;
|
|
|
|
+
|
|
|
|
+ const fileStream = fs.createReadStream(filePath, {
|
|
|
|
+ start: Math.max(position, 0),
|
|
|
|
+ end: fileSize
|
|
|
|
+ });
|
|
|
|
+
|
|
|
|
+ for await (const chunk of fileStream) {
|
|
|
|
+ buffer = chunk + buffer;
|
|
|
|
+ let lineEndIndex = buffer.length;
|
|
|
|
+
|
|
|
|
+ while (lineEndIndex >= 0 && lines.length < maxLines) {
|
|
|
|
+ const lineStartIndex = buffer.lastIndexOf('\n', lineEndIndex - 1);
|
|
|
|
+ if (lineStartIndex === -1) {
|
|
|
|
+ break;
|
|
|
|
+ }
|
|
|
|
+ const line = buffer.substring(lineStartIndex + 1, lineEndIndex).trim();
|
|
|
|
+ if (line) {
|
|
|
|
+ lines.push(line);
|
|
|
|
+ }
|
|
|
|
+ lineEndIndex = lineStartIndex;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if (lines.length >= maxLines) {
|
|
|
|
+ break;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ position -= chunkSize;
|
|
|
|
+ if (position < 0) {
|
|
|
|
+ position = 0;
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ return lines;
|
|
|
|
+}
|
|
async function readLines(filePath, maxLines) {
|
|
async function readLines(filePath, maxLines) {
|
|
const fileStream = fs.createReadStream(filePath);
|
|
const fileStream = fs.createReadStream(filePath);
|
|
const rl = readline.createInterface({
|
|
const rl = readline.createInterface({
|