perf: 使用stream读取文件防止内存溢出

lib/Base.js

@@ -254,19 +254,19 @@ const Base = {
     },
     /**
      * 读取dyid文件
-     * @returns {string}
+     * @returns {fs.ReadStream}
      */
     readDyidFile: () => {
         const fpath = path.join('./lib', 'dyid.txt');
-        return fs.readFileSync(fpath).toString()
+        return fs.createReadStream(fpath, { encoding: 'utf8', highWaterMark: 19 * 1000 })
     },
     /**
      * 追加dyid
-     * @param {string} dyid
+     * @returns {fs.WriteStream}
      */
-    writeDyidFile: (dyid) => {
+    writeDyidFile: () => {
         const fpath = path.join('./lib', 'dyid.txt');
-        return fs.writeFileSync(fpath, dyid, { flag: 'w' })
+        return fs.createWriteStream(fpath, { flags: 'a' })
     }
 };
 

lib/GithubAPI.js

@@ -78,7 +78,7 @@ const GihubAPI = {
                             resolve(true)
                         }))
                     }).on('error', () => {
-                        wtbs.close()
+                        wtbs.destroy()
                         resolve(false)
                     })
                 },

lib/Monitor.js

@@ -19,7 +19,6 @@ class Monitor extends Public {
         typeof param === 'number' ? this.UID = param : this.tag_name = param;
         this.tagid = 0; /* tagid初始化为默认分组 */
         this.attentionList = ''; /* 转为字符串的所有关注的up主uid */
-        this.AllMyLotteryInfo = ''; /* 转发过的动态信息 */
         this.isAttentionErr = false;
     }
     /**
@@ -30,20 +29,6 @@ class Monitor extends Public {
         if (GlobalVar.Lottery.length === 0) { console.log('抽奖信息为空'); return; }
         this.tagid = await BiliAPI.checkMyPartition(); /* 检查关注分区 */
         this.attentionList = await BiliAPI.getAttentionList(GlobalVar.myUID); /* 获取关注列表 */
-        const alldyid = await MyStorage.getDyid(); /* 获取储存的之前转过的所有动态 */
-        if (alldyid.length < 19) {
-            const { allModifyDynamicResArray: AllDynamic } = (await this.checkAllDynamic(GlobalVar.myUID, 5));
-            let newdyid = '';
-            for (let index = 0; index < AllDynamic.length; index++) {
-                const oneDynamicObj = AllDynamic[index];
-                if (typeof oneDynamicObj.origin_dynamic_id === 'string')
-                    newdyid += oneDynamicObj.origin_dynamic_id + ',';
-            }
-            this.AllMyLotteryInfo = (Array.from(new Set([...newdyid.split(','), ...alldyid.split(',')]))).toString();
-            await MyStorage.updateDyid(this.AllMyLotteryInfo);
-        } else {
-            this.AllMyLotteryInfo = alldyid;
-        }
         await this.startLottery();
     }
     /**
@@ -60,7 +45,7 @@ class Monitor extends Public {
                 if (!status) break
                 dyids.push(Lottery.dyid);
             }
-            MyStorage.updateDyid((Array.from(new Set([...dyids, ...this.AllMyLotteryInfo.split(',')]))).toString())
+            if (dyids.length) MyStorage.updateDyid(dyids.toString())
             console.log('开始转发下一组动态');
         } else {
             status = 1;
@@ -129,7 +114,7 @@ class Monitor extends Public {
                 const isFollowed = (new RegExp(uid)).test(self.attentionList);
                 if (this.isAttentionErr && !isFollowed) continue;
                 /* 判断是否转发过 */
-                const isRelay = (new RegExp(dyid)).test(self.AllMyLotteryInfo);
+                const isRelay = await MyStorage.searchDyid(dyid);
                 /* 获取黑名单并去重合并 */
                 const { blacklist: remote_blacklist } = GlobalVar.remoteconfig;
                 const new_blacklist = remote_blacklist ?

lib/MyStorage.js

@@ -28,19 +28,45 @@ const MyStorage = {
         });
     },
     /**
-     * 获取dyid
+     * 搜索dyid
+     * @returns {Promise<boolean>}
      */
-    getDyid: () => {
-        console.log('获取已转发过的动态信息');
-        return Base.readDyidFile()
+    searchDyid: (dyid) => {
+        return new Promise((resolve) => {
+            const Rdyid = new RegExp(dyid);
+            const rs = Base.readDyidFile();
+            let status = false;
+            rs.on('data', chunk => {
+                if (Rdyid.test(chunk)) {
+                    status = true
+                }
+            })
+            rs.on('end', () => {
+                resolve(status)
+            })
+            rs.on('error', err => {
+                console.log(err)
+                resolve(status)
+            })
+        })
     },
     /**
      * 更新dyid
      * @param {string} dyid
      */
     updateDyid: (dyid) => {
-        console.log('更新已转发过的动态信息');
-        return Base.writeDyidFile(dyid)
+        console.log('写入已转发过的动态信息');
+        return new Promise((resolve) => {
+            const ws = Base.writeDyidFile();
+            ws.write(dyid + ',', () => {
+                ws.destroy();
+                resolve()
+            })
+            ws.on('error', err => {
+                console.log(err)
+                resolve()
+            })
+        });
     }
 }
 

test/huge_file/read.js

@@ -0,0 +1,26 @@
+const { createReadStream, createWriteStream } = require('fs');
+
+const file = 'test/huge_file/dyid.txt';
+const test = 'test/huge_file/test.txt';
+
+const ws = createReadStream(file, { encoding: 'utf8', highWaterMark: 19 * 1000 })
+
+const rs = createWriteStream(test, { flags: 'a' })
+
+rs.write(Date.now() + '->', () => {
+    console.log('写入完毕');
+    rs.destroy()
+})
+
+let i = 0;
+ws.on('data', chunk => {
+    i++;
+    if (/000000000000000000/.test(chunk)) {
+        console.log('success')
+        console.log('in ' + i)
+        // 14764824 bytes
+        // success
+        // in 778
+        // [Done] exited with code=0 in 0.331 seconds
+    }
+})

test/huge_file/test.txt

@@ -0,0 +1 @@
+===1614266851666->