From 3cf3939b4976e920c22cad1dd96131a57112421e Mon Sep 17 00:00:00 2001 From: shanmite Date: Mon, 6 Dec 2021 16:12:41 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E8=AE=BE=E7=BD=AE`ch?= =?UTF-8?q?eck=5Fif=5Fduplicated`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 检查是否重复转发 - 不检查 -1 - 通过是否点赞判断 0 - 检索本地dyids文件 1 --- lib/core/monitor.js | 47 +++++++++++++++++++++++++++---------- lib/core/searcher.js | 42 ++++++++++++++++++++------------- lib/data/config.js | 8 +++++++ lib/data/global_var.js | 9 ++++--- lib/helper/d_storage.js | 52 +++++++++++++++++++++++++++++++++++++++++ lib/utils.js | 46 ++++++++++++++++++++++++++++++++++++ main.js | 2 +- my_config.example.js | 10 +++++++- 8 files changed, 183 insertions(+), 33 deletions(-) create mode 100644 lib/helper/d_storage.js diff --git a/lib/core/monitor.js b/lib/core/monitor.js index 641658d..94c8c7c 100644 --- a/lib/core/monitor.js +++ b/lib/core/monitor.js @@ -7,6 +7,7 @@ const { randomDynamic } = require('../helper/randomDynamic') const { Searcher } = require('./searcher'); const global_var = require("../data/global_var"); const config = require("../data/config"); +const d_storage = require('../helper/d_storage'); /** * 监视器 @@ -124,11 +125,13 @@ class Monitor extends Searcher { } if (ts < ts_10) { log.info('过滤', '已过开奖时间') + d_storage.updateDyid(lottery.dyid) await delay(filter_wait) continue } if (ts > ts_10 + config.maxday * 86400) { log.info('过滤', '超过指定开奖时间') + d_storage.updateDyid(lottery.dyid) await delay(filter_wait) continue } @@ -141,6 +144,7 @@ class Monitor extends Searcher { } if (followerNum < minfollower) { log.info('过滤', `粉丝数(${followerNum})小于指定数量`) + d_storage.updateDyid(lottery.dyid) await delay(filter_wait) continue } @@ -189,6 +193,8 @@ class Monitor extends Searcher { return status } + d_storage.updateDyid(lottery.dyid) + await delay(wait * (Math.random() + 0.5)); } log.info('抽奖', '开始转发下一组动态'); @@ -230,7 +236,7 @@ class Monitor extends Searcher { /** 所有抽奖信息 */ let alllotteryinfo = []; const - { set_lottery_info_url, key_words, model, chatmodel, max_create_time, is_imitator, only_followed, at_users, blockword, blacklist, use_public_blacklist } = config, + { check_if_duplicated, set_lottery_info_url, key_words, model, chatmodel, max_create_time, is_imitator, only_followed, at_users, blockword, blacklist, use_public_blacklist } = config, now_ts = Date.now() / 1000; /** @@ -249,6 +255,15 @@ class Monitor extends Searcher { log.info('筛选动态', `去重后(${protoLotteryInfo.length})`); + /**并发查询dyid */ + await Promise.all( + [...dyids_map.keys()] + .map(it => d_storage + .searchDyid(it) + .then(hasIt => dyids_map.set(it, hasIt)) + ) + ) + if (lottery_param[0] !== "APIs" && set_lottery_info_url && protoLotteryInfo.length) { log.info("上传抽奖信息", "开始") await new Promise((resolve) => { @@ -283,7 +298,12 @@ class Monitor extends Searcher { log.debug('正在筛选的动态信息', lottery_info); /* 遇到转发过就退出 */ - if (is_liked) { + if ( + (!check_if_duplicated + && is_liked) + || (check_if_duplicated === 1 + && dyids_map.get(dyid)) + ) { log.info("筛选动态", `已转发(https://t.bilibili.com/${dyid})`) return false } @@ -466,11 +486,12 @@ class Monitor extends Searcher { } let status = 0 - const { uid, dyid, chat_type, rid, relay_chat, ctrl } = option; + const + { uid, dyid, chat_type, rid, relay_chat, ctrl } = option, + { chat, check_if_duplicated } = config; /* 评论 */ if (rid && chat_type) { - const { chat } = config; status = await retryfn( 5, @@ -504,15 +525,17 @@ class Monitor extends Searcher { } /* 点赞 */ - status = await retryfn( - 5, - [1, 2, 3], - () => bili.autolike(dyid) - ) + if (!check_if_duplicated) { + status = await retryfn( + 5, + [1, 2, 3], + () => bili.autolike(dyid) + ) - if (status) { - log.warn("抽奖信息", `dyid: ${dyid}`) - return 4000 + status + if (status) { + log.warn("抽奖信息", `dyid: ${dyid}`) + return 4000 + status + } } /* 转发 */ diff --git a/lib/core/searcher.js b/lib/core/searcher.js index 6c79615..c6bb306 100644 --- a/lib/core/searcher.js +++ b/lib/core/searcher.js @@ -1,7 +1,8 @@ const utils = require('../utils'); const bili = require('../net/bili'); const { send } = require("../net/http"); -const { article_scan_page, article_create_time, not_check_article, get_dynamic_detail_wait, uid_scan_page, search_wait, tag_scan_page } = require("../data/config"); +const { check_if_duplicated, article_scan_page, article_create_time, not_check_article, get_dynamic_detail_wait, uid_scan_page, search_wait, tag_scan_page } = require("../data/config"); +const d_storage = require("../helper/d_storage") const { log } = utils @@ -369,6 +370,7 @@ class Searcher { for (const dyid of dyids_set) { if (dyid.length === utils.dyid_length) { + log.info('获取动态', `查看专栏中所提及动态(${dyid}) (${length--})`) const card = await bili.getOneDynamicByDyid(dyid) if (card) { @@ -377,8 +379,12 @@ class Searcher { const parsed_card = parseDynamicCard(card) , { is_liked } = parsed_card; - log.info('获取动态', `查看专栏中所提及动态(${dyid})是否点赞 (${length--})`) - if (is_liked) { + if ( + (!check_if_duplicated + && is_liked) + || (check_if_duplicated === 1 + && await d_storage.searchDyid(dyid)) + ) { log.info('获取动态', `动态(${dyid})已转发过`) _weight += 1; } @@ -445,23 +451,27 @@ class Searcher { if (length) { const lottery_info = raw_lottery_info .reduce(async (pre, cur) => { - const results = await pre + let results = await pre , { dyid } = cur; - const card = await bili.getOneDynamicByDyid(dyid) - - if (card) { - await utils.delay(get_dynamic_detail_wait) - - const { is_liked } = parseDynamicCard(card) - + if (!check_if_duplicated) { log.info('获取动态', `查看动态(${dyid})是否点赞 (${length--})`) - if (is_liked) { - log.info('获取动态', `动态(${dyid})已转发过`) - } else { - cur.is_liked = is_liked - return [...results, cur] + const card = await bili.getOneDynamicByDyid(dyid) + + if (card) { + await utils.delay(get_dynamic_detail_wait) + + const { is_liked } = parseDynamicCard(card) + + if (is_liked) { + log.info('获取动态', `动态(${dyid})已转发过`) + } else { + cur.is_liked = is_liked + results.push(cur) + } } + } else { + results.push(cur) } return results diff --git a/lib/data/config.js b/lib/data/config.js index e9cc233..3b0cc5e 100644 --- a/lib/data/config.js +++ b/lib/data/config.js @@ -69,6 +69,14 @@ const config = { */ chatmodel: '01', + /** + * 检查是否重复转发 + * - 不检查 -1 + * - 通过是否点赞判断 0 + * - 检索本地dyids文件 1 + */ + check_if_duplicated: 0, + /** * - 动态创建时间 * - 多少天前 diff --git a/lib/data/global_var.js b/lib/data/global_var.js index a5e332d..286cb0d 100644 --- a/lib/data/global_var.js +++ b/lib/data/global_var.js @@ -1,4 +1,4 @@ -const { getRemoteConfig } = require("../utils"); +const { getRemoteConfig, createDir, createFile } = require("../utils"); const config = require("../data/config"); const key_map = new Map([['DedeUserID', 'myUID'], ['bili_jct', 'csrf']]); @@ -15,10 +15,11 @@ let global_var = { * 全局变量初始化 * 更新config * @param {string} cookie + * @param {string} num */ - async init(cookie) { + async init(cookie, num) { if (cookie) { - config.updata(process.env.NUMBER); + config.updata(num); this.set('cookie', cookie); @@ -37,6 +38,8 @@ let global_var = { ]); this.set('remoteconfig', await getRemoteConfig()); } + await createDir('dyids'); + await createFile(num < 2 ? 'dyid.txt' : `dyid${num}.txt`, '', 'a') } }; diff --git a/lib/helper/d_storage.js b/lib/helper/d_storage.js new file mode 100644 index 0000000..c57710a --- /dev/null +++ b/lib/helper/d_storage.js @@ -0,0 +1,52 @@ +const { log, readDyidFile, writeDyidFile, dyid_length } = require("../utils"); + +const d_storage = { + /** + * 搜索dyid + * @param {string} dyid + * @returns {Promise} + */ + searchDyid: (dyid) => { + return new Promise((resolve) => { + const Rdyid = new RegExp(dyid); + const rs = readDyidFile(Number(process.env.NUMBER)); + let status = false; + rs.on('data', chunk => { + if (Rdyid.test(chunk)) { + status = true + } + }) + rs.on('end', () => { + resolve(status) + }) + rs.on('error', err => { + log.error('搜索dyid', err) + resolve(status) + }) + }) + }, + /** + * 更新dyid + * @param {string} dyid + */ + updateDyid: (dyid) => { + log.info('更新dyid', `写入${dyid}`); + if (dyid.length !== dyid_length) { + log.error('更新dyid', `dyid(${dyid})长度不为18 若出现此问题请即时通知开发者`) + } + return new Promise((resolve) => { + const ws = writeDyidFile(Number(process.env.NUMBER)); + ws.write(dyid + ',', () => { + ws.destroy(); + resolve() + }) + ws.on('error', err => { + log.error('更新dyid', err) + resolve() + }) + }); + } +} + + +module.exports = d_storage; \ No newline at end of file diff --git a/lib/utils.js b/lib/utils.js index 372a67f..c38dd90 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -13,6 +13,8 @@ const utils = { env_file: path.join(process.cwd(), "env.js"), /**配置文件 */ config_file: path.join(process.cwd(), "my_config.js"), + /**dyids存放目录 */ + dyids_dir: path.join(process.cwd(), "dyids"), /**dyid长度 */ dyid_length: 18, /** @@ -325,6 +327,50 @@ const utils = { resolve() }) }); + }, + /** + * CreateFile + * @param {string} filepath 相对于dyids的文件路径 + * @param {string} [defaultValue] 写入默认值 + * @param {string} flag + * @returns {Promise} + */ + createFile(filepath, defaultValue, flag) { + const fpath = path.join(utils.dyids_dir, filepath); + const buffer = Buffer.from(defaultValue); + return new Promise((resolve, rejects) => { + fs.open(fpath, flag, (err, fd) => { + if (err) { + rejects(err) + } else { + fs.write(fd, buffer, 0, buffer.length, 0, err => { + if (err) { + rejects(err) + } else { + resolve(); + } + }) + } + }) + }); + }, + /** + * 读取dyid文件 + * @param {number} num + * @returns {fs.ReadStream} + */ + readDyidFile(num) { + const fpath = num < 2 ? path.join(utils.dyids_dir, 'dyid.txt') : path.join(utils.dyids_dir, `dyid${num}.txt`); + return fs.createReadStream(fpath, { encoding: 'utf8', highWaterMark: (utils.dyid_length + 1) * 1000 }) + }, + /** + * 追加dyid + * @param {number} num + * @returns {fs.WriteStream} + */ + writeDyidFile(num) { + const fpath = num < 2 ? path.join(utils.dyids_dir, 'dyid.txt') : path.join(utils.dyids_dir, `dyid${num}.txt`); + return fs.createWriteStream(fpath, { flags: 'a' }) } }; diff --git a/main.js b/main.js index 51f91db..86523ca 100644 --- a/main.js +++ b/main.js @@ -45,7 +45,7 @@ async function main() { process.env.ENABLE_MULTIPLE_ACCOUNT = ENABLE_MULTIPLE_ACCOUNT; } else if (COOKIE) { const global_var = require("./lib/data/global_var"); - await global_var.init(COOKIE, Number(NUMBER)); + await global_var.init(COOKIE, NUMBER); /**引入基础功能 */ const { start, isMe, clear, update, checkCookie } = require("./lib/index"); diff --git a/my_config.example.js b/my_config.example.js index 3e53f71..c371dbe 100644 --- a/my_config.example.js +++ b/my_config.example.js @@ -48,7 +48,7 @@ module.exports = Object.freeze({ * API传回数据类型 {RespondBody} * 获取抽奖信息的链接字符串 * @example - * "https://raw.fastgit.org/spiritLHL/sync_lottery/master/archive_datas/datas.json" + * "https://github.com/spiritLHL/sync_lottery" */ APIs: [], @@ -83,6 +83,14 @@ module.exports = Object.freeze({ */ chatmodel: '01', + /** + * 检查是否重复转发 + * - 不检查 -1 + * - 通过是否点赞判断 0 + * - 检索本地dyids文件 1 + */ + check_if_duplicated: 0, + /** * - 动态创建时间 * - 多少天前