From 3cce44351f8a4ca5db7de9406c735fdf57f190eb Mon Sep 17 00:00:00 2001 From: huzhengrong Date: Thu, 23 Oct 2025 15:11:29 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat(picc):=20=E5=A2=9E=E5=BC=BA?= =?UTF-8?q?=E7=88=AC=E8=99=AB=E5=8A=9F=E8=83=BD=EF=BC=8C=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E5=85=B3=E9=94=AE=E8=AF=8D=E8=BF=87=E6=BB=A4=EF=BC=8C=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E6=97=A5=E5=BF=97=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- picc.js | 58 +++++++++++++++++++++++--------------------------------- utils.js | 23 +++++++--------------- 2 files changed, 31 insertions(+), 50 deletions(-) diff --git a/picc.js b/picc.js index ddbb4c5..ba748ac 100644 --- a/picc.js +++ b/picc.js @@ -1,14 +1,16 @@ import axios from "axios"; import fs from "fs"; import path from "path"; -import { timestampToDate, loopCall } from "./utils.js"; +import { timestampToDate, loopCall, keywordsInclude } from "./utils.js"; import config from "./config.js"; import { SQLiteMessageQueue } from "./sqlite.js"; class PICC { - constructor() { + constructor(key,categoryId) { + this.key = key; + this.categoryId = categoryId; this.info = []; - console.log("中国人民保险 爬虫启动..."); + console.log(`中国人民保险-${key} 爬虫启动...`); this.queue = new SQLiteMessageQueue(); this.start(); } @@ -17,7 +19,7 @@ class PICC { try { await this.init(); } catch (err) { - console.error("启动失败:", err); + console.error(`启动失败:-${this.key}`, err); } } async init() { @@ -30,7 +32,7 @@ class PICC { } // 全量爬取 async fullFetch() { - console.log("开始全量爬取..."); + console.log(`${this.key}-开始全量爬取...`); try { await loopCall(this.getInfo.bind(this), { time: config.fullFetchTime, @@ -46,7 +48,7 @@ class PICC { }, complete: (result) => { this.info.push(...result.info); - console.log(`爬取完成，共获取 ${this.info.length} 条有效数据`); + console.log(`${this.key}-爬取完成，共获取 ${this.info.length} 条有效数据`); try { if (this.info.length > 0) { this.queue.saveAnnouncements("中国人民保险", this.info); @@ -54,20 +56,20 @@ class PICC { this.queue.addMessage("中国人民保险", this.info); } } catch (error) { - console.error("数据库操作失败:", error); + console.error(`${this.key}-数据库操作失败:`, error); } }, }); } catch (error) { - console.error("全量爬取失败:", error); + console.error(`${this.key}-全量爬取失败:`, error); } - console.log("开始增量爬取..."); + console.log(`开始增量爬取...-${this.key}`); this.increment(); } // 增量爬取 async increment() { - console.log("开始增量爬取模式，每5分钟检查一次新数据..."); + console.log(`${this.key}-开始增量爬取模式，每5分钟检查一次新数据...`); try { await loopCall(this.getInfo.bind(this), { time: config.incrementFetchTime, // 5分钟间隔 @@ -80,7 +82,7 @@ class PICC { ); // 存在新数据 if (newInfo.length > 0) { - console.log(`发现 ${newInfo.length} 条新数据`); + console.log(`${this.key}-发现 ${newInfo.length} 条新数据`); // this.info.push(...newInfo); this.queue.saveAnnouncements("中国人民保险", newInfo); // this.writeFile(this.info); @@ -93,25 +95,25 @@ class PICC { return 1; } } else { - console.log("没有发现新数据，继续监控..."); + console.log(`${this.key}-没有发现新数据，继续监控...`); return 1; // 重新从第一页开始 } } catch (error) { - console.error("数据库操作失败:", error); + console.error(`${this.key}-数据库操作失败:`, error); } }, }); } catch (error) { - console.error("增量爬取失败:", error); + console.error(`${this.key}-增量爬取失败:`, error); } } async getInfo(pagenumber = 1) { let info = []; - console.log(`正在获取第 ${pagenumber} 页数据...`); + console.log(`${this.key}-正在获取第 ${pagenumber} 页数据...`); let result = await this.getList(pagenumber); if (result[0]) { // 出错, 记录错误日志 - console.error("获取页面数据失败:", result[0]); + console.error(`${this.key}-获取页面数据失败:`, result[0]); return { pages: 0, info: [] }; } else { let total = result[1].res.total; @@ -126,7 +128,7 @@ class PICC { ); // 命中关键词 if ( - this.keywordsInclude(item.title) && + keywordsInclude(item.title) && endTime && +new Date(endTime) >= Date.now() ) { @@ -152,7 +154,7 @@ class PICC { url: "https://ec.picc.com/cms/api/dynamicData/queryContentPage", data: { dto:{ - categoryId:"211,213,214,215,216,217", + categoryId:this.categoryId, city:"", county:"", purchaseMode:"", @@ -184,7 +186,7 @@ class PICC { }) .then((res) => { let result = res.data; - console.log("then",JSON.stringify(result.res.rows, null, 2)) + console.log(`${this.key}-then`,JSON.stringify(result.res.rows.map(item=>item.title), null, 2)) if (result.msg === "操作成功" && result.code === 0) { return [null, result]; } else { @@ -192,23 +194,11 @@ class PICC { } }) .catch((err) => { - console.log('catch', err) + console.log(`${this.key}-catch`, err) return [err, null]; }); } - - keywordsInclude(name) { - let keywords = [ - "保险", - "车险", - "非车险", - "科技", - "大模型", - "承保", - "第三方平台", - ]; - return keywords.some((keyword) => name.includes(keyword)); - } } -new PICC(); +new PICC("集中采购","211,213,214,215,216,217"); +new PICC("分散采购","251,253,254,255,256,257"); diff --git a/utils.js b/utils.js index 5223a1c..77d251c 100644 --- a/utils.js +++ b/utils.js @@ -131,22 +131,13 @@ async function loopCall(fn, options = {}) { } function keywordsInclude(name) { let keywords = [ - "海外", - "国际", - "内容", - "营销", - "运营", - "直播", - "品牌", - "事件", - "策略", - "传播", - "执行", - "社媒", - "视频", - "制作", - "拍摄", - "效果", + "保险", + "车险", + "非车险", + "科技", + "大模型", + "承保", + "第三方平台", ]; return keywords.some((keyword) => name.includes(keyword)); }