🐛 feat(qhr-spider): 添加前海再保险爬虫及相关逻辑
This commit is contained in:
parent
ad69302e17
commit
c35ab2bc3c
|
|
@ -113,5 +113,21 @@ module.exports = {
|
|||
log_file: "./logs/sinosafe-combined.log",
|
||||
time: true,
|
||||
},
|
||||
{//前海再保险爬虫
|
||||
name: "qhr-spider",
|
||||
script: "./service/qhr.js",
|
||||
instances: 1,
|
||||
autorestart: true,
|
||||
watch: false,
|
||||
max_memory_restart: "300M",
|
||||
env: {
|
||||
NODE_ENV: "production",
|
||||
SPIDER_NAME: "qhr",
|
||||
},
|
||||
error_file: "./logs/qhr-error.log",
|
||||
out_file: "./logs/qhr-out.log",
|
||||
log_file: "./logs/qhr-combined.log",
|
||||
time: true,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,180 @@
|
|||
//前海再保险
|
||||
import axios from "axios";
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { timestampToDate, loopCall, keywordsInclude, extractDateTimeAndTitle } from "../utils.js";
|
||||
import config from "../config.js";
|
||||
import { SQLiteMessageQueue } from "../sqlite.js";
|
||||
import * as cheerio from "cheerio";
|
||||
|
||||
class QHR {
|
||||
constructor() {
|
||||
this.jsonMap = [
|
||||
{
|
||||
name: "前海再保险",
|
||||
info: [],
|
||||
options: {
|
||||
name: "前海再保险",
|
||||
url: "https://www.qianhaire.com/",
|
||||
homeIndex: "company-info-10.html",
|
||||
},
|
||||
},
|
||||
];
|
||||
console.log("前海再保险 爬虫启动...");
|
||||
this.queue = new SQLiteMessageQueue();
|
||||
this.start();
|
||||
}
|
||||
|
||||
async start() {
|
||||
try {
|
||||
await this.init();
|
||||
} catch (err) {
|
||||
console.error("启动失败:", err);
|
||||
}
|
||||
}
|
||||
async init() {
|
||||
for (let item of this.jsonMap) {
|
||||
let announcements = this.queue.getAnnouncementsBySpider(item.name);
|
||||
if (announcements.length > 0) {
|
||||
this.loopFetchIncrement(item);
|
||||
} else {
|
||||
this.loopFetchFull(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 全量爬取
|
||||
loopFetchFull(props) {
|
||||
try {
|
||||
loopCall(this.getInfo.bind(this), {
|
||||
time: config.fullFetchTime,
|
||||
pagenumber: 1,
|
||||
additional: props.options,
|
||||
stopWhen: (pagenumber, result) => {
|
||||
//只有一页
|
||||
return (
|
||||
pagenumber >1
|
||||
);
|
||||
},
|
||||
readyForNext: (pagenumber, result) => {
|
||||
props.info.push(...result.info);
|
||||
return pagenumber + 1;
|
||||
},
|
||||
complete: (result) => {
|
||||
props.info.push(...result.info);
|
||||
console.log(`爬取完成，共获取 ${props.info.length} 条有效数据`);
|
||||
try {
|
||||
if (props.info.length > 0) {
|
||||
this.queue.saveAnnouncements(props.name, props.info);
|
||||
// this.writeFile(props);
|
||||
this.queue.addMessage(props.name, props.info);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("数据库操作失败:", error);
|
||||
}
|
||||
this.loopFetchIncrement(props);
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(`${props.options.name}全量爬取失败:`, error);
|
||||
}
|
||||
}
|
||||
loopFetchIncrement(props) {
|
||||
try {
|
||||
loopCall(this.getInfo.bind(this), {
|
||||
time: config.incrementFetchTime, // 5分钟间隔
|
||||
pagenumber: 1,
|
||||
additional: props.options,
|
||||
readyForNext: (pagenumber, result) => {
|
||||
try {
|
||||
let newInfo = this.queue.filterNewAnnouncements(
|
||||
props.name,
|
||||
result.info
|
||||
);
|
||||
// 存在新数据
|
||||
if (newInfo.length > 0) {
|
||||
console.log(`发现 ${newInfo.length} 条新数据`);
|
||||
// props.info.push(...newInfo);
|
||||
this.queue.saveAnnouncements(props.name, newInfo);
|
||||
// this.writeFile(props);
|
||||
this.queue.addMessage(props.name, newInfo);
|
||||
// 全是新数据，继续下一页
|
||||
if (newInfo.length === result.info.length) {
|
||||
return pagenumber + 1;
|
||||
} else {
|
||||
// 有部分重复数据，重新从第一页开始
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
console.log("没有发现新数据，继续监控...");
|
||||
return 1; // 重新从第一页开始
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("数据库操作失败:", error);
|
||||
}
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error(`${props.options.name}增量爬取失败:`, error);
|
||||
}
|
||||
}
|
||||
async getInfo(pagenumber = 1, config) {
|
||||
let info = [];
|
||||
console.log(`${config.name}--获取第 ${pagenumber} 页数据...`);
|
||||
let result = await this.getList(config);
|
||||
if (result[0]) {
|
||||
// 出错, 记录错误日志
|
||||
console.error("获取页面数据失败:", result[0].status);
|
||||
return { pages: 0, info: [] };
|
||||
} else {
|
||||
// 列表没有结束时间字段，默认5页
|
||||
let pages = 5;
|
||||
let html = result[1];
|
||||
const $ = cheerio.load(html);
|
||||
$(".uk-container .uk-card").each((index, element) => {
|
||||
let id = $(element).find("a").attr("href");
|
||||
let processedText = extractDateTimeAndTitle($(element).find("a").text());
|
||||
let name = processedText.title;
|
||||
let publishTime = processedText.date;
|
||||
let endTime = null;
|
||||
let urls = 'https://www.qianhaire.com/'+$(element).find("a").attr("href");
|
||||
if (
|
||||
// endTime &&
|
||||
// +new Date(endTime) >= Date.now() &&
|
||||
keywordsInclude(name)
|
||||
) {
|
||||
console.log("处理项目:", id, name);
|
||||
info.push({
|
||||
id: id,
|
||||
name: name,
|
||||
publishTime: publishTime,
|
||||
endTime: endTime,
|
||||
urls: urls,
|
||||
});
|
||||
}
|
||||
});
|
||||
return { pages, info };
|
||||
}
|
||||
}
|
||||
// 分页获取数据
|
||||
getList(config) {
|
||||
let url = config.url;
|
||||
// if (pagenumber === 1) {
|
||||
url += config.homeIndex;
|
||||
// } else {
|
||||
// url += `index_${pagenumber}.html`;
|
||||
// }
|
||||
return axios({
|
||||
url: url,
|
||||
method: "get",
|
||||
})
|
||||
.then((res) => {
|
||||
let result = res.data;
|
||||
return [null, result];
|
||||
})
|
||||
.catch((err) => {
|
||||
return [err, null];
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
new QHR();
|
||||
26
utils.js
26
utils.js
|
|
@ -271,6 +271,29 @@ async function sendQYWechatMessage(message) {
|
|||
throw error; // 重新抛出错误以便调用方可以处理
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从字符串中提取日期和标题
|
||||
* @param {string} input - 输入字符串，格式如"2025.05.27 万得软件及数据服务项目单一来源采购结果公示"
|
||||
* @returns {object|null} - 包含date和title属性的对象，如果不符合格式则返回null
|
||||
*/
|
||||
function extractDateTimeAndTitle(input) {
|
||||
if (!input) return null;
|
||||
|
||||
// 匹配日期格式 YYYY.MM.DD 或 YYYY-MM-DD
|
||||
const dateRegex = /^(\d{4}[.\-]\d{2}[.\-]\d{2})\s+(.+)$/;
|
||||
const match = input.match(dateRegex);
|
||||
|
||||
if (match) {
|
||||
return {
|
||||
date: match[1],
|
||||
title: match[2].trim()
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export {
|
||||
timestampToDate,
|
||||
loopCall,
|
||||
|
|
@ -279,6 +302,7 @@ export {
|
|||
parseToGgDetailsParams,
|
||||
addToMessageQueue,
|
||||
md5,
|
||||
sendQYWechatMessage
|
||||
sendQYWechatMessage,
|
||||
extractDateTimeAndTitle
|
||||
// wechatPush
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in New Issue