310 lines
9.3 KiB
JavaScript
310 lines
9.3 KiB
JavaScript
import axios from "axios";
|
|
import fs from "fs";
|
|
import path from "path";
|
|
import JSON5 from "json5";
|
|
import { timestampToDate, loopCall, keywordsInclude } from "./utils.js";
|
|
import config from "./config.js";
|
|
import { SQLiteMessageQueue } from "./sqlite.js";
|
|
import * as cheerio from "cheerio";
|
|
|
|
class Third {
|
|
constructor(jsonMap) {
|
|
this.axiosInstance = axios.create({ timeout: 30000, maxRedirects: 5 });
|
|
this.axiosInstance.interceptors.request.use((config) => {
|
|
// 添加cookie到请求头
|
|
const cookieString = Array.from(this.cookiePair.entries())
|
|
.map(([name, value]) => `${name}=${value}`)
|
|
.join("; ");
|
|
config.headers.Cookie = cookieString;
|
|
// console.log(config);
|
|
return config;
|
|
});
|
|
this.axiosInstance.interceptors.response.use(
|
|
(response) => {
|
|
// 更新cookie到请求头
|
|
let cookieArr = response.headers["set-cookie"] || [];
|
|
this.extractCookie(cookieArr);
|
|
return response;
|
|
},
|
|
(error) => {
|
|
return Promise.reject(error);
|
|
}
|
|
);
|
|
this.cookiePair = new Map();
|
|
// this.csrfToken = "";
|
|
this.jsonMap = jsonMap;
|
|
console.log("三方平台 爬虫启动...");
|
|
this.queue = new SQLiteMessageQueue();
|
|
this.start();
|
|
}
|
|
|
|
async start() {
|
|
try {
|
|
await this.init();
|
|
} catch (err) {
|
|
console.error("启动失败:", err);
|
|
}
|
|
}
|
|
async init() {
|
|
for (let item of this.jsonMap) {
|
|
let announcements = this.queue.getAnnouncementsBySpider(item.name);
|
|
if (announcements.length > 0) {
|
|
this.loopFetchIncrement(item);
|
|
} else {
|
|
this.loopFetchFull(item);
|
|
}
|
|
}
|
|
}
|
|
async initializeCookie() {
|
|
try {
|
|
let headers = {
|
|
headers: {
|
|
Accept: "text/plain, */*; q=0.01",
|
|
"Accept-Language": "zh-CN,zh;q=0.9",
|
|
"Cache-Control": "no-cache",
|
|
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
|
|
Origin: "https://www.chinabidding.com",
|
|
Pragma: "no-cache",
|
|
Priority: "u=1, i",
|
|
Referer: "https://www.chinabidding.com/search/proj.htm",
|
|
"Sec-Ch-Ua":
|
|
'"Not)A;Brand";v="8", "Chromium";v="138", "Google Chrome";v="138"',
|
|
"Sec-Ch-Ua-Mobile": "?0",
|
|
"Sec-Ch-Ua-Platform": '"macOS"',
|
|
"Sec-Fetch-Dest": "empty",
|
|
"Sec-Fetch-Mode": "cors",
|
|
"Sec-Fetch-Site": "same-origin",
|
|
"User-Agent":
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
|
|
"X-Requested-With": "XMLHttpRequest",
|
|
},
|
|
};
|
|
const homeResponse = await this.axiosInstance.get(
|
|
"https://www.chinabidding.com/search/proj.htm",
|
|
headers
|
|
);
|
|
} catch (err) {
|
|
console.log("err", err);
|
|
throw err;
|
|
}
|
|
}
|
|
extractCookie(cookieArr) {
|
|
for (let cookie of cookieArr) {
|
|
let [key, value] = cookie.split(";")[0].split("=");
|
|
this.cookiePair.set(key, value);
|
|
}
|
|
// console.log(this.cookiePair);
|
|
}
|
|
// 全量爬取
|
|
loopFetchFull(props) {
|
|
console.log("开始全量爬取");
|
|
try {
|
|
loopCall(this.getInfo.bind(this), {
|
|
time: config.fullFetchTime,
|
|
pagenumber: 1,
|
|
additional: props.options,
|
|
stopWhen: (pagenumber, result) => {
|
|
return (
|
|
pagenumber >= result.pages || pagenumber >= config.pageNumberLimit
|
|
);
|
|
},
|
|
readyForNext: (pagenumber, result) => {
|
|
props.info.push(...result.info);
|
|
return pagenumber + 1;
|
|
},
|
|
complete: (result) => {
|
|
props.info.push(...result.info);
|
|
console.log(`爬取完成，共获取 ${props.info.length} 条有效数据`);
|
|
try {
|
|
if (props.info.length > 0) {
|
|
this.queue.saveAnnouncements(props.name, props.info);
|
|
this.queue.addMessage(props.name, props.info);
|
|
}
|
|
} catch (error) {
|
|
console.error("数据库操作失败:", error);
|
|
}
|
|
this.loopFetchIncrement(props);
|
|
},
|
|
});
|
|
} catch (error) {
|
|
console.error(`${props.options.name}全量爬取失败:`, error);
|
|
}
|
|
}
|
|
loopFetchIncrement(props) {
|
|
console.log("开始增量爬取");
|
|
try {
|
|
loopCall(this.getInfo.bind(this), {
|
|
time: config.incrementFetchTime, // 5分钟间隔
|
|
pagenumber: 1,
|
|
additional: props.options,
|
|
readyForNext: (pagenumber, result) => {
|
|
try {
|
|
let newInfo = this.queue.filterNewAnnouncements(
|
|
props.name,
|
|
result.info
|
|
);
|
|
// 存在新数据
|
|
if (newInfo.length > 0) {
|
|
console.log(`发现 ${newInfo.length} 条新数据`);
|
|
// props.info.push(...newInfo);
|
|
this.queue.saveAnnouncements(props.name, newInfo);
|
|
// this.writeFile(props);
|
|
this.queue.addMessage(props.name, newInfo);
|
|
// 全是新数据，继续下一页
|
|
if (newInfo.length === result.info.length) {
|
|
return pagenumber + 1;
|
|
} else {
|
|
// 有部分重复数据，重新从第一页开始
|
|
return 1;
|
|
}
|
|
} else {
|
|
console.log("没有发现新数据，继续监控...");
|
|
return 1; // 重新从第一页开始
|
|
}
|
|
} catch (error) {
|
|
console.error("数据库操作失败:", error);
|
|
}
|
|
},
|
|
});
|
|
} catch (error) {
|
|
console.error(`${props.options.name}增量爬取失败:`, error);
|
|
}
|
|
}
|
|
|
|
async getNoticeDetail(url) {
|
|
try {
|
|
let result = await axios.get(url);
|
|
return result.data;
|
|
} catch (err) {
|
|
return "err";
|
|
}
|
|
}
|
|
async getInfo(pagenumber = 1, config) {
|
|
let info = [];
|
|
console.log(`${config.name}--获取第 ${pagenumber} 页数据...`);
|
|
let result = await this.getList(pagenumber, config);
|
|
if (result[0]) {
|
|
// 出错, 记录错误日志
|
|
console.error("获取页面数据失败: ", result[0]);
|
|
return { pages: 0, info: [] };
|
|
} else {
|
|
let pages = 3;
|
|
let html = result[1];
|
|
const $ = cheerio.load(html);
|
|
$(".as-pager-body li").each((index, element) => {
|
|
let idmatch = $(element)
|
|
.find(".as-pager-item")
|
|
.attr("href")
|
|
.match(/\/bidDetail\/(\d+)\.html/);
|
|
let id = idmatch ? idmatch[1] : "";
|
|
let name = $(element).find(".txt").attr("title");
|
|
|
|
let url = $(element).find(".as-pager-item").attr("href");
|
|
if (keywordsInclude(name)) {
|
|
console.log("处理项目：", name);
|
|
info.push({
|
|
id: id,
|
|
name: name,
|
|
urls: url,
|
|
publishTime: "--",
|
|
endTime: "--",
|
|
});
|
|
}
|
|
});
|
|
return { pages, info };
|
|
}
|
|
}
|
|
async getList(pagenumber, config) {
|
|
let data = config.data;
|
|
data.currentPage = pagenumber;
|
|
let headers = {
|
|
Accept: "text/plain, */*; q=0.01",
|
|
"Accept-Language": "zh-CN,zh;q=0.9",
|
|
"Cache-Control": "no-cache",
|
|
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
|
|
Origin: "https://www.chinabidding.com",
|
|
Pragma: "no-cache",
|
|
Priority: "u=1, i",
|
|
Referer: "https://www.chinabidding.com/search/proj.htm",
|
|
"Sec-Ch-Ua":
|
|
'"Not)A;Brand";v="8", "Chromium";v="138", "Google Chrome";v="138"',
|
|
"Sec-Ch-Ua-Mobile": "?0",
|
|
"Sec-Ch-Ua-Platform": '"macOS"',
|
|
"Sec-Fetch-Dest": "empty",
|
|
"Sec-Fetch-Mode": "cors",
|
|
"Sec-Fetch-Site": "same-origin",
|
|
"User-Agent":
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36",
|
|
"X-Requested-With": "XMLHttpRequest",
|
|
};
|
|
try {
|
|
const response = await this.axiosInstance({
|
|
url: config.url,
|
|
data,
|
|
method: "post",
|
|
headers,
|
|
});
|
|
let result = response.data;
|
|
return [null, result];
|
|
} catch (err) {
|
|
console.log("cookie不对");
|
|
try {
|
|
await this.initializeCookie();
|
|
const retryResponse = await this.axiosInstance({
|
|
url: config.url,
|
|
data,
|
|
method: "post",
|
|
headers,
|
|
});
|
|
let result = retryResponse.data;
|
|
return [null, result];
|
|
} catch (retryErr) {
|
|
return [retryErr, null];
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
new Third([
|
|
{
|
|
name: "机电项目招投标【招标公告】",
|
|
info: [],
|
|
options: {
|
|
name: "机电项目招投标【招标公告】",
|
|
url: "https://www.chinabidding.com/search/proj.htm",
|
|
data: {
|
|
fullText: "",
|
|
pubDate: "",
|
|
infoClassCodes: "(0105 0103)",
|
|
normIndustry: "",
|
|
zoneCode: "",
|
|
fundSourceCodes: "",
|
|
poClass: "BidNotice",
|
|
rangeType: "",
|
|
currentPage: 1,
|
|
},
|
|
},
|
|
},
|
|
]);
|
|
new Third([
|
|
{
|
|
name: "机电项目招投标【招标变更公告】",
|
|
info: [],
|
|
options: {
|
|
name: "机电项目招投标【招标变更公告】",
|
|
url: "https://www.chinabidding.com/search/proj.htm",
|
|
data: {
|
|
fullText: "",
|
|
pubDate: "",
|
|
infoClassCodes: "(0106 0104)",
|
|
normIndustry: "",
|
|
zoneCode: "",
|
|
fundSourceCodes: "",
|
|
poClass: "BidNotice",
|
|
rangeType: "",
|
|
currentPage: 1,
|
|
},
|
|
},
|
|
},
|
|
]);
|