insurance-spider/utils.js

import crypto from "crypto";
import axios from "axios";
/**
 * 将时间戳（毫秒）转换为 yyyy-mm-dd 格式的字符串
 * @param {number} timestamp - 毫秒级时间戳
 * @returns {string} yyyy-mm-dd 格式日期
 */
function timestampToDate(timestamp, mode) {
  const date = new Date(timestamp);
  const year = date.getFullYear();
  // 补零
  const month = String(date.getMonth() + 1).padStart(2, "0");
  const day = String(date.getDate()).padStart(2, "0");
  if (!mode) {
    return `${year}-${month}-${day}`;
  } else {
    const hours = String(date.getHours()).padStart(2, "0");
    const minutes = String(date.getMinutes()).padStart(2, "0");
    const seconds = String(date.getSeconds()).padStart(2, "0");
    return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
  }
}

function md5(text, inputEncoding = "utf8", outputEncoding = "hex") {
  return crypto
    .createHash("md5")
    .update(text, inputEncoding)
    .digest(outputEncoding);
}
function getSign(timestamp) {
  let secret = "cpwyyds";
  let uri = "/common/message/push";
  const url = uri + timestamp + secret;
  const myCalc = md5(url);
  let sign =
    myCalc.substring(5, 13) +
    myCalc.substring(29, 31) +
    myCalc.substring(18, 27);
  //sign 转大写
  sign = sign.toUpperCase();
  return sign;
}
// 微信推送
// function wechatPush(spiderName, arr) {
//   for (let item of arr) {
//     let timestamp = new Date().getTime();
//     let sign = getSign(timestamp);
//     let url = "";
//     if (typeof item.urls === "string") {
//       url = item.urls;
//     } else {
//       url = item.urls[0];
//     }
//     let data = {
//       timestamp,
//       sign,
//       templateNo: "A002",
//       url,
//       paramList: [
//         {
//           key: "thing8",
//           value: spiderName,
//         },
//         {
//           key: "thing2",
//           value:
//             item.name.length > 20
//               ? item.name.substring(0, 16) + "..."
//               : item.name,
//         },
//         {
//           key: "time14",
//           value: item.publishTime,
//         },
//         {
//           key: "time17",
//           value: item.endTime,
//         },
//       ],
//     };
//     axios({
//       url: "https://advert.shenlintech.com/platform/common/message/push",
//       method: "post",
//       data,
//     });
//   }
// }
// 废弃
function addToMessageQueue(spiderName, data) {
  const message = {
    id: Date.now() + "-" + Math.random().toString(36).substr(2, 9),
    spiderName,
    data,
    timestamp: new Date().toISOString(),
    status: "pending",
  };
  let queue = [];
  const queueFile = "message_queue.json";
  if (fs.existsSync(queueFile)) {
    queue = JSON.parse(fs.readFileSync(queueFile, "utf-8"));
  }
  // 添加新消息
  queue.push(message);

  fs.writeFileSync(queueFile, JSON.stringify(queue, null, 2));
  console.log(`📤 添加消息到队列: ${spiderName} - ${data.length} 条数据`);
}

async function loopCall(fn, options = {}) {
  let { time, pagenumber, stopWhen, readyForNext, complete, additional } =
    options;
  let shouldContinue = true;
  while (shouldContinue) {
    try {
      let result = await fn(pagenumber, additional);
      // console.log(`页面 ${pagenumber} 处理完成`);

      // 检查停止条件
      if (stopWhen && stopWhen(pagenumber, result)) {
        complete && complete(result);
        shouldContinue = false;
      } else {
        pagenumber = readyForNext(pagenumber, result);
        await new Promise((resolve) => setTimeout(resolve, time));
      }
    } catch (err) {
      console.error("loopCall 出错:", err);
      shouldContinue = false;
    }
  }
}
function keywordsInclude(name) {
  let keywords = [
    "保险",
    "车险",
    "非车险",
    "科技",
    "大模型",
    "承保",
    "第三方平台",
  ];
  return keywords.some((keyword) => name.includes(keyword));
}
// 一汽专用获取公告链接的方法
function getYiqiNoticeUrl(gongGaoType, guid, version, origin) {
  let baseUrl = "https://etp.faw.cn/";
  //是否对参数加密
  var isSecrect = false;

  //候选人公示加密
  if (gongGaoType == 7) {
    isSecrect = true;
  }
  if (isSecrect) {
    var url = baseUrl + "/gg/toGongGaoDetail";
    guid = encodeSixF(guid);
    // var params = {
    //   guid: guid,
    //   gongGaoType: gongGaoType,
    //   version: dealNullAndUndefined(version),
    //   statusCode: 1,
    //   isNew: 1,
    // };
    // try {
    //   await httpPostCurrent(url, params);
    // } catch (err) {
    //   console.log(err);
    //   return "加密链接";
    // }
    return "加密链接，请直接上对应网站查看";
  } else {
    var url =
      baseUrl +
      "/gg/toGongGaoDetail?guid=" +
      guid +
      "&gongGaoType=" +
      gongGaoType +
      "&version=" +
      version +
      "&isNew=1";
    return url;
  }
}
function parseToGgDetailsParams(funcStr) {
  // funcStr = "toGgDetails('6','642ed424-cd9b-4cb0-8b74-9cc868d8f95a:2','2','1','')"

  const match = funcStr.match(/toGgDetails\(([^)]+)\)/);
  if (match) {
    // 解析参数字符串
    const paramsStr = match[1];
    // 简单的参数解析（处理引号包围的参数）
    const params = paramsStr
      .split(",")
      .map((param) => param.trim().replace(/['"]/g, ""));
    return params;
  }
  return null;
}
function encodeSixF(input) {
  var keyStr =
    "ABCDEFGHIJKLMNOP" +
    "QRSTUVWXYZabcdef" +
    "ghijklmnopqrstuv" +
    "wxyz0123456789+/" +
    "=";
  var output = "";
  var chr1,
    chr2,
    chr3 = "";
  var enc1,
    enc2,
    enc3,
    enc4 = "";
  var i = 0;
  do {
    chr1 = input.charCodeAt(i++);
    chr2 = input.charCodeAt(i++);
    chr3 = input.charCodeAt(i++);
    enc1 = chr1 >> 2;
    enc2 = ((chr1 & 3) << 4) | (chr2 >> 4);
    enc3 = ((chr2 & 15) << 2) | (chr3 >> 6);
    enc4 = chr3 & 63;
    if (isNaN(chr2)) {
      enc3 = enc4 = 64;
    } else if (isNaN(chr3)) {
      enc4 = 64;
    }
    output =
      output +
      keyStr.charAt(enc1) +
      keyStr.charAt(enc2) +
      keyStr.charAt(enc3) +
      keyStr.charAt(enc4);
    chr1 = chr2 = chr3 = "";
    enc1 = enc2 = enc3 = enc4 = "";
  } while (i < input.length);

  if (output != null && output.indexOf("=") != -1) {
    var reg = new RegExp("=", "g");
    var outputNew = output.replace(reg, "r1e2p3l4");
    output = outputNew;
  }

  return output + "+*+";
}
function dealNullAndUndefined(value) {
  if (typeof value == "undefined") return "";
  if (value == null) return "";
  if (value == "null") return "";
  if (value == "undefined") return "";
  return value;
}
export {
  timestampToDate,
  loopCall,
  keywordsInclude,
  getYiqiNoticeUrl,
  parseToGgDetailsParams,
  addToMessageQueue,
  md5,
  // wechatPush
};