Update public-api.js via upload script - 2026-11-10 17:25:49

This commit is contained in:
mike 2026-11-10 17:26:02 +08:00
parent 8569235173
commit 27f9bfc72a
1 changed files with 272 additions and 3 deletions

View File

@ -644,6 +644,188 @@ const brochureOf = (p, lang) => {
return b && b.fileName && /\.(html?|pdf)$/i.test(b.fileName) ? b : null; return b && b.fileName && /\.(html?|pdf)$/i.test(b.fileName) ? b : null;
}; };
// —— 微信公众号文章导入(网站管理 → 近期动态 → 从链接导入)——
// 服务器抓取 mp.weixin.qq.com 文章页 → 取标题、公众号、发布时间、封面、正文(文字 + 图片 + 视频)→ 下载图片到 uploads/<新闻id>/
// (微信图片有防盗链,网站不能直接引用)→ 返回一篇新闻草稿,后台可以再改排版、日期时间、分类。
// 不依赖第三方库:用一个很小的 HTML 解析器把正文转成"块":p 段落 / h 小标题 / quote 引言 / img 单图 / pair 两图 / gallery 图集 / yt 视频。
const WX_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36';
const VOID_TAGS = new Set(['img', 'br', 'hr', 'input', 'meta', 'link', 'source', 'wbr', 'col', 'area', 'embed', 'param', 'track']);
const BLOCK_TAGS = new Set(['p', 'section', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'ul', 'ol', 'blockquote', 'figure', 'figcaption', 'table', 'tr', 'pre', 'center']);
const decodeEnt = (s) => String(s || '')
.replace(/&nbsp;|&#160;/g, ' ').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"').replace(/&#39;|&apos;/g, "'")
.replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16))).replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(+d)).replace(/&amp;/g, '&');
const attrsOf = (s) => { const a = {}; String(s || '').replace(/([\w:-]+)\s*(?:=\s*("([^"]*)"|'([^']*)'|([^\s>]+)))?/g, (_, k, __, v1, v2, v3) => { a[k.toLowerCase()] = decodeEnt(v1 ?? v2 ?? v3 ?? ''); }); return a; };
// 极简 HTML → 树(容错:未闭合的标签自动收尾)
function parseHtml(html) {
const root = { tag: '#root', attrs: {}, kids: [] };
const stack = [root];
const re = /<!--[\s\S]*?-->|<(script|style)\b[^>]*>[\s\S]*?<\/\1>|<\/([a-zA-Z][\w-]*)\s*>|<([a-zA-Z][\w-]*)([^>]*?)(\/?)>|([^<]+)/g;
let m;
while ((m = re.exec(html))) {
const top = stack[stack.length - 1];
if (m[2]) { // 结束标签
const t = m[2].toLowerCase();
for (let i = stack.length - 1; i > 0; i--) if (stack[i].tag === t) { stack.length = i; break; }
} else if (m[3]) {
const t = m[3].toLowerCase();
const node = { tag: t, attrs: attrsOf(m[4]), kids: [] };
top.kids.push(node);
if (!VOID_TAGS.has(t) && !m[5]) stack.push(node);
} else if (m[6]) top.kids.push({ tag: '#text', text: decodeEnt(m[6]) });
}
return root;
}
// 从整页里切出 id="js_content" 的那个 div(按 div 嵌套层数配对)
function sliceById(html, id) {
const start = html.search(new RegExp(`<div[^>]*\\bid=["']${id}["'][^>]*>`, 'i'));
if (start < 0) return '';
const open = html.indexOf('>', start) + 1;
const re = /<div\b[^>]*>|<\/div\s*>/gi; re.lastIndex = open;
let depth = 1, m;
while ((m = re.exec(html))) { depth += m[0][1] === '/' ? -1 : 1; if (depth === 0) return html.slice(open, m.index); }
return html.slice(open);
}
const jsVar = (html, name) => { const m = html.match(new RegExp(`var\\s+${name}\\s*=\\s*(?:htmlDecode\\()?["']([^"']*)["']`)); return m ? decodeEnt(m[1]) : ''; };
const metaOf = (html, prop) => { const m = html.match(new RegExp(`<meta[^>]+(?:property|name)=["']${prop}["'][^>]+content=["']([^"']*)["']`, 'i')) || html.match(new RegExp(`<meta[^>]+content=["']([^"']*)["'][^>]+(?:property|name)=["']${prop}["']`, 'i')); return m ? decodeEnt(m[1]) : ''; };
// 正文树 → 块。文字按块级元素分段;图片单独成块;连续图片合并成两图 / 图集
const NOISE = /^(点击|长按|扫码|识别二维码|关注我们|欢迎关注|点个在看|阅读原文|▼|▲|↓|END$|end$)|^[—\-·•\s]+$|^(公众号|微信号|往期推荐|往期回顾|推荐阅读|分享|收藏|点赞|在看)[::]?$|关注我们$|二维码/;
function toBlocks(root) {
const out = [];
let buf = '', bold = 0, all = 0, inQuote = false, center = false;
const flush = (forceTag) => {
const text = buf.replace(/[ \t\r\f\v]+/g, ' ').replace(/\s*\n\s*/g, '\n').trim();
if (text && !NOISE.test(text)) {
const isHead = forceTag === 'h' || (all > 0 && bold / all > 0.85 && text.length <= 40 && !/[。!?.!?]$/.test(text));
out.push({ t: forceTag === 'quote' || inQuote ? 'quote' : isHead ? 'h' : 'p', zh: text, ...(center && text.length <= 50 ? { center: true } : {}) });
}
buf = ''; bold = 0; all = 0; center = false;
};
const walk = (n, b) => {
if (n.tag === '#text') { const t = n.text; if (t.trim()) { buf += t; all += t.trim().length; if (b) bold += t.trim().length; } else if (t) buf += ' '; return; }
const t = n.tag, a = n.attrs || {}, style = String(a.style || '');
if (/display\s*:\s*none/.test(style)) return;
if (t === 'img') {
const src = a['data-src'] || a.src || '';
if (/^https?:\/\//.test(src) && !/res\.wx\.qq\.com|mmbiz\.qlogo|\/emoji\//.test(src)) {
const w = +(a['data-w'] || 0), ratio = +(a['data-ratio'] || 0);
if (!(w && w < 120) && !(ratio && ratio < 0.12)) { flush(); out.push({ t: 'img', url: src, ratio }); }
}
return;
}
if (t === 'iframe' || t === 'mpvideosnap' || t === 'mp-common-videosnap' || t === 'video') {
const src = a['data-src'] || a.src || '';
const yt = src.match(/(?:youtu\.be\/|v=|embed\/)([\w-]{11})/);
flush();
out.push(yt ? { t: 'yt', url: `https://www.youtube.com/watch?v=${yt[1]}` } : { t: 'video', url: src, note: '微信视频无法搬到网站,请上传到 YouTube 后把链接贴到这里' });
return;
}
if (t === 'br') { buf += '\n'; return; }
const isBlock = BLOCK_TAGS.has(t);
const head = /^h[1-4]$/.test(t);
if (isBlock) flush();
if (t === 'blockquote') inQuote = true;
if (/text-align\s*:\s*center/.test(style)) center = true;
const nb = b || t === 'strong' || t === 'b' || /font-weight\s*:\s*(bold|[6-9]00)/.test(style);
for (const k of n.kids || []) walk(k, nb);
if (isBlock) flush(head ? 'h' : undefined);
if (t === 'blockquote') inQuote = false;
};
walk(root, false);
// 段落里的换行:短行(像列表)保留成多段
const blocks = [];
for (const b of out) {
if (b.t === 'p' && b.zh.includes('\n')) b.zh.split('\n').map((s) => s.trim()).filter(Boolean).forEach((s) => blocks.push({ t: 'p', zh: s }));
else blocks.push(b);
}
// 合并连续图片:2 张 → pair,3 张以上 → gallery
const merged = [];
for (const b of blocks) {
const last = merged[merged.length - 1];
if (b.t === 'img' && last && (last.t === 'img' || last.t === 'pair' || last.t === 'gallery')) {
const urls = last.t === 'img' ? [last.url] : last.urls;
merged[merged.length - 1] = { t: urls.length + 1 === 2 ? 'pair' : 'gallery', urls: [...urls, b.url] };
} else merged.push(b);
}
// 去掉重复段落(公众号常见:标题在正文里又出现一次)
return merged.filter((b, i) => !(b.zh && merged.findIndex((x) => x.zh === b.zh) !== i));
}
function parseWechat(html, url) {
const title = (jsVar(html, 'msg_title') || metaOf(html, 'og:title') || (html.match(/id=["']activity-name["'][^>]*>([\s\S]*?)<\//) || [])[1] || '').replace(/<[^>]+>/g, '').trim();
const account = (jsVar(html, 'nickname') || (html.match(/id=["']js_name["'][^>]*>([\s\S]*?)<\//) || [])[1] || '').replace(/<[^>]+>/g, '').trim();
const ts = +(jsVar(html, 'ct') || (html.match(/create_time\s*[:=]\s*["']?(\d{10})/) || [])[1] || 0);
// 发布时间按北京时间显示:YYYY-MM-DDTHH:mm
const ctStr = (html.match(/createTime\s*=\s*['"](\d{4}-\d{2}-\d{2} \d{2}:\d{2})/) || [])[1];
const datetime = ts ? new Date((ts + 8 * 3600) * 1000).toISOString().slice(0, 16) : ctStr ? ctStr.replace(' ', 'T') : '';
const cover = jsVar(html, 'msg_cdn_url') || metaOf(html, 'og:image');
const desc = jsVar(html, 'msg_desc') || metaOf(html, 'og:description') || metaOf(html, 'description');
const body = sliceById(html, 'js_content');
const blocks = body ? toBlocks(parseHtml(body)) : [];
return { title: decodeEnt(title), account: decodeEnt(account), datetime, cover, desc: decodeEnt(desc), blocks, srcUrl: url };
}
async function fetchWechatHtml(url) {
const r = await fetch(url, { headers: { 'User-Agent': WX_UA, 'Accept-Language': 'zh-CN,zh;q=0.9' }, redirect: 'follow' });
if (!r.ok) throw new Error(`打开文章失败(HTTP ${r.status})`);
const html = await r.text();
if (/环境异常|完成验证后即可继续访问|verify/.test(html) && !/js_content/.test(html)) throw new Error('微信要求验证,暂时无法抓取。请稍后再试,或换一篇文章链接');
if (!/js_content/.test(html)) throw new Error('没有找到文章正文。请确认是公众号文章链接(mp.weixin.qq.com/s/…),且文章没有被删除');
return html;
}
const WX_EXT = { jpeg: 'jpg', jpg: 'jpg', png: 'png', gif: 'gif', webp: 'webp' };
async function downloadWxImage(url, dir, name) {
const r = await fetch(url, { headers: { 'User-Agent': WX_UA, Referer: 'https://mp.weixin.qq.com/' } });
if (!r.ok) throw new Error(`HTTP ${r.status}`);
const buf = Buffer.from(await r.arrayBuffer());
if (buf.length < 200) throw new Error('empty');
const fmt = (url.match(/wx_fmt=(\w+)/) || [])[1] || (r.headers.get('content-type') || '').split('/')[1] || 'jpg';
const file = `${name}.${WX_EXT[String(fmt).toLowerCase()] || 'jpg'}`;
fs.writeFileSync(path.join(dir, file), buf);
return file;
}
// —— 新闻文章(kind:'article')的发布输出 ——
// 文字按"原文"存(导入时是中文 zh);手动填了 en/es/ru 的优先,没填的自动翻译。中文 → 英/西/俄;英文原文 → 西/俄。
async function translateAny(ai, strings) {
const out = { en: {}, es: {}, ru: {} };
const uniq = [...new Set(strings.map((x) => String(x || '').trim()).filter(Boolean))];
const zh = uniq.filter(hasZh), en = uniq.filter((t) => !hasZh(t));
const cache = loadCache();
let dirty = false;
if (ai && zh.length) {
for (const [lang, desc] of Object.entries(ZH_TARGETS)) {
const list = zh.filter((t) => !cache[cacheKey('zh>' + lang, t)]);
// 按字数分批:长段落多的文章不会超出单次回复长度
let chunk = [], size = 0;
const run = async () => {
if (!chunk.length) return;
try {
const res = await translateBatch(ai, chunk, desc, 'Simplified Chinese (company news article; keep a factual press-release tone)');
chunk.forEach((src, j) => { cache[cacheKey('zh>' + lang, src)] = res[j]; });
dirty = true;
} catch (e) { console.warn(`[public-api] 新闻翻译成 ${lang} 失败:${e.message}`); }
chunk = []; size = 0;
};
for (const t of list) { if (chunk.length && (size + t.length > 1200 || chunk.length >= 12)) await run(); chunk.push(t); size += t.length; }
await run();
}
}
if (dirty) { try { saveCache(cache); } catch (e) {} }
for (const t of zh) for (const lang of Object.keys(ZH_TARGETS)) { const v = cache[cacheKey('zh>' + lang, t)]; if (v) out[lang][t] = v; }
const trEn = await translateStrings(ai, en);
for (const t of en) { out.en[t] = t; for (const lang of Object.keys(WEB_TRANSLATE_LANGS)) if (trEn[lang][t]) out[lang][t] = trEn[lang][t]; }
return out;
}
const NEWS_CATS = ['projects', 'products', 'events', 'company'];
const newsSlugify = (s) => String(s || '').toLowerCase().normalize('NFKD').replace(/[^\w\s-]/g, '').trim().replace(/[\s_]+/g, '-').replace(/-+/g, '-').slice(0, 70).replace(/-$/, '');
const srcText = (o) => String((o && (o.zh || o.en)) || '').trim();
const articleTexts = (n) => [srcText(n.title), srcText(n.excerpt), srcText(n.place),
...(Array.isArray(n.blocks) ? n.blocks : []).flatMap((b) => [srcText(b), srcText(b.cap), ...(Array.isArray(b.caps) ? b.caps.map(srcText) : [])])];
const articleFiles = (n) => [n.cover, ...(Array.isArray(n.blocks) ? n.blocks : []).flatMap((b) => (b.file ? [b.file] : Array.isArray(b.files) ? b.files : []))].filter(Boolean).map(String);
function checkKey(req, res, next) { function checkKey(req, res, next) {
const want = conf('SITE_BUILD_KEY'); const want = conf('SITE_BUILD_KEY');
const got = req.get('x-site-key') || ''; const got = req.get('x-site-key') || '';
@ -750,7 +932,7 @@ module.exports = function mountPublicApi(app, { dataFile, uploadsDir, requireAut
try { try {
const base = conf('PUBLIC_BASE_URL'); const base = conf('PUBLIC_BASE_URL');
const items = siteAssetItems(dataFile); const items = siteAssetItems(dataFile);
const rawNews = readKey(dataFile, SITE_NEWS_KEY).filter((n) => n && n.title && String(n.title.en || '').trim()); const rawNews = readKey(dataFile, SITE_NEWS_KEY).filter((n) => n && n.kind !== 'article' && n.title && String(n.title.en || '').trim());
const captionsEn = Object.values(items).flat().map((x) => String((x.caption && x.caption.en) || '').trim()); const captionsEn = Object.values(items).flat().map((x) => String((x.caption && x.caption.en) || '').trim());
const tr = await translateStrings(ai, [...rawNews.flatMap((n) => [n.title.en, (n.text && n.text.en) || '']), ...captionsEn]); const tr = await translateStrings(ai, [...rawNews.flatMap((n) => [n.title.en, (n.text && n.text.en) || '']), ...captionsEn]);
const i18n = (o) => { const i18n = (o) => {
@ -765,12 +947,99 @@ module.exports = function mountPublicApi(app, { dataFile, uploadsDir, requireAut
assets[k] = list.map((x) => ({ url: `${base}/api/public/site-media/${encodeURIComponent(x.file)}`, caption: i18n(x.caption) })); assets[k] = list.map((x) => ({ url: `${base}/api/public/site-media/${encodeURIComponent(x.file)}`, caption: i18n(x.caption) }));
} }
const news = rawNews const news = rawNews
.map((n) => ({ date: String(n.date || '').slice(0, 10), title: i18n(n.title), text: i18n(n.text), youtube: youtubeId(n.youtubeUrl) || undefined })) .map((n) => ({ date: String(n.date || '').slice(0, 10), title: i18n(n.title), text: i18n(n.text), youtube: youtubeId(n.youtubeUrl) || undefined }));
.sort((a, b) => (a.date < b.date ? 1 : a.date > b.date ? -1 : 0)); // 新版图文文章(含从公众号导入的)
const arts = readKey(dataFile, SITE_NEWS_KEY).filter((n) => n && n.kind === 'article' && n.published !== false && srcText(n.title));
const at = await translateAny(ai, arts.flatMap(articleTexts));
const L = (o) => {
const src = srcText(o); const r = {};
for (const lang of Object.keys(ZH_TARGETS)) r[lang] = String((o && lang !== 'zh' && o[lang]) || '').trim() || (src && at[lang][src]) || (hasZh(src) ? '' : src);
return r;
};
const usedSlugs = new Set();
for (const n of arts) {
const dt = String(n.datetime || n.date || '').slice(0, 16);
const title = L(n.title);
let slug = newsSlugify(n.slug) || [dt.slice(0, 10), newsSlugify(title.en)].filter(Boolean).join('-') || `news-${n.id}`;
while (usedSlugs.has(slug)) slug += '-2';
usedSlugs.add(slug);
const media = (f) => `${base}/api/public/news-media/${encodeURIComponent(n.id)}/${encodeURIComponent(f)}`;
const blocks = [];
for (const b of Array.isArray(n.blocks) ? n.blocks : []) {
if (!b || !b.t) continue;
if (b.t === 'p' || b.t === 'h' || b.t === 'quote') { const text = L(b); if (text.en) blocks.push({ t: b.t, text, ...(b.lead ? { lead: true } : {}) }); }
else if (b.t === 'img' && b.file) blocks.push({ t: 'img', src: media(b.file), cap: srcText(b.cap) ? L(b.cap) : undefined, fit: b.fit || undefined });
else if ((b.t === 'pair' || b.t === 'gallery') && Array.isArray(b.files) && b.files.length) {
const items = b.files.map((f, i) => ({ src: media(f), cap: b.caps && srcText(b.caps[i]) ? L(b.caps[i]) : undefined }));
blocks.push({ t: items.length === 1 ? 'img' : items.length === 2 && b.t === 'pair' ? 'pair' : 'gallery', ...(items.length === 1 ? items[0] : { items }) });
}
else if (b.t === 'yt' && youtubeId(b.url)) blocks.push({ t: 'yt', id: youtubeId(b.url), cap: srcText(b.cap) ? L(b.cap) : undefined });
else if (b.t === 'links' && Array.isArray(b.items) && b.items.length) blocks.push({ t: 'links', items: b.items.map(String).filter((x) => /^[\w/-]+$/.test(x)) });
}
const firstImg = articleFiles(n)[0];
news.push({
kind: 'article', slug, date: dt.slice(0, 10), datetime: dt, cat: NEWS_CATS.includes(n.cat) ? n.cat : 'company',
place: srcText(n.place) ? L(n.place) : undefined, title, excerpt: srcText(n.excerpt) ? L(n.excerpt) : undefined,
cover: n.cover || firstImg ? media(n.cover || firstImg) : undefined, coverFit: n.coverFit || undefined, blocks,
});
}
news.sort((a, b) => ((a.datetime || a.date) < (b.datetime || b.date) ? 1 : (a.datetime || a.date) > (b.datetime || b.date) ? -1 : 0));
res.json({ assets, news }); res.json({ assets, news });
} catch (e) { res.status(500).json({ error: e.message }); } } catch (e) { res.status(500).json({ error: e.message }); }
}); });
// 新闻文章图片:只允许读取"已发布文章里用到的"文件(存放在 uploads/<文章id>/)
app.get('/api/public/news-media/:nid/:file', checkKey, (req, res) => {
const nid = String(req.params.nid), file = path.basename(req.params.file);
if (!/^[A-Za-z0-9_-]+$/.test(nid)) return res.status(404).end();
const n = readKey(dataFile, SITE_NEWS_KEY).find((x) => x && x.kind === 'article' && x.id === nid && x.published !== false);
if (!n || !articleFiles(n).includes(file)) return res.status(404).end();
res.sendFile(path.resolve(uploadsDir, nid, file), (err) => { if (err) res.status(404).end(); });
});
// 从公众号链接导入:抓文章 → 下载图片到 uploads/<新文章id>/ → 返回草稿(后台再保存)
app.post('/api/site/news/import', requireAuth, async (req, res) => {
try {
const url = String((req.body && req.body.url) || '').trim();
if (!/^https?:\/\/mp\.weixin\.qq\.com\/s[\/?]/.test(url)) return res.status(400).json({ error: '请粘贴公众号文章链接(https://mp.weixin.qq.com/s/…)' });
const art = parseWechat(await fetchWechatHtml(url), url);
if (!art.blocks.length) return res.status(422).json({ error: '文章正文是空的(可能是纯图片消息或已被删除)' });
const id = 'nw' + Date.now().toString(36) + crypto.randomBytes(3).toString('hex');
const dir = path.join(uploadsDir, id);
fs.mkdirSync(dir, { recursive: true });
// 收集所有要下载的图片(去重,最多 60 张),4 张并行
const urls = [...new Set([art.cover, ...art.blocks.flatMap((b) => (b.url && b.t === 'img' ? [b.url] : b.urls || []))].filter(Boolean))].slice(0, 61);
const fileOf = new Map(); let failed = 0, k = 0;
const worker = async () => { while (k < urls.length) { const i = k++; try { fileOf.set(urls[i], await downloadWxImage(urls[i], dir, (i === 0 && urls[i] === art.cover ? 'cover' : 'img' + String(i).padStart(2, '0')))); } catch { failed++; } } };
await Promise.all([worker(), worker(), worker(), worker()]);
const blocks = [];
let firstP = true;
for (const b of art.blocks) {
if (b.t === 'img') { const f = fileOf.get(b.url); if (f) blocks.push({ t: 'img', file: f }); }
else if (b.t === 'pair' || b.t === 'gallery') {
const files = b.urls.map((u) => fileOf.get(u)).filter(Boolean);
if (files.length === 1) blocks.push({ t: 'img', file: files[0] });
else if (files.length) blocks.push({ t: files.length === 2 ? 'pair' : 'gallery', files });
}
else if (b.t === 'yt') blocks.push({ t: 'yt', url: b.url });
else if (b.t === 'video') blocks.push({ t: 'yt', url: '', note: b.note });
else { const x = { t: b.t, zh: b.zh }; if (b.t === 'p' && firstP) { x.lead = true; firstP = false; } blocks.push(x); }
}
const firstText = (art.blocks.find((b) => b.t === 'p') || {}).zh || '';
const excerpt = art.desc || (firstText.length > 90 ? firstText.slice(0, 88) + '…' : firstText);
const coverFile = fileOf.get(art.cover) || articleFiles({ blocks })[0] || '';
// 正文第一张图就是封面的话,正文里不再重复
res.json({
draft: {
id, kind: 'article', src: 'wechat', srcUrl: url, account: art.account,
datetime: art.datetime || new Date(Date.now() + 8 * 3600e3).toISOString().slice(0, 16),
cat: 'company', title: { zh: art.title }, excerpt: { zh: excerpt }, cover: coverFile, blocks, published: false,
},
stats: { images: fileOf.size, failed, blocks: blocks.length },
});
} catch (e) { res.status(502).json({ error: e.message }); }
});
// 网站管理的图片:只允许读取"已分配到某个图片位"的文件 // 网站管理的图片:只允许读取"已分配到某个图片位"的文件
app.get('/api/public/site-media/:file', checkKey, (req, res) => { app.get('/api/public/site-media/:file', checkKey, (req, res) => {
const file = path.basename(req.params.file); const file = path.basename(req.params.file);