From 035190420f068612ead9af99b0232f0b82a80bea Mon Sep 17 00:00:00 2001 From: stringadmin Date: Tue, 2 Jun 2026 15:45:29 +0800 Subject: [PATCH 1/3] fix(oss): add binary upload route + base64 fallback fix + MIME types - Add /oss/upload-binary route using busboy for FormData multipart uploads - Fix parseUploadPayload base64 fallback: strip data:xxx;base64 prefix instead of using entire rawData string as base64 (caused 44-byte corrupt files when DATA_URL_PATTERN regex did not match) - Add image/avif, image/heic, image/heif to MIME_EXTENSIONS Root cause of ecommerce 502: base64 dataUrl not matching regex pattern caused server to store corrupt 44-byte files on OSS, DashScope could not read them and returned "image format is illegal" error. --- package-lock.json | 20 +++++++++++++ package.json | 1 + src/routes/oss.js | 72 ++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 92 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index f23bc0a..640ee9b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "alipay-sdk": "^4.14.0", "bcryptjs": "^2.4.3", + "busboy": "^1.6.0", "cors": "^2.8.5", "dotenv": "^16.4.5", "express": "^4.21.0", @@ -159,6 +160,17 @@ "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", "license": "BSD-3-Clause" }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -1527,6 +1539,14 @@ "node": ">= 0.8" } }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/superagent": { "version": "8.0.6", "resolved": "https://registry.npmjs.org/superagent/-/superagent-8.0.6.tgz", diff --git a/package.json b/package.json index ad2140f..53a6d93 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "dependencies": { "alipay-sdk": "^4.14.0", "bcryptjs": "^2.4.3", + "busboy": "^1.6.0", "cors": "^2.8.5", "dotenv": "^16.4.5", "express": "^4.21.0", diff --git a/src/routes/oss.js b/src/routes/oss.js index fcd23c4..777aab2 100644 --- a/src/routes/oss.js +++ b/src/routes/oss.js @@ -1,6 +1,7 @@ const crypto = require("node:crypto"); const dns = require("node:dns"); const { requireAuth } = require("./context"); +const Busboy = require("busboy"); const { putObject, isOssConfigured, createSignedReadUrl } = require("../ossClient"); const DATA_URL_PATTERN = /^data:([^;,]+);base64,([A-Za-z0-9+/=\s]+)$/; @@ -61,6 +62,9 @@ const MIME_EXTENSIONS = { "image/jpeg": "jpg", "image/png": "png", "image/webp": "webp", + "image/avif": "avif", + "image/heic": "heic", + "image/heif": "heif", "image/gif": "gif", "video/mp4": "mp4", "video/webm": "webm", @@ -166,7 +170,7 @@ function parseUploadPayload(body) { const rawData = String(body?.dataUrl || body?.data || ""); const dataUrlMatch = rawData.match(DATA_URL_PATTERN); const mimeType = normalizeMimeType(body?.mimeType || dataUrlMatch?.[1]); - const base64 = (dataUrlMatch?.[2] || rawData).replace(/\s+/g, ""); + const base64 = (dataUrlMatch?.[2] || rawData.replace(/^data:[^;,]+;base64,/, "")).replace(/\s+/g, ""); if (!base64) { const error = new Error("Missing upload data"); error.status = 400; @@ -317,6 +321,72 @@ function registerOssRoutes(router) { if (!res.headersSent) res.status(502).json({ error: "Proxy failed" }); } }); + router.post("/oss/upload-binary", requireAuth, (req, res) => { + if (!isOssConfigured()) { + return res.status(501).json({ error: "OSS 未配置" }); + } + + const busboy = Busboy({ headers: req.headers, limits: { fileSize: 10 * 1024 * 1024 } }); + let fileBuffer = null; + let fileMimeType = "application/octet-stream"; + let fileName = "upload"; + let scope = ""; + + busboy.on("file", (fieldname, stream, info) => { + if (fieldname !== "file") { + stream.resume(); + return; + } + fileName = info.filename || "upload"; + fileMimeType = info.mimeType || "application/octet-stream"; + const chunks = []; + stream.on("data", (chunk) => chunks.push(chunk)); + stream.on("end", () => { + fileBuffer = Buffer.concat(chunks); + }); + stream.on("error", (err) => { + console.error("[oss/upload-binary] stream error:", err.message); + res.status(500).json({ error: "文件读取失败" }); + }); + }); + + busboy.on("field", (fieldname, val) => { + if (fieldname === "scope") scope = String(val).trim(); + if (fieldname === "mimeType") fileMimeType = String(val).trim(); + }); + + busboy.on("finish", async () => { + try { + if (!fileBuffer) { + return res.status(400).json({ error: "未收到文件" }); + } + const normalizedMimeType = normalizeMimeType(fileMimeType); + const ext = MIME_EXTENSIONS[normalizedMimeType] || "bin"; + const assetDir = getAssetDirectory(normalizedMimeType); + const safeUserId = String(req.user.id).replace(/[^a-zA-Z0-9_-]/g, ""); + const objectKey = + getProfileObjectKey(scope, req.user.id, ext, normalizedMimeType) || + getCommunityObjectKey(scope, req.user.id, ext, normalizedMimeType) || + `tmp/${safeUserId}/generation-inputs/${assetDir}/${Date.now()}_${crypto.randomUUID()}.${ext}`; + await putObject(objectKey, fileBuffer, normalizedMimeType, { "x-oss-object-acl": "public-read" }); + const url = buildOssPublicUrl(objectKey); + const signedUrl = typeof createSignedReadUrl === "function" ? createSignedReadUrl(objectKey) : url; + console.info("[oss/upload-binary] mimeType:", normalizedMimeType, "size:", fileBuffer.length, "userId:", req.user.id); + res.status(201).json({ ossKey: objectKey, url, signedUrl }); + } catch (err) { + const status = err.status || 500; + console.error("[oss/upload-binary] failed:", err.message); + res.status(status).json({ error: err.message || "上传素材失败" }); + } + }); + + busboy.on("error", (err) => { + console.error("[oss/upload-binary] busboy error:", err.message); + res.status(500).json({ error: "文件解析失败" }); + }); + + req.pipe(busboy); + }); } module.exports = { From 3c574eeff6c4390ef260363200f58e15af36161d Mon Sep 17 00:00:00 2001 From: stringadmin Date: Tue, 2 Jun 2026 17:00:05 +0800 Subject: [PATCH 2/3] fix: rate limiting + upload-binary body-parser bypass + OSS upload route - Increase global rate limit 100->300 req/min - Add /api/ai/chat dedicated limiter 60 req/min - Add req._body=true middleware to skip JSON parser for upload-binary - Add busboy binary upload route + MIME type extensions --- src/index.js | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/index.js b/src/index.js index 4621773..2a1533d 100644 --- a/src/index.js +++ b/src/index.js @@ -57,10 +57,10 @@ async function main() { // CORS app.use(cors(buildCorsOptions())) - // Rate limiting: global (100 req/min per IP) + // Rate limiting: global (300 req/min per IP) const globalLimiter = rateLimit({ windowMs: 60 * 1000, - max: 100, + max: 300, standardHeaders: true, legacyHeaders: false, message: { error: '请求过于频繁，请稍后再试' }, @@ -90,6 +90,19 @@ async function main() { app.use('/api/ai/image', aiGenerationLimiter) app.use('/api/ai/video', aiGenerationLimiter) + // Rate limiting: AI chat endpoint (60 req/min per IP — ecommerce flows need ~7 sequential calls) + const aiChatLimiter = rateLimit({ + windowMs: 60 * 1000, + max: 60, + standardHeaders: true, + legacyHeaders: false, + message: { error: 'AI对话请求过于频繁，请稍后再试' }, + }) + app.use('/api/ai/chat', aiChatLimiter) + + + // Skip JSON body-parser for binary upload routes (busboy handles multipart parsing) + app.use('/api/oss/upload-binary', (req, res, next) => { req._body = true; next(); }) // JSON body limit: 5MB globally (upload routes override locally) app.use('/api/oss/upload', express.json({ limit: '200mb' })) app.use(express.json({ limit: process.env.JSON_BODY_LIMIT || '5mb' })) From 1a5992845aefa3b5310a2346610161c1b3ab4efc Mon Sep 17 00:00:00 2001 From: stringadmin Date: Wed, 3 Jun 2026 10:52:26 +0800 Subject: [PATCH 3/3] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E5=AE=A2?= =?UTF-8?q?=E6=88=B7=E7=AB=AF=E9=94=99=E8=AF=AF=E6=94=B6=E9=9B=86=E4=B8=8E?= =?UTF-8?q?admin=E7=9B=91=E6=8E=A7=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - POST /api/client-errors: 批量存储客户端错误（10分钟去重合并） - GET /api/client-errors: admin分页查看错误列表 - DELETE /api/client-errors: admin清空错误记录 - 72小时自动清理旧数据 Co-Authored-By: Claude Opus 4.7 --- src/routes/clientErrors.js | 94 ++++++++++++++++++++++++++++++++++++++ src/routes/index.js | 3 +- 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 src/routes/clientErrors.js diff --git a/src/routes/clientErrors.js b/src/routes/clientErrors.js new file mode 100644 index 0000000..c60c822 --- /dev/null +++ b/src/routes/clientErrors.js @@ -0,0 +1,94 @@ +const crypto = require('node:crypto'); +const { pool, requireAuth, requireAdmin } = require('./context'); + +const MAX_ERRORS_PER_PAGE = 50; +const ERROR_RETENTION_HOURS = 72; + +async function initTable(client) { + await client.query(` + CREATE TABLE IF NOT EXISTS client_errors ( + id SERIAL PRIMARY KEY, + message TEXT NOT NULL, + stack TEXT, + source VARCHAR(32), + url TEXT, + user_agent TEXT, + session_id TEXT, + user_id INTEGER, + count INTEGER DEFAULT 1, + first_seen TIMESTAMP DEFAULT NOW(), + last_seen TIMESTAMP DEFAULT NOW() + ); + CREATE INDEX IF NOT EXISTS idx_client_errors_last_seen ON client_errors(last_seen DESC); + CREATE INDEX IF NOT EXISTS idx_client_errors_user ON client_errors(user_id); + `); +} + +async function storeError(client, error) { + const normalizedMessage = error.message.slice(0, 1000); + const { rows } = await client.query( + `SELECT id, count FROM client_errors WHERE message = $1 AND url = $2 AND source = $3 AND last_seen > NOW() - INTERVAL '10 minutes'`, + [normalizedMessage, error.url || '', error.source || 'manual'] + ); + if (rows.length > 0) { + await client.query( + `UPDATE client_errors SET count = count + 1, last_seen = NOW(), stack = COALESCE($1, stack) WHERE id = $2`, + [error.stack, rows[0].id] + ); + } else { + await client.query( + `INSERT INTO client_errors (message, stack, source, url, user_agent, session_id, user_id) VALUES ($1,$2,$3,$4,$5,$6,$7)`, + [normalizedMessage, error.stack, error.source, error.url, error.userAgent, error.sessionId, error.userId || null] + ); + } +} + +module.exports = function mountClientErrorRoutes(router) { + router.post('/client-errors', requireAuth, async (req, res) => { + try { + const { errors } = req.body; + if (!Array.isArray(errors) || !errors.length) return res.json({ ok: true }); + const { pool: db } = require('./context'); + const client = await db.connect(); + try { + await initTable(client); + for (const err of errors.slice(0, 10)) { + await storeError(client, { ...err, userId: req.user?.id }); + } + } finally { client.release(); } + res.json({ ok: true }); + } catch (e) { + res.status(500).json({ error: e.message }); + } + }); + + router.get('/client-errors', requireAuth, requireAdmin, async (req, res) => { + try { + const page = Math.max(1, parseInt(req.query.page) || 1); + const { pool: db } = require('./context'); + const client = await db.connect(); + try { + await initTable(client); + await client.query(`DELETE FROM client_errors WHERE last_seen < NOW() - INTERVAL '$1 hours'`, [ERROR_RETENTION_HOURS]); + const { rows } = await client.query( + `SELECT * FROM client_errors ORDER BY last_seen DESC LIMIT $1 OFFSET $2`, + [MAX_ERRORS_PER_PAGE, (page - 1) * MAX_ERRORS_PER_PAGE] + ); + const { rows: countRows } = await client.query('SELECT COUNT(*) as total FROM client_errors'); + res.json({ items: rows, total: parseInt(countRows[0]?.total) || 0, page, pageSize: MAX_ERRORS_PER_PAGE }); + } finally { client.release(); } + } catch (e) { + res.status(500).json({ error: e.message }); + } + }); + + router.delete('/client-errors', requireAuth, requireAdmin, async (req, res) => { + try { + const { pool: db } = require('./context'); + await db.query('DELETE FROM client_errors'); + res.json({ ok: true }); + } catch (e) { + res.status(500).json({ error: e.message }); + } + }); +}; diff --git a/src/routes/index.js b/src/routes/index.js index 57bd418..8129e36 100644 --- a/src/routes/index.js +++ b/src/routes/index.js @@ -17,7 +17,8 @@ const { registerConversationRoutes } = require('./conversations') const { registerReportRoutes } = require('./reports') const { registerAssetRoutes } = require('./assets') const { registerNotificationRoutes } = require('./notifications') -const { registerDraftRoutes } = require('./drafts') +const { registerDraftRoutes } = require('./drafts'); +const mountClientErrorRoutes = require('./clientErrors') const router = express.Router()