Initial commit: OmniAI backend server
This commit is contained in:
stringadmin
@@ -0,0 +1,259 @@
|
||||
/**
|
||||
* Provider Health Monitor — periodic health checks for DashScope and other providers.
|
||||
*
|
||||
* - Every 5 minutes, probes DashScope with a lightweight text call
|
||||
* - If Arrearage or auth failure detected, logs error + inserts admin notification
|
||||
* - Tracks provider health status in-memory for the /api/admin/providers/status endpoint
|
||||
*/
|
||||
|
||||
const { pool } = require("./db");
|
||||
|
||||
const CHECK_INTERVAL_MS = 5 * 60 * 1000;
|
||||
const DASHSCOPE_TEST_MODEL = "qwen-max";
|
||||
const DASHSCOPE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions";
|
||||
const LOW_BALANCE_THRESHOLD = 500; // cents — alert when balance below this
|
||||
|
||||
let timerId = null;
|
||||
|
||||
// In-memory health cache for the status endpoint
|
||||
const providerHealthCache = {
|
||||
dashscope: { status: "unknown", lastCheck: null, lastError: null, details: null },
|
||||
grsai: { status: "unknown", lastCheck: null, lastError: null, details: null },
|
||||
};
|
||||
|
||||
async function getDashScopeKey() {
|
||||
const { rows } = await pool.query(
|
||||
"SELECT id, api_key FROM api_keys WHERE provider LIKE '%dashscope%' AND enabled = 1 ORDER BY id LIMIT 1"
|
||||
);
|
||||
if (!rows.length) return null;
|
||||
return rows[0].api_key;
|
||||
}
|
||||
|
||||
async function getGrsaiKey() {
|
||||
const { rows } = await pool.query(
|
||||
"SELECT id, api_key FROM api_keys WHERE provider = 'grsai' AND enabled = 1 ORDER BY id LIMIT 1"
|
||||
);
|
||||
if (!rows.length) return null;
|
||||
return rows[0].api_key;
|
||||
}
|
||||
|
||||
async function probeDashScope(apiKey) {
|
||||
const body = {
|
||||
model: DASHSCOPE_TEST_MODEL,
|
||||
messages: [{ role: "user", content: "ping" }],
|
||||
stream: false,
|
||||
max_tokens: 4,
|
||||
enable_thinking: false,
|
||||
};
|
||||
const res = await fetch(DASHSCOPE_URL, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Authorization: "Bearer " + apiKey },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
const text = await res.text();
|
||||
|
||||
if (res.status === 400 || res.status === 403) {
|
||||
let json = {};
|
||||
try { json = JSON.parse(text); } catch {}
|
||||
const errorCode = json.error?.code || "";
|
||||
if (errorCode === "Arrearage") {
|
||||
return { ok: false, status: "arrears", error: "DashScope 账户欠费，所有 qwen 模型不可用", code: errorCode };
|
||||
}
|
||||
if (errorCode === "AccessDenied" || res.status === 403) {
|
||||
return { ok: false, status: "denied", error: "DashScope 访问被拒绝", code: errorCode };
|
||||
}
|
||||
return { ok: false, status: "error", error: `DashScope 返回 HTTP ${res.status}: ${errorCode}`, code: errorCode };
|
||||
}
|
||||
|
||||
if (!res.ok) {
|
||||
return { ok: false, status: "error", error: `DashScope 返回 HTTP ${res.status}`, code: "http_error" };
|
||||
}
|
||||
|
||||
return { ok: true, status: "healthy", error: null };
|
||||
}
|
||||
|
||||
async function probeGrsai(apiKey) {
|
||||
// GrsAI uses the same OpenAI-compatible endpoint
|
||||
const GRSAI_BASE = "https://grsai.dakka.com.cn";
|
||||
const url = `${GRSAI_BASE}/v1/chat/completions`;
|
||||
const body = {
|
||||
model: "gemini-3.1-pro",
|
||||
messages: [{ role: "user", content: "ping" }],
|
||||
stream: false,
|
||||
max_tokens: 4,
|
||||
};
|
||||
const res = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Authorization: "Bearer " + apiKey },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const errText = await res.text().catch(() => "");
|
||||
return { ok: false, status: "error", error: `GrsAI 返回 HTTP ${res.status}: ${errText.slice(0, 200)}` };
|
||||
}
|
||||
|
||||
return { ok: true, status: "healthy", error: null };
|
||||
}
|
||||
|
||||
async function notifyAdmin(title, description) {
|
||||
// Find admin users to notify
|
||||
const { rows: admins } = await pool.query(
|
||||
"SELECT id FROM users WHERE role = 'admin' AND enabled = 1"
|
||||
);
|
||||
if (!admins.length) {
|
||||
console.error("[providerHealthMonitor] No admin users found for notification");
|
||||
return;
|
||||
}
|
||||
for (const admin of admins) {
|
||||
await pool.query(
|
||||
`INSERT INTO web_notifications (user_id, type, title, description, metadata_json)
|
||||
VALUES ($1, 'provider_health', $2, $3, '{}')`,
|
||||
[admin.id, title, description]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function runHealthCheck() {
|
||||
// ── DashScope ──
|
||||
const dashKey = await getDashScopeKey();
|
||||
if (dashKey) {
|
||||
try {
|
||||
const result = await probeDashScope(dashKey);
|
||||
const prev = providerHealthCache.dashscope.status;
|
||||
providerHealthCache.dashscope = {
|
||||
status: result.status,
|
||||
lastCheck: new Date().toISOString(),
|
||||
lastError: result.error,
|
||||
details: result,
|
||||
};
|
||||
|
||||
if (!result.ok) {
|
||||
console.error(`[providerHealthMonitor] DashScope unhealthy: ${result.error}`);
|
||||
// Only notify on state change (healthy → unhealthy)
|
||||
if (prev === "healthy" || prev === "unknown") {
|
||||
await notifyAdmin("DashScope 服务异常", result.error);
|
||||
}
|
||||
} else {
|
||||
// Recovery notification
|
||||
if (prev !== "healthy" && prev !== "unknown") {
|
||||
console.log("[providerHealthMonitor] DashScope recovered");
|
||||
await notifyAdmin("DashScope 服务恢复正常", "DashScope 已恢复正常可用状态");
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
providerHealthCache.dashscope = {
|
||||
status: "timeout",
|
||||
lastCheck: new Date().toISOString(),
|
||||
lastError: err.message,
|
||||
details: null,
|
||||
};
|
||||
console.error("[providerHealthMonitor] DashScope probe failed:", err.message);
|
||||
}
|
||||
} else {
|
||||
providerHealthCache.dashscope = {
|
||||
status: "no_key",
|
||||
lastCheck: new Date().toISOString(),
|
||||
lastError: "No DashScope API key found in database",
|
||||
details: null,
|
||||
};
|
||||
}
|
||||
|
||||
// ── GrsAI ──
|
||||
const grsaiKey = await getGrsaiKey();
|
||||
if (grsaiKey) {
|
||||
try {
|
||||
const result = await probeGrsai(grsaiKey);
|
||||
const prev = providerHealthCache.grsai.status;
|
||||
providerHealthCache.grsai = {
|
||||
status: result.status,
|
||||
lastCheck: new Date().toISOString(),
|
||||
lastError: result.error,
|
||||
details: result,
|
||||
};
|
||||
|
||||
if (!result.ok) {
|
||||
console.error(`[providerHealthMonitor] GrsAI unhealthy: ${result.error}`);
|
||||
if (prev === "healthy" || prev === "unknown") {
|
||||
await notifyAdmin("GrsAI 服务异常", result.error);
|
||||
}
|
||||
} else {
|
||||
if (prev !== "healthy" && prev !== "unknown") {
|
||||
console.log("[providerHealthMonitor] GrsAI recovered");
|
||||
await notifyAdmin("GrsAI 服务恢复正常", "GrsAI 已恢复正常可用状态");
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
providerHealthCache.grsai = {
|
||||
status: "timeout",
|
||||
lastCheck: new Date().toISOString(),
|
||||
lastError: err.message,
|
||||
details: null,
|
||||
};
|
||||
console.error("[providerHealthMonitor] GrsAI probe failed:", err.message);
|
||||
}
|
||||
} else {
|
||||
providerHealthCache.grsai = {
|
||||
status: "no_key",
|
||||
lastCheck: new Date().toISOString(),
|
||||
lastError: "No GrsAI API key found in database",
|
||||
details: null,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Circuit breaker summary ──
|
||||
try {
|
||||
const cb = require("./providerCircuitBreaker");
|
||||
providerHealthCache.circuitBreaker = cb.getProviderStatusMap ? cb.getProviderStatusMap() : null;
|
||||
} catch {}
|
||||
|
||||
// ── Admin low-balance alert ──
|
||||
try {
|
||||
const { rows } = await pool.query(
|
||||
"SELECT id, username, balance_cents FROM users WHERE role = 'admin' AND enabled = 1 AND balance_cents < $1",
|
||||
[LOW_BALANCE_THRESHOLD]
|
||||
);
|
||||
for (const user of rows) {
|
||||
console.warn(`[providerHealthMonitor] Admin ${user.username} balance low: ${user.balance_cents} cents`);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
function startProviderHealthMonitor() {
|
||||
if (timerId) return;
|
||||
|
||||
runHealthCheck().catch((err) => {
|
||||
console.error("[providerHealthMonitor] initial run failed:", err.message);
|
||||
});
|
||||
|
||||
timerId = setInterval(() => {
|
||||
runHealthCheck().catch((err) => {
|
||||
console.error("[providerHealthMonitor] periodic run failed:", err.message);
|
||||
});
|
||||
}, CHECK_INTERVAL_MS);
|
||||
|
||||
if (timerId.unref) timerId.unref();
|
||||
|
||||
console.log(`[providerHealthMonitor] started (interval=${CHECK_INTERVAL_MS}ms)`);
|
||||
}
|
||||
|
||||
function stopProviderHealthMonitor() {
|
||||
if (timerId) {
|
||||
clearInterval(timerId);
|
||||
timerId = null;
|
||||
console.log("[providerHealthMonitor] stopped");
|
||||
}
|
||||
}
|
||||
|
||||
function getProviderHealthCache() {
|
||||
return providerHealthCache;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
startProviderHealthMonitor,
|
||||
stopProviderHealthMonitor,
|
||||
getProviderHealthCache,
|
||||
runHealthCheck,
|
||||
};
|
||||
Reference in New Issue
Block a user