147 lines
6.0 KiB
TypeScript
147 lines
6.0 KiB
TypeScript
/**
|
||
* 对比测试：chat/completions vs completions 接口耗时
|
||
* 使用书籍解析 Prompt + 同一段样本文本，各调用 3 次，记录耗时并汇总。
|
||
*/
|
||
|
||
import OpenAI from 'openai';
|
||
import * as dotenv from 'dotenv';
|
||
import { DEFAULT_BOOK_CONTENT_PROMPT } from '../src/services/bookPromptConfigService';
|
||
|
||
dotenv.config();
|
||
|
||
// 与 modelProvider 一致：无 env 时用同一 fallback（仅便于在服务器跑对比测试）
|
||
const DOUBAO_API_KEY = process.env.DOUBAO_API_KEY || 'a3e13a85-437f-448c-aaa9-14292cd5e0ab';
|
||
const DOUBAO_BASE_URL = 'https://ark.cn-beijing.volces.com/api/v3';
|
||
const DOUBAO_MODEL = 'doubao-seed-1-6-lite-251015';
|
||
|
||
// 与书籍解析一致的 system：把 {{sourceText}} 替换为「见下文」
|
||
const SYSTEM_PROMPT = DEFAULT_BOOK_CONTENT_PROMPT.replace(/\{\{sourceText\}\}/g, '见下文');
|
||
|
||
// 固定样本文本（单层结构，便于模型快速输出）
|
||
const SAMPLE_SOURCE_TEXT = `第一章 什么是有效沟通
|
||
|
||
有效沟通是人与人之间准确传递信息、情感与意图的过程。它包含清晰表达、积极倾听和及时反馈。
|
||
|
||
第二章 倾听的重要性
|
||
|
||
倾听不仅是听到对方说了什么，更要理解其背后的需求与情绪。主动倾听能让对方感到被尊重，减少误解。
|
||
|
||
第三章 反馈的技巧
|
||
|
||
反馈要具体、及时、对事不对人。用「我观察到…」「我的感受是…」等表述，能降低对方的防御心理。`;
|
||
|
||
const RUNS = 3;
|
||
const MAX_TOKENS = 2000;
|
||
|
||
async function run() {
|
||
if (!DOUBAO_API_KEY) {
|
||
console.error('请设置 DOUBAO_API_KEY（.env 或环境变量）');
|
||
process.exit(1);
|
||
}
|
||
// 若使用 fallback key，仅打印提示，不暴露完整 key
|
||
const keySource = process.env.DOUBAO_API_KEY ? 'env' : 'fallback';
|
||
console.log('API Key 来源:', keySource);
|
||
|
||
const client = new OpenAI({
|
||
apiKey: DOUBAO_API_KEY,
|
||
baseURL: DOUBAO_BASE_URL,
|
||
});
|
||
|
||
console.log('═══════════════════════════════════════════════════════════');
|
||
console.log(' 书籍解析 Prompt 对比：chat/completions vs completions');
|
||
console.log(' 同一文本，各调用 ' + RUNS + ' 次');
|
||
console.log('═══════════════════════════════════════════════════════════');
|
||
console.log('Base URL:', DOUBAO_BASE_URL);
|
||
console.log('Model:', DOUBAO_MODEL);
|
||
console.log('样本文本长度:', SAMPLE_SOURCE_TEXT.length, '字符');
|
||
console.log('max_tokens:', MAX_TOKENS);
|
||
console.log('');
|
||
|
||
const chatDurations: number[] = [];
|
||
const compDurations: number[] = [];
|
||
|
||
// ---------- 1) Chat Completions，调用 RUNS 次 ----------
|
||
console.log('--- /chat/completions 调用 ' + RUNS + ' 次 ---');
|
||
for (let i = 0; i < RUNS; i++) {
|
||
try {
|
||
const t0 = Date.now();
|
||
await client.chat.completions.create({
|
||
model: DOUBAO_MODEL,
|
||
messages: [
|
||
{ role: 'system', content: SYSTEM_PROMPT },
|
||
{ role: 'user', content: SAMPLE_SOURCE_TEXT },
|
||
],
|
||
max_tokens: MAX_TOKENS,
|
||
temperature: 0.7,
|
||
});
|
||
const ms = Date.now() - t0;
|
||
chatDurations.push(ms);
|
||
console.log(' 第 ' + (i + 1) + ' 次: ' + ms + ' ms');
|
||
} catch (e: any) {
|
||
console.log(' 第 ' + (i + 1) + ' 次: 失败 -', e?.message || e);
|
||
}
|
||
}
|
||
console.log('');
|
||
|
||
// ---------- 2) Completions（合并为单条 prompt），调用 RUNS 次 ----------
|
||
const singlePrompt =
|
||
SYSTEM_PROMPT + '\n\n--- 用户消息（原始材料）---\n\n' + SAMPLE_SOURCE_TEXT;
|
||
console.log('--- /completions 调用 ' + RUNS + ' 次 ---');
|
||
for (let i = 0; i < RUNS; i++) {
|
||
try {
|
||
const t0 = Date.now();
|
||
await client.completions.create({
|
||
model: DOUBAO_MODEL,
|
||
prompt: singlePrompt,
|
||
max_tokens: MAX_TOKENS,
|
||
temperature: 0.7,
|
||
});
|
||
const ms = Date.now() - t0;
|
||
compDurations.push(ms);
|
||
console.log(' 第 ' + (i + 1) + ' 次: ' + ms + ' ms');
|
||
} catch (e: any) {
|
||
console.log(' 第 ' + (i + 1) + ' 次: 失败 -', e?.message || e);
|
||
}
|
||
}
|
||
console.log('');
|
||
|
||
// ---------- 汇总 ----------
|
||
console.log('═══════════════════════════════════════════════════════════');
|
||
console.log(' 汇总');
|
||
console.log('═══════════════════════════════════════════════════════════');
|
||
if (chatDurations.length > 0) {
|
||
const sum = chatDurations.reduce((a, b) => a + b, 0);
|
||
const avg = Math.round(sum / chatDurations.length);
|
||
console.log('chat/completions: 成功 ' + chatDurations.length + ' 次');
|
||
console.log(' 各次(ms): ' + chatDurations.join(', '));
|
||
console.log(' 平均(ms): ' + avg);
|
||
} else {
|
||
console.log('chat/completions: 无成功调用');
|
||
}
|
||
console.log('');
|
||
if (compDurations.length > 0) {
|
||
const sum = compDurations.reduce((a, b) => a + b, 0);
|
||
const compAvg = sum / compDurations.length;
|
||
console.log('completions: 成功 ' + compDurations.length + ' 次');
|
||
console.log(' 各次(ms): ' + compDurations.join(', '));
|
||
console.log(' 平均(ms): ' + Math.round(compAvg));
|
||
if (chatDurations.length > 0) {
|
||
const chatAvg =
|
||
chatDurations.reduce((a, b) => a + b, 0) / chatDurations.length;
|
||
const diff = Math.round(compAvg - chatAvg);
|
||
const pct = chatAvg > 0 ? ((diff / chatAvg) * 100).toFixed(1) : '0';
|
||
console.log(
|
||
'与 chat 平均差异: ' + (diff >= 0 ? '+' : '') + diff + ' ms (' + pct + '%)'
|
||
);
|
||
}
|
||
} else {
|
||
console.log('completions: 无成功调用（豆包可能未开放此端点）');
|
||
}
|
||
console.log('═══════════════════════════════════════════════════════════');
|
||
}
|
||
|
||
run().catch((e) => {
|
||
console.error(e);
|
||
process.exit(1);
|
||
});
|