132 lines
3.2 KiB
TypeScript
132 lines
3.2 KiB
TypeScript
/**
|
|
* LLM Client for api.riotpiao.com
|
|
*
|
|
* Usage:
|
|
* import { chat, streamChat } from './llm-client';
|
|
*
|
|
* // Non-streaming
|
|
* const response = await chat({ model: 'reasoning', messages: [...] });
|
|
*
|
|
* // Streaming
|
|
* for await (const chunk of streamChat({ model: 'reasoning', messages: [...] })) {
|
|
* process.stdout.write(chunk);
|
|
* }
|
|
*/
|
|
|
|
export type Model = 'reasoning' | 'ornith:35b' | 'qwen2.5:3b-instruct';
|
|
|
|
export interface Message {
|
|
role: 'system' | 'user' | 'assistant';
|
|
content: string;
|
|
}
|
|
|
|
export interface ChatRequest {
|
|
model: Model;
|
|
messages: Message[];
|
|
max_tokens?: number;
|
|
temperature?: number;
|
|
stream?: boolean;
|
|
}
|
|
|
|
export interface ChatResponse {
|
|
id: string;
|
|
object: string;
|
|
created: number;
|
|
model: string;
|
|
choices: {
|
|
index: number;
|
|
message: Message;
|
|
finish_reason: string;
|
|
}[];
|
|
usage?: {
|
|
prompt_tokens: number;
|
|
completion_tokens: number;
|
|
total_tokens: number;
|
|
};
|
|
}
|
|
|
|
const LLM_BASE_URL = process.env.LLM_BASE_URL || 'https://api.riotpiao.com/v1';
|
|
|
|
/**
|
|
* Non-streaming chat completion
|
|
*/
|
|
export async function chat(request: ChatRequest): Promise<ChatResponse> {
|
|
const response = await fetch(`${LLM_BASE_URL}/chat/completions`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ ...request, stream: false }),
|
|
});
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`LLM API error: ${response.status} ${await response.text()}`);
|
|
}
|
|
|
|
return response.json();
|
|
}
|
|
|
|
/**
|
|
* Streaming chat completion - yields content chunks
|
|
*/
|
|
export async function* streamChat(request: ChatRequest): AsyncGenerator<string> {
|
|
const response = await fetch(`${LLM_BASE_URL}/chat/completions`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ ...request, stream: true }),
|
|
});
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`LLM API error: ${response.status} ${await response.text()}`);
|
|
}
|
|
|
|
const reader = response.body?.getReader();
|
|
if (!reader) throw new Error('No response body');
|
|
|
|
const decoder = new TextDecoder();
|
|
let buffer = '';
|
|
|
|
while (true) {
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
|
|
buffer += decoder.decode(value, { stream: true });
|
|
const lines = buffer.split('\n');
|
|
buffer = lines.pop() || '';
|
|
|
|
for (const line of lines) {
|
|
if (line.startsWith('data: ')) {
|
|
const data = line.slice(6);
|
|
if (data === '[DONE]') return;
|
|
try {
|
|
const parsed = JSON.parse(data);
|
|
const content = parsed.choices?.[0]?.delta?.content;
|
|
if (content) yield content;
|
|
} catch {
|
|
// Skip malformed chunks
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* React hook compatible streaming (returns ReadableStream for Response)
|
|
*/
|
|
export function createStreamResponse(request: ChatRequest): Response {
|
|
const stream = new ReadableStream({
|
|
async start(controller) {
|
|
try {
|
|
for await (const chunk of streamChat(request)) {
|
|
controller.enqueue(new TextEncoder().encode(chunk));
|
|
}
|
|
controller.close();
|
|
} catch (err) {
|
|
controller.error(err);
|
|
}
|
|
},
|
|
});
|
|
|
|
return new Response(stream, {
|
|
headers: { 'Content-Type': 'text/plain; charset=utf-8' },
|
|
});
|
|
}
|