feat: P4 多模态 — 图片上传 + 沙盒 vision 支持
后端: - ChatMessage 接口支持 text | image_url 多部分内容 - sandbox chat 接受 images[] 参数 - 自动将图片转为 image_url 格式发送到 AI - 上传模块支持 jpg/png/gif/webp/svg 前端: - 沙盒输入区 + 图片上传按钮 - 图片缩略图预览 + 删除 - 上传后随消息发送到 API - uploads/ 目录持久化存储
This commit is contained in:
@@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
|
||||
|
||||
interface ChatMessage {
|
||||
role: 'user' | 'assistant' | 'system';
|
||||
content: string;
|
||||
content: string | { type: 'text' | 'image_url'; text?: string; image_url?: { url: string } }[];
|
||||
}
|
||||
|
||||
export interface ChatOptions {
|
||||
@@ -112,7 +112,8 @@ export class AIGatewayService {
|
||||
}
|
||||
|
||||
private fallback(messages: ChatMessage[]): string {
|
||||
const lastMsg = messages[messages.length - 1]?.content || '';
|
||||
const lastMsg = typeof messages[messages.length - 1]?.content === 'string'
|
||||
? messages[messages.length - 1]?.content as string : '';
|
||||
const mockReplies: Record<string, string> = {
|
||||
'你好': '你好!我是宇之然 AI 助手,很高兴为你服务!',
|
||||
'hello': 'Hello! I am YuZhiRan AI assistant, nice to meet you!',
|
||||
|
||||
@@ -12,9 +12,9 @@ export class SandboxController {
|
||||
constructor(private sandboxService: SandboxService) {}
|
||||
|
||||
@Post('chat')
|
||||
@ApiBody({ schema: { example: { conversationId: 'uuid', model: 'general', messages: [{ role: 'user', content: 'hi' }], temperature: 0.7, top_p: 1, max_tokens: 2000 } } })
|
||||
async chat(@Req() req: any, @Body() body: { conversationId?: string; model: string; messages: { role: string; content: string }[] } & ChatOptions) {
|
||||
return this.sandboxService.chat(req.user.userId, body.conversationId, body.model, body.messages, body);
|
||||
@ApiBody({ schema: { example: { conversationId: 'uuid', model: 'general', messages: [{ role: 'user', content: 'hi' }], images: ['https://example.com/img.png'], temperature: 0.7, top_p: 1, max_tokens: 2000 } } })
|
||||
async chat(@Req() req: any, @Body() body: { conversationId?: string; model: string; messages: { role: string; content: string }[]; images?: string[] } & ChatOptions) {
|
||||
return this.sandboxService.chat(req.user.userId, body.conversationId, body.model, body.messages, body, body.images);
|
||||
}
|
||||
|
||||
@Get('sessions')
|
||||
|
||||
@@ -10,7 +10,7 @@ export class SandboxService {
|
||||
private aiGateway: AIGatewayService,
|
||||
) {}
|
||||
|
||||
async chat(userId: number, conversationId: string | undefined, model: string, messages: { role: string; content: string }[], options?: ChatOptions) {
|
||||
async chat(userId: number, conversationId: string | undefined, model: string, messages: { role: string; content: string }[], options?: ChatOptions, images?: string[]) {
|
||||
const user = await this.prisma.user.findUnique({ where: { id: userId } });
|
||||
if (!user || user.status !== 'ACTIVE') {
|
||||
throw new HttpException('用户不可用', HttpStatus.FORBIDDEN);
|
||||
@@ -33,7 +33,22 @@ export class SandboxService {
|
||||
}
|
||||
}
|
||||
|
||||
let reply = await this.aiGateway.chat(model, messages as any, options);
|
||||
// Convert images to structured message content
|
||||
let aiMessages = messages as any[];
|
||||
if (images && images.length > 0) {
|
||||
aiMessages = messages.map(m => {
|
||||
if (m.role === 'user' && m === messages[messages.length - 1]) {
|
||||
const parts: any[] = [{ type: 'text', text: m.content }];
|
||||
for (const img of images) {
|
||||
parts.push({ type: 'image_url', image_url: { url: img } });
|
||||
}
|
||||
return { role: m.role, content: parts };
|
||||
}
|
||||
return m;
|
||||
});
|
||||
}
|
||||
|
||||
let reply = await this.aiGateway.chat(model, aiMessages, options);
|
||||
if (typeof reply !== 'string') {
|
||||
reply = '抱歉,AI 返回了无效的回复,请重试。';
|
||||
}
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user