feat: P4 多模态 — 图片上传 + 沙盒 vision 支持

后端:
- ChatMessage 接口支持 text | image_url 多部分内容
- sandbox chat 接受 images[] 参数
- 自动将图片转为 image_url 格式发送到 AI
- 上传模块支持 jpg/png/gif/webp/svg

前端:
- 沙盒输入区 + 图片上传按钮
- 图片缩略图预览 + 删除
- 上传后随消息发送到 API
- uploads/ 目录持久化存储
This commit is contained in:
yuzhiran-dev
2026-05-18 14:15:56 +08:00
parent 0ed017b212
commit 79d7535c1c
6 changed files with 71 additions and 15 deletions
+3 -2
View File
@@ -2,7 +2,7 @@ import { Injectable, Logger } from '@nestjs/common';
interface ChatMessage {
role: 'user' | 'assistant' | 'system';
content: string;
content: string | { type: 'text' | 'image_url'; text?: string; image_url?: { url: string } }[];
}
export interface ChatOptions {
@@ -112,7 +112,8 @@ export class AIGatewayService {
}
private fallback(messages: ChatMessage[]): string {
const lastMsg = messages[messages.length - 1]?.content || '';
const lastMsg = typeof messages[messages.length - 1]?.content === 'string'
? messages[messages.length - 1]?.content as string : '';
const mockReplies: Record<string, string> = {
'你好': '你好!我是宇之然 AI 助手,很高兴为你服务!',
'hello': 'Hello! I am YuZhiRan AI assistant, nice to meet you!',
@@ -12,9 +12,9 @@ export class SandboxController {
constructor(private sandboxService: SandboxService) {}
@Post('chat')
@ApiBody({ schema: { example: { conversationId: 'uuid', model: 'general', messages: [{ role: 'user', content: 'hi' }], temperature: 0.7, top_p: 1, max_tokens: 2000 } } })
async chat(@Req() req: any, @Body() body: { conversationId?: string; model: string; messages: { role: string; content: string }[] } & ChatOptions) {
return this.sandboxService.chat(req.user.userId, body.conversationId, body.model, body.messages, body);
@ApiBody({ schema: { example: { conversationId: 'uuid', model: 'general', messages: [{ role: 'user', content: 'hi' }], images: ['https://example.com/img.png'], temperature: 0.7, top_p: 1, max_tokens: 2000 } } })
async chat(@Req() req: any, @Body() body: { conversationId?: string; model: string; messages: { role: string; content: string }[]; images?: string[] } & ChatOptions) {
return this.sandboxService.chat(req.user.userId, body.conversationId, body.model, body.messages, body, body.images);
}
@Get('sessions')
+17 -2
View File
@@ -10,7 +10,7 @@ export class SandboxService {
private aiGateway: AIGatewayService,
) {}
async chat(userId: number, conversationId: string | undefined, model: string, messages: { role: string; content: string }[], options?: ChatOptions) {
async chat(userId: number, conversationId: string | undefined, model: string, messages: { role: string; content: string }[], options?: ChatOptions, images?: string[]) {
const user = await this.prisma.user.findUnique({ where: { id: userId } });
if (!user || user.status !== 'ACTIVE') {
throw new HttpException('用户不可用', HttpStatus.FORBIDDEN);
@@ -33,7 +33,22 @@ export class SandboxService {
}
}
let reply = await this.aiGateway.chat(model, messages as any, options);
// Convert images to structured message content
let aiMessages = messages as any[];
if (images && images.length > 0) {
aiMessages = messages.map(m => {
if (m.role === 'user' && m === messages[messages.length - 1]) {
const parts: any[] = [{ type: 'text', text: m.content }];
for (const img of images) {
parts.push({ type: 'image_url', image_url: { url: img } });
}
return { role: m.role, content: parts };
}
return m;
});
}
let reply = await this.aiGateway.chat(model, aiMessages, options);
if (typeof reply !== 'string') {
reply = '抱歉,AI 返回了无效的回复,请重试。';
}
File diff suppressed because one or more lines are too long