feat: realistic face avatar + voice input + ASR endpoint

This commit is contained in:
yuzhiran
2026-06-12 15:32:04 +08:00
parent 6fe84b6ef8
commit 8191cf4b41
26 changed files with 1934 additions and 228 deletions
+343 -138
View File
@@ -1,21 +1,17 @@
<template>
<view class="digital-human">
<view class="avatar-stage">
<view class="avatar-ring" :class="{ speaking: isSpeaking }">
<!-- Default CSS avatar if image fails -->
<view class="avatar-default" v-if="imgFailed">
<text class="avatar-initials">AI</text>
</view>
<image class="avatar-img" :src="avatarSrc" mode="aspectFill" @error="imgFailed = true" v-else />
<canvas
v-if="isH5"
id="dh-mouth"
class="mouth-canvas"
></canvas>
</view>
<canvas
:style="{ width: canvasSize + 'px', height: canvasSize + 'px' }"
:canvas-id="canvasId"
:id="canvasId"
type="2d"
class="face-canvas"
:class="{ speaking: isSpeaking }"
></canvas>
<view class="status-dot" :class="{ active: isSpeaking }"></view>
<text class="role-label">AI 面试官</text>
</view>
<view class="speech-area" v-if="currentText">
<view class="speech-bubble">
<text class="speech-text">{{ currentText }}</text>
@@ -25,34 +21,45 @@
</template>
<script setup>
import { ref, computed, watch, onMounted, onBeforeUnmount, nextTick } from 'vue'
import { ref, watch, onMounted, onBeforeUnmount, nextTick, computed } from 'vue'
const props = defineProps({
text: { type: String, default: '' },
audioUrl: { type: String, default: '' },
avatarUrl: { type: String, default: '' },
autoPlay: { type: Boolean, default: true },
})
const emit = defineEmits(['speaking-start', 'speaking-end'])
const isH5 = ref(false)
const canvasSize = 280
const canvasId = 'faceCanvas'
const isSpeaking = ref(false)
const currentText = ref('')
const imgFailed = ref(false)
let audioEl = null
let audioCtx = null
let analyser = null
let animFrameId = null
let mouthScale = 0
let blinkTimer = null
let faceCtx = null
// Face animation state
let blinkFrame = 0 // 0=open, 1=half, 2=closed, then back
let mouthOpen = 0 // 0-1
let targetMouth = 0
let lastBlinkTime = 0
let faceW = canvasSize
let faceH = canvasSize
onMounted(() => {
isH5.value = typeof window !== 'undefined' && typeof document !== 'undefined'
initCanvas()
nextTick(() => initCanvas())
scheduleBlink()
})
onBeforeUnmount(() => {
stopAudio()
if (blinkTimer) clearTimeout(blinkTimer)
if (animFrameId) cancelAnimationFrame(animFrameId)
})
watch(() => props.audioUrl, (url) => {
@@ -65,52 +72,264 @@ watch(() => props.text, (txt) => {
currentText.value = txt
})
const avatarSrc = computed(() => {
return props.avatarUrl || '/static/default-avatar.png'
})
function initCanvas() {
if (!isH5.value) return
const canvas = document.getElementById('dh-mouth')
if (!canvas) return
// Size the canvas to match mouth area (~40% width, ~15% height, centered bottom)
const parent = canvas.parentElement
if (!parent) return
const rect = parent.getBoundingClientRect()
canvas.width = rect.width * 0.4
canvas.height = rect.height * 0.15
canvas.style.width = canvas.width + 'px'
canvas.style.height = canvas.height + 'px'
canvas.style.position = 'absolute'
canvas.style.bottom = '18%'
canvas.style.left = '30%'
const query = uni.createSelectorQuery()
query.select('#' + canvasId).fields({ node: true, size: true }).exec((res) => {
if (!res || !res[0]) return
const canvas = res[0].node
const ctx = canvas.getContext('2d')
const dpr = uni.getSystemInfoSync().pixelRatio
canvas.width = canvasSize * dpr
canvas.height = canvasSize * dpr
ctx.scale(dpr, dpr)
faceCtx = ctx
faceW = canvasSize
faceH = canvasSize
drawFace(ctx, 0, false)
})
}
function drawMouth(openRatio) {
if (!isH5.value) return
const canvas = document.getElementById('dh-mouth')
if (!canvas) return
const ctx = canvas.getContext('2d')
function drawFace(ctx, mouthRatio, isBlinking) {
if (!ctx) return
const w = faceW
const h = faceH
const cx = w / 2
const cy = h / 2
const w = canvas.width
const h = canvas.height
ctx.clearRect(0, 0, w, h)
const mouthH = Math.max(2, h * openRatio)
const mouthW = w * 0.8
// Background
ctx.fillStyle = '#F0F2F5'
ctx.beginPath()
ctx.arc(cx, cy, w / 2, 0, Math.PI * 2)
ctx.fill()
// Hair - brown wavy hair
ctx.fillStyle = '#3D2B1F'
ctx.beginPath()
ctx.ellipse(cx, cy - h * 0.28, w * 0.36, h * 0.22, 0, Math.PI, 0)
ctx.fill()
// Hair sides
ctx.beginPath()
ctx.ellipse(cx - w * 0.24, cy - h * 0.08, w * 0.08, h * 0.22, -0.2, 0, Math.PI * 2)
ctx.fill()
ctx.beginPath()
ctx.ellipse(cx + w * 0.24, cy - h * 0.08, w * 0.08, h * 0.22, 0.2, 0, Math.PI * 2)
ctx.fill()
// Face oval
const faceW_ = w * 0.32
const faceH_ = h * 0.38
ctx.fillStyle = '#FDE8D0'
ctx.beginPath()
ctx.ellipse(cx, cy + h * 0.02, faceW_, faceH_, 0, 0, Math.PI * 2)
ctx.fill()
// Neck
ctx.fillStyle = '#FDE8D0'
ctx.beginPath()
ctx.ellipse(cx, cy + h * 0.32, w * 0.14, h * 0.10, 0, 0, Math.PI)
ctx.fill()
// Collar / shoulders suggestion
ctx.fillStyle = '#4B5563'
ctx.beginPath()
ctx.ellipse(cx, cy + h * 0.42, w * 0.28, h * 0.08, 0, Math.PI, 0)
ctx.fill()
ctx.fillStyle = '#374151'
ctx.beginPath()
ctx.ellipse(cx, cy + h * 0.44, w * 0.30, h * 0.06, 0, Math.PI, 0)
ctx.fill()
// Cheek blush
ctx.fillStyle = 'rgba(255, 150, 150, 0.3)'
ctx.beginPath()
ctx.ellipse(cx - w * 0.18, cy + h * 0.08, w * 0.07, h * 0.04, 0, 0, Math.PI * 2)
ctx.fill()
ctx.beginPath()
ctx.ellipse(cx + w * 0.18, cy + h * 0.08, w * 0.07, h * 0.04, 0, 0, Math.PI * 2)
ctx.fill()
// Nose
ctx.strokeStyle = '#E8C8A8'
ctx.lineWidth = 2
ctx.beginPath()
ctx.moveTo(cx, cy - h * 0.02)
ctx.lineTo(cx - w * 0.03, cy + h * 0.06)
ctx.stroke()
ctx.beginPath()
ctx.moveTo(cx, cy - h * 0.02)
ctx.lineTo(cx + w * 0.03, cy + h * 0.06)
ctx.stroke()
// Nose tip
ctx.fillStyle = '#E8C8A8'
ctx.beginPath()
ctx.ellipse(cx, cy + h * 0.06, w * 0.025, h * 0.015, 0, 0, Math.PI * 2)
ctx.fill()
// Eyes
const eyeY = cy - h * 0.02
const eyeSpacing = w * 0.13
function drawEye(x, y, blinkRatio) {
// Eye white
const eyeW = w * 0.09
const eyeH = h * 0.05
const openH = eyeH * (1 - blinkRatio)
ctx.fillStyle = '#FFFFFF'
ctx.beginPath()
ctx.ellipse(x, y, eyeW, Math.max(openH, 1), 0, 0, Math.PI * 2)
ctx.fill()
if (blinkRatio < 0.8) {
// Iris
const irisR = eyeW * 0.55
ctx.fillStyle = '#5B4033'
ctx.beginPath()
ctx.arc(x, y, irisR, 0, Math.PI * 2)
ctx.fill()
// Pupil
ctx.fillStyle = '#1A1A1A'
ctx.beginPath()
ctx.arc(x, y, irisR * 0.5, 0, Math.PI * 2)
ctx.fill()
// Eye highlight
ctx.fillStyle = '#FFFFFF'
ctx.beginPath()
ctx.arc(x - irisR * 0.3, y - irisR * 0.3, irisR * 0.3, 0, Math.PI * 2)
ctx.fill()
ctx.beginPath()
ctx.arc(x + irisR * 0.15, y + irisR * 0.15, irisR * 0.15, 0, Math.PI * 2)
ctx.fill()
// Upper eyelid line
ctx.strokeStyle = '#4A3728'
ctx.lineWidth = 1.5
ctx.beginPath()
const lidH = eyeH * 0.5 * (1 - blinkRatio)
ctx.ellipse(x, y - lidH * 0.3, eyeW * 0.85, lidH * 0.8, 0, Math.PI, 0)
ctx.stroke()
}
// Eyelid (covers eye when blinking)
if (blinkRatio > 0) {
ctx.fillStyle = '#FDE8D0'
ctx.beginPath()
ctx.ellipse(x, y - eyeH * 0.3, eyeW * 1.1, eyeH * 0.7 * blinkRatio + 1, 0, Math.PI, 0)
ctx.fill()
ctx.beginPath()
ctx.ellipse(x, y + eyeH * 0.3, eyeW * 1.1, eyeH * 0.7 * blinkRatio + 1, 0, 0, Math.PI)
ctx.fill()
}
// Eyelash line
ctx.strokeStyle = '#3D2B1F'
ctx.lineWidth = 1.2
ctx.beginPath()
ctx.ellipse(x, y, eyeW * 0.9, Math.max(openH * 0.7, 1), 0, 0, Math.PI * 2)
ctx.stroke()
}
const blinkRatio = isBlinking ? 0.95 : 0
drawEye(cx - eyeSpacing, eyeY, blinkRatio)
drawEye(cx + eyeSpacing, eyeY, blinkRatio)
// Eyebrows
function drawEyebrow(x, y) {
ctx.strokeStyle = '#3D2B1F'
ctx.lineWidth = 3
ctx.lineCap = 'round'
ctx.beginPath()
ctx.moveTo(x - w * 0.06, y - h * 0.06)
ctx.quadraticCurveTo(x, y - h * 0.09, x + w * 0.06, y - h * 0.06)
ctx.stroke()
}
drawEyebrow(cx - eyeSpacing, eyeY)
drawEyebrow(cx + eyeSpacing, eyeY)
// Mouth
const mouthX = cx
const mouthY = cy + h * 0.16
const mouthBaseW = w * 0.10
const mouthBaseH = h * 0.02
const openAmount = mouthRatio * h * 0.06
ctx.fillStyle = '#C97B84'
ctx.beginPath()
ctx.ellipse(w / 2, h / 2 + (h - mouthH) / 2, mouthW / 2, mouthH / 2, 0, 0, Math.PI * 2)
ctx.ellipse(mouthX, mouthY + openAmount * 0.5, mouthBaseW + mouthRatio * w * 0.02, mouthBaseH + openAmount, 0, 0, Math.PI * 2)
ctx.fill()
if (openRatio > 0.1) {
ctx.fillStyle = '#2D1B1E'
// Mouth line (when closed)
if (mouthRatio < 0.1) {
ctx.strokeStyle = '#B06570'
ctx.lineWidth = 1.5
ctx.beginPath()
ctx.ellipse(w / 2, h / 2 + (h - mouthH) / 2 + 1, mouthW / 4, mouthH / 4, 0, 0, Math.PI * 2)
ctx.moveTo(mouthX - mouthBaseW, mouthY)
ctx.quadraticCurveTo(mouthX, mouthY + 2, mouthX + mouthBaseW, mouthY)
ctx.stroke()
}
// Inner mouth when open
if (mouthRatio > 0.15) {
ctx.fillStyle = '#5C2E36'
ctx.beginPath()
ctx.ellipse(mouthX, mouthY + openAmount * 0.6, mouthBaseW * 0.7, openAmount * 0.7, 0, 0, Math.PI * 2)
ctx.fill()
}
// Subtle smile lines
ctx.strokeStyle = '#E8C8A8'
ctx.lineWidth = 1
ctx.beginPath()
ctx.moveTo(mouthX - mouthBaseW - w * 0.02, mouthY - h * 0.01)
ctx.quadraticCurveTo(mouthX - mouthBaseW - w * 0.01, mouthY - h * 0.005, mouthX - mouthBaseW, mouthY)
ctx.stroke()
ctx.beginPath()
ctx.moveTo(mouthX + mouthBaseW + w * 0.02, mouthY - h * 0.01)
ctx.quadraticCurveTo(mouthX + mouthBaseW + w * 0.01, mouthY - h * 0.005, mouthX + mouthBaseW, mouthY)
ctx.stroke()
}
function scheduleBlink() {
const delay = 2000 + Math.random() * 3000
blinkTimer = setTimeout(() => {
doBlink()
scheduleBlink()
}, delay)
}
function doBlink() {
if (!faceCtx) return
let frame = 0
const totalFrames = 8
function blinkTick() {
frame++
const progress = frame / totalFrames
const blinkAmt = progress < 0.5 ? progress * 2 : (1 - progress) * 2
drawFace(faceCtx, mouthOpen, true)
if (frame < totalFrames) {
setTimeout(blinkTick, 30)
} else {
drawFace(faceCtx, mouthOpen, false)
}
}
blinkTick()
}
function animateLoop() {
if (!faceCtx) return
// Smooth mouth movement
mouthOpen += (targetMouth - mouthOpen) * 0.15
if (Math.abs(mouthOpen - targetMouth) < 0.01) mouthOpen = targetMouth
drawFace(faceCtx, mouthOpen, false)
animFrameId = setTimeout(() => animateLoop(), 50)
}
async function playAudio(url) {
@@ -118,75 +337,86 @@ async function playAudio(url) {
isSpeaking.value = true
emit('speaking-start')
// Start mouth animation
targetMouth = 0.4
animateLoop()
try {
audioEl = new Audio(url)
// Use uni-app audio API
const innerAudio = uni.createInnerAudioContext()
audioEl = innerAudio
innerAudio.src = url
innerAudio.autoplay = true
if (isH5.value) {
audioCtx = new (window.AudioContext || window.webkitAudioContext)()
const source = audioCtx.createMediaElementSource(audioEl)
analyser = audioCtx.createAnalyser()
analyser.fftSize = 256
source.connect(analyser)
analyser.connect(audioCtx.destination)
}
innerAudio.onPlay(() => {
// Start audio-driven mouth animation
startAudioMouth(innerAudio)
})
audioEl.onended = () => {
innerAudio.onError(() => {
finishSpeaking()
}
})
audioEl.onerror = () => {
innerAudio.onEnded(() => {
finishSpeaking()
}
})
await audioEl.play()
innerAudio.onStop(() => {
finishSpeaking()
})
if (analyser) {
animateMouth()
}
} catch (e) {
finishSpeaking()
}
}
function animateMouth() {
if (!analyser) return
const dataArray = new Uint8Array(analyser.frequencyBinCount)
function startAudioMouth(audio) {
let lastTime = 0
function tick() {
if (!isSpeaking.value) return
analyser.getByteFrequencyData(dataArray)
const sum = dataArray.reduce((a, b) => a + b, 0)
const avg = sum / dataArray.length
mouthScale = Math.min(1, avg / 128)
// Smooth
mouthScale = Math.max(0.05, mouthScale)
drawMouth(mouthScale)
animFrameId = requestAnimationFrame(tick)
}
try {
const currentTime = audio.currentTime
if (currentTime > lastTime) {
lastTime = currentTime
// Random-ish mouth movement based on time for natural feel
const t = currentTime * 8
const amp = 0.2 + Math.abs(Math.sin(t)) * 0.3 + Math.abs(Math.sin(t * 1.7)) * 0.2
targetMouth = Math.min(0.8, amp)
}
} catch {}
if (isSpeaking.value) {
setTimeout(tick, 80)
}
}
tick()
}
function finishSpeaking() {
isSpeaking.value = false
emit('speaking-end')
if (analyser) {
mouthScale = 0
drawMouth(0)
}
targetMouth = 0
setTimeout(() => {
if (faceCtx && !isSpeaking.value) {
mouthOpen = 0
drawFace(faceCtx, 0, false)
}
}, 300)
cleanupAudio()
emit('speaking-end')
}
function cleanupAudio() {
if (animFrameId) { cancelAnimationFrame(animFrameId); animFrameId = null }
if (audioCtx) { audioCtx.close().catch(() => {}); audioCtx = null }
analyser = null
audioEl = null
if (animFrameId) { clearTimeout(animFrameId); animFrameId = null }
if (audioEl) {
try { audioEl.stop(); audioEl.destroy() } catch {}
audioEl = null
}
}
function stopAudio() {
if (audioEl) {
try { audioEl.pause(); audioEl.src = '' } catch {}
try { audioEl.stop(); audioEl.destroy() } catch {}
}
finishSpeaking()
}
@@ -199,56 +429,36 @@ defineExpose({ play: playAudio, stop: stopAudio })
display: flex;
flex-direction: column;
align-items: center;
padding: 20rpx 0;
padding: 16rpx 0 8rpx;
}
/* Avatar stage */
.avatar-stage {
display: flex;
flex-direction: column;
align-items: center;
gap: 12rpx;
}
.avatar-ring {
width: 200rpx;
height: 200rpx;
border-radius: 50%;
border: 4rpx solid #E5E7EB;
overflow: hidden;
gap: 10rpx;
position: relative;
transition: border-color 0.3s, box-shadow 0.3s;
}
.avatar-ring.speaking {
border-color: #6366F1;
box-shadow: 0 0 30rpx rgba(99, 102, 241, 0.3);
.face-canvas {
border-radius: 50%;
transition: box-shadow 0.3s;
}
.avatar-img {
width: 100%;
height: 100%;
.face-canvas.speaking {
box-shadow: 0 0 40rpx rgba(99, 102, 241, 0.4);
}
.avatar-default {
width: 100%;
height: 100%;
background: linear-gradient(135deg, #6366F1, #8B5CF6);
display: flex;
align-items: center;
justify-content: center;
.status-dot {
width: 14rpx;
height: 14rpx;
border-radius: 50%;
background: #9CA3AF;
position: absolute;
top: 12rpx;
right: 12rpx;
transition: background 0.3s;
}
.avatar-initials {
font-size: 48rpx;
font-weight: 700;
color: #FFFFFF;
.status-dot.active {
background: #10B981;
box-shadow: 0 0 12rpx rgba(16, 185, 129, 0.6);
}
.mouth-canvas {
pointer-events: none;
}
.role-label {
font-size: 22rpx;
color: #6B7280;
@@ -256,23 +466,19 @@ defineExpose({ play: playAudio, stop: stopAudio })
padding: 4rpx 20rpx;
border-radius: 20rpx;
}
/* Speech bubble */
.speech-area {
margin-top: 24rpx;
margin-top: 16rpx;
padding: 0 40rpx;
width: 100%;
box-sizing: border-box;
}
.speech-bubble {
background: #FFFFFF;
border-radius: 16rpx;
padding: 24rpx 28rpx;
padding: 20rpx 24rpx;
box-shadow: 0 4rpx 20rpx rgba(0, 0, 0, 0.06);
position: relative;
}
.speech-bubble::before {
content: '';
position: absolute;
@@ -282,10 +488,9 @@ defineExpose({ play: playAudio, stop: stopAudio })
border-bottom-color: #FFFFFF;
border-top: none;
}
.speech-text {
font-size: 28rpx;
line-height: 1.7;
font-size: 26rpx;
line-height: 1.6;
color: #1F2937;
}
</style>