<template>
<view class="container">
<!-- 背景视频 -->
<!-- 双视频缓冲区 -->
<video v-if="videoBuffers[0]" v-show="currentVideoIndex === 0" :src="videoBuffers[0]" autoplay loop muted
class="background-video" :class="{ 'video-active': currentVideoIndex === 0 }" @loadeddata="onVideoLoaded(0)"
ref="video0" />
<video v-if="videoBuffers[1]" v-show="currentVideoIndex === 1" :src="videoBuffers[1]" autoplay loop muted
class="background-video" :class="{ 'video-active': currentVideoIndex === 1 }" @loadeddata="onVideoLoaded(1)"
ref="video1" />
<!-- 回显数据展示区域 -->
<view class="message-container">
<view v-for="(message, index) in messages" :key="index" :class="{ 'user-css': message.isUser }"
class="message-item">
<text class="message-text" :class="{ 'user-message': message.isUser }">{{ message.text }}</text>
</view>
</view>
<!-- 底部操作栏 -->
<view class="bottom-bar">
<view class="bottom-input">
<!-- 文本输入框 -->
<input type="text" v-model="inputText" placeholder="说任何东西..." class="input-box" />
<!-- 发送按钮 -->
<button @click="sendText" class="send-btn">
<text class="send-text">→</text>
</button>
</view>
<!-- 录音按钮 -->
<image :src="isRecording ? '/static/removeRm.png' : '/static/starRm.png'" mode="" @click="toggleRecording"
class="record-btn"></image>
</view>
<!-- 指示器 -->
<view v-if="isRecording" class="recording-indicator">
<view class="recording-circle"></view>
</view>
</view>
</template>
<script>
import {
Recorder
} from 'opus-recorder';
export default {
data() {
return {
isRecording: false,
audioChunks: [],
recorder: null,
stream: null,
socketTask: null,
isConnected: false,
reconnectAttempts: 0,
maxReconnectAttempts: 5,
reconnectDelay: 1000,
inputText: "", // 输入的文本
messages: [], // 存储所有消息的数组
recorderManager: null, // 录音管理器
// 视频缓冲相关
videoBuffers: ["", ""], // 双缓冲区
currentVideoIndex: 0, // 当前显示的视频索引
videoMap: {
['开心']: "../../static/RoleA-00.mp4",
['你好呀']: "../../static/RoleA-01.mp4",
['有什么我可以帮你的吗?']: "../../static/RoleA-01.mp4",
default: "../../static/RoleA-00.mp4"
},
preloadedVideos: new Map(), // 预加载的视频缓存
isTransitioning: false, // 是否正在切换
videosLoaded: false, // 视频是否初始化完成
currentSessionId: 0, // 当前会话ID,用于区分不同的对话轮次
lastProcessedMessages: new Set(), // 存储当前会话中已处理的消息
autoScroll: true, // 控制是否自动滚动
isUserScrolling: false, // 用户是否正在手动滚动
maxMessages: 50, // 限制消息数量
messageCleanupThreshold: 100, // 清理阈值
};
},
onLoad() {
// 页面加载时建立 WebSocket 连接
this.connectWebSocket();
// #ifdef APP-PLUS
this.recorderManager = uni.getRecorderManager();
this.recorderManager.onStop((res) => {
console.log("录音停止,文件路径:", res.tempFilePath);
// 读取文件并发送二进制数据
uni.getFileSystemManager().readFile({
filePath: res.tempFilePath,
success: (r) => {
this.sendAudioChunk(r.data);
},
fail: (err) => console.error("读取音频文件失败", err),
});
this.sendListenStopMessage();
});
// #endif
// 初始化视频系统
this.initVideoSystem();
// 监听用户滚动行为
this.addScrollListener();
},
methods: {
// 初始化视频系统
async initVideoSystem() {
try {
// 立即设置初始视频
const initialVideo = this.videoMap["default"];
this.videoBuffers[0] = initialVideo;
this.videosLoaded = true;
// 在后台预加载其他视频
setTimeout(() => {
this.preloadAllVideos();
}, 100);
console.log("视频系统初始化完成");
} catch (error) {
console.error("视频系统初始化失败:", error);
}
},
// 视频预加载策略
async preloadAllVideos() {
const videoUrls = [...new Set(Object.values(this.videoMap))];
// 使用Promise.allSettled而不是for循环,提高并发性
const results = await Promise.allSettled(
videoUrls.map(url => this.preloadSingleVideo(url))
);
// 记录失败的视频
const failed = results.filter(r => r.status === 'rejected');
if (failed.length > 0) {
console.warn(`${failed.length}个视频预加载失败`);
}
console.log(`视频预加载完成: ${results.length - failed.length}/${results.length}`);
},
// 预加载单个视频
preloadSingleVideo(url) {
return new Promise((resolve, reject) => {
if (this.preloadedVideos.has(url)) {
resolve();
return;
}
// #ifdef H5
const video = document.createElement('video');
video.src = url;
video.preload = "auto";
video.muted = true;
video.loop = true;
video.addEventListener('canplaythrough', () => {
this.preloadedVideos.set(url, video);
resolve();
});
video.addEventListener('error', (e) => {
console.error(`视频预加载失败: ${url}`, e);
reject(e);
});
video.load();
// #endif
// #ifdef APP-PLUS
// 小程序端直接标记为已预加载
this.preloadedVideos.set(url, true);
resolve();
// #endif
});
},
// 设置视频URL(简化版本)
setVideoUrl(scene) {
if (this.isTransitioning) return;
const newVideoUrl = this.videoMap[scene] || this.videoMap["default"];
const currentUrl = this.videoBuffers[this.currentVideoIndex];
if (currentUrl === newVideoUrl) return;
this.isTransitioning = true;
// 如果当前只有一个视频,直接替换
if (!this.videoBuffers[1 - this.currentVideoIndex]) {
this.videoBuffers[1 - this.currentVideoIndex] = newVideoUrl;
// 等待 DOM 更新后切换
this.$nextTick(() => {
setTimeout(() => {
this.currentVideoIndex = 1 - this.currentVideoIndex;
this.isTransitioning = false;
}, 300);
});
} else {
// 双缓冲切换
this.smoothTransition(newVideoUrl);
}
},
// 平滑过渡动画(简化版)
smoothTransition(newVideoUrl) {
const nextIndex = 1 - this.currentVideoIndex;
this.videoBuffers[nextIndex] = newVideoUrl;
// 强制更新 DOM
this.$forceUpdate();
// 延迟切换,确保过渡动画完成
setTimeout(() => {
this.currentVideoIndex = nextIndex;
this.isTransitioning = false;
}, 300); // 过渡动画时长
},
// 视频加载完成
onVideoLoaded(index) {
console.log(`视频${index}加载完成`);
},
// 建立 WebSocket 连接
connectWebSocket() {
const wsUrl =
"ws://106.13.215.232:8000/aiservers/v1/?device-id=1A:34:B4:02:5D:F6&client-id=web_test_client"; // 替换为你的 WebSocket 服务器地址
this.socketTask = uni.connectSocket({
url: wsUrl,
success: () => {
console.log("WebSocket连接成功");
},
fail: (err) => {
console.error("WebSocket连接失败:", err);
}
});
// 监听 WebSocket 连接打开事件
uni.onSocketOpen((res) => {
this.isConnected = true;
console.log("WebSocket连接已打开:", res);
this.sendHello();
});
// 监听 WebSocket 接收到消息事件
uni.onSocketMessage((res) => {
this.handleMessage(res);
});
// 监听 WebSocket 连接关闭事件
uni.onSocketClose((res) => {
this.isConnected = false;
console.log("WebSocket连接已关闭:", res);
if (this.reconnectAttempts < this.maxReconnectAttempts) {
setTimeout(() => {
this.reconnectAttempts++;
console.log(
`尝试重连... (${this.reconnectAttempts}/${this.maxReconnectAttempts})`);
this.connectWebSocket();
}, this.reconnectDelay * this.reconnectAttempts);
} else {
this.addMessage("系统消息", "连接已断开,请刷新页面重试", false);
}
});
// 监听 WebSocket 错误事件
uni.onSocketError((res) => {
console.error("WebSocket发生错误:", res);
});
},
// 发送握手消息
sendHello() {
const message = {
type: "hello",
device_id: "1A:34:B4:02:5D:F6",
device_name: "UniApp设备",
device_mac: "1A:34:B4:02:5D:F6",
token: "your-token1",
features: {
mcp: true
}
};
uni.sendSocketMessage({
data: JSON.stringify(message),
success: () => {
console.log("握手消息发送成功");
},
fail: (err) => {
console.error("握手消息发送失败:", err);
}
});
},
// 发送文本消息
sendText() {
if (!this.isConnected) {
this.addMessage("系统消息", "请先建立 WebSocket 连接", false);
return;
}
// 开始新的会话轮次
this.startNewSession();
const message = {
type: "listen",
mode: "manual",
state: "detect",
text: this.inputText
};
uni.sendSocketMessage({
data: JSON.stringify(message),
success: () => {
console.log("文本消息发送成功");
this.addMessage("用户", this.inputText, true);
},
fail: (err) => {
console.error("文本消息发送失败:", err);
this.addMessage("系统消息", "消息发送失败", false);
}
});
this.inputText = ""; // 清空输入框
},
// 滚动监听器
addScrollListener() {
this.$nextTick(() => {
// #ifdef H5
const messageContainer = document.querySelector('.message-container');
if (messageContainer) {
messageContainer.addEventListener('scroll', this.handleScroll);
}
// #endif
});
},
// 处理滚动事件
handleScroll(event) {
const container = event.target;
const isAtBottom = container.scrollTop + container.clientHeight >= container.scrollHeight - 10;
// 如果用户滚动到底部,启用自动滚动
// 如果用户向上滚动,暂停自动滚动
this.autoScroll = isAtBottom;
},
// 处理服务器返回的消息
handleMessage(res) {
if (typeof res.data === 'string') {
try {
const jsonMessage = JSON.parse(res.data);
this.handleJsonMessage(jsonMessage);
} catch (error) {
console.error('解析 JSON 失败:', error);
}
} else if (res.data instanceof ArrayBuffer) {
this.handleAudioData(res.data);
}
},
// 处理 JSON 消息
handleJsonMessage(message) {
switch (message.type) {
case 'stt':
// this.addMessage("语音识别", message.text, false);
break;
case 'llm':
this.addMessage("回复消息", message.text, false);
break;
case 'tts':
if ((message.state === 'sentence_start' || message.state === 'sentence_end') && message.text !==
undefined) {
this.addMessage("语音合成", message.text, false);
this.setVideoUrl(message.text)
}
break;
default:
console.log('未知类型消息:', message);
}
},
// 添加消息
addMessage(name, text, isUser) {
// 为每个消息创建唯一标识符
const messageKey = `${this.currentSessionId}-${name}-${text}-${isUser}`;
// 检查是否是同一会话中的重复消息
if (this.lastProcessedMessages.has(messageKey)) {
console.log("同一会话中的重复消息,未添加:", text);
return;
}
// 添加到已处理消息集合
this.lastProcessedMessages.add(messageKey);
// 添加消息到数组
this.messages.push({
name,
text,
isUser,
time: new Date().toLocaleTimeString(),
sessionId: this.currentSessionId // 添加会话ID标识
});
console.log("消息已添加:", text);
// 只有在自动滚动启用时才滚动到底部
if (this.autoScroll) {
this.scrollToBottom();
}
// 内存管理:限制消息数量
if (this.messages.length > this.messageCleanupThreshold) {
this.messages.splice(0, this.messages.length - this.maxMessages);
// 同时清理对应的已处理消息记录
this.cleanupProcessedMessages();
}
},
cleanupProcessedMessages() {
// 只保留最近的消息标识符
const recentMessages = new Set();
this.messages.slice(-this.maxMessages).forEach(msg => {
const key = `${msg.sessionId}-${msg.name}-${msg.text}-${msg.isUser}`;
recentMessages.add(key);
});
this.lastProcessedMessages = recentMessages;
},
// 滚动到底部
scrollToBottom() {
// 使用 nextTick 确保 DOM 更新完成后再滚动
this.$nextTick(() => {
// #ifdef H5
const messageContainer = document.querySelector('.message-container');
if (messageContainer) {
messageContainer.scrollTop = messageContainer.scrollHeight;
}
// #endif
// #ifdef APP-PLUS || MP-WEIXIN
const query = uni.createSelectorQuery().in(this);
query.select('.message-container').boundingClientRect((rect) => {
if (rect) {
uni.pageScrollTo({
selector: '.message-container',
scrollTop: rect.height,
duration: 300 // 平滑滚动动画时间
});
}
}).exec();
// #endif
});
},
// 处理音频数据
handleAudioData(arrayBuffer) {
// this.playAudio(arrayBuffer);
},
// 播放音频数据
playAudio(arrayBuffer) {
const audioContext = new(window.AudioContext || window.webkitAudioContext)();
audioContext.decodeAudioData(arrayBuffer, (audioBuffer) => {
const source = audioContext.createBufferSource();
source.buffer = audioBuffer;
source.connect(audioContext.destination);
source.start();
});
},
// 切换录音状态
toggleRecording() {
this.isRecording ? this.stopRecording() : this.startRecording();
},
// 开始录音
async startRecording() {
// 开始新的会话轮次
this.startNewSession();
// #ifdef APP-PLUS
if (!this.recorderManager) {
this.addMessage("系统消息", "当前平台不支持原生录音", false);
return;
}
this.recorderManager.start({
duration: 60000,
sampleRate: 16000,
numberOfChannels: 1,
encodeBitRate: 96000,
format: "aac",
frameSize: 50,
});
this.isRecording = true;
this.sendListenStartMessage();
return;
// #endif
// #ifdef H5
try {
// 1. 获取麦克风权限
this.stream = await navigator.mediaDevices.getUserMedia({
audio: true
});
// 2. 创建录音器并配置
this.recorder = new MediaRecorder(this.stream, {
mimeType: 'audio/webm;codecs=opus',
audioBitsPerSecond: 16000
});
this.recorder.ondataavailable = async (event) => {
if (event.data && event.data.size > 0) {
// 转换为ArrayBuffer发送
const buffer = await event.data.arrayBuffer();
if (this.isRecording) {
this.sendAudioChunk(buffer);
}
}
};
// 4. 录音停止时的清理
this.recorder.onstop = () => {
this.stream.getTracks().forEach(track => track.stop());
this.sendListenStopMessage();
};
// 5. 开始录音(每100ms发送一次数据块)
this.recorder.start(200);
this.isRecording = true;
// 6. 通知服务器开始接收音频
this.sendListenStartMessage();
} catch (error) {
//TODO handle the exception
console.error('录音启动失败:', error);
}
// #endif
},
// 开始新会话
startNewSession() {
this.currentSessionId++;
this.lastProcessedMessages.clear();
console.log("开始新会话,会话ID:", this.currentSessionId);
},
// 停止录音
stopRecording() {
// #ifdef APP-PLUS
this.recorderManager.stop();
this.isRecording = false;
return;
// #endif
// H5 浏览器端停止
// #ifdef H5
this.recorder.stop();
this.stream.getTracks().forEach((t) => t.stop());
this.isRecording = false;
// #endif
},
// 发送开始录音消息
sendListenStartMessage() {
if (!this.isConnected) return;
const msg = {
type: "listen",
mode: "manual",
state: "start"
};
uni.sendSocketMessage({
data: JSON.stringify(msg),
success: () => console.log("已通知服务器开始录音"),
fail: (err) => console.error("开始录音消息发送失败:", err)
});
},
// 告诉服务器:结束语音流
sendListenStopMessage() {
if (!this.isConnected) return;
const msg = {
type: "listen",
mode: "manual",
state: "stop"
};
uni.sendSocketMessage({
data: JSON.stringify(msg),
success: () => console.log("已通知服务器停止录音"),
fail: (err) => console.error("停止录音消息发送失败:", err)
});
},
// 实时发送音频数据
sendAudioChunk(buffer) {
if (!this.isConnected) return;
uni.sendSocketMessage({
data: buffer,
success: () => {},
fail: (err) => {
console.error("音频数据发送失败:", err);
}
});
}
},
// 页面卸载时关闭 WebSocket 连接
onUnload() {
if (this.socketTask) {
uni.closeSocket();
console.log("关闭 WebSocket 连接");
}
if (this.stream) {
this.stream.getTracks().forEach(track => track.stop());
}
}
};
</script>
<style>
.container {
position: relative;
width: 100%;
height: 100vh;
overflow: hidden;
}
.background-video {
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
object-fit: cover;
z-index: -1;
opacity: 1;
/* 直接设置为1,简化逻辑 */
transition: opacity 0.3s ease-in-out;
animation: fade 0.3s forwards;
}
@keyframes fade {
from {
opacity: 0;
}
to {
opacity: 1;
}
}
.background-video.loaded {
opacity: 1;
/* 加载完成后变为不透明 */
}
.message-container {
position: absolute;
top: 90px;
width: min(70%, 400px);
/* 限制最大宽度 */
right: 20px;
max-height: calc(100vh - 180px);
/* 动态计算高度 */
overflow-y: auto;
z-index: 2;
scroll-behavior: smooth;
}
/* 移动端适配 */
/* @media (max-width: 768px) {
.message-container {
width: calc(100% - 40px);
right: 20px;
left: 20px;
}
} */
.message-container::-webkit-scrollbar {
width: 0px;
}
.message-container::-webkit-scrollbar-track {
background: transparent;
}
.message-container::-webkit-scrollbar-thumb {
background-color: transparent;
border-radius: 0px;
}
.message-container::-webkit-scrollbar-thumb:hover {
background-color: transparent);
}
.message-item {
margin: 0 10px;
}
.message-text {
padding: 10px 15px;
border-radius: 18px;
font-size: 14px;
line-height: 1.4;
word-wrap: break-word;
max-width: 80%;
display: inline-block;
margin-bottom: 8px;
}
.user-css {
text-align: end;
}
.user-message {
background-color: #4CAF50;
color: white;
}
.message-text:not(.user-message) {
background-color: #ffffff;
color: #333333;
}
.bottom-bar {
position: fixed;
bottom: 0;
width: 100%;
display: flex;
justify-content: space-between;
align-items: center;
padding: 10px 0px;
z-index: 10;
box-shadow: 0 -2px 5px rgba(0, 0, 0, 0.1);
}
.bottom-input {
display: flex;
flex: 1;
position: relative;
}
.record-btn {
width: 40px;
height: 40px;
border-radius: 50%;
display: flex;
justify-content: center;
align-items: center;
}
.record-text {
font-size: 14px;
}
.input-box {
flex: 1;
margin: 0 10px;
height: 40px;
border: none;
background-color: #f5f5f5;
border-radius: 20px;
padding: 0 15px;
font-size: 14px;
outline: none;
}
.send-btn {
width: 40px;
height: 40px;
border-radius: 50%;
background-color: #4CAF50;
color: white;
display: flex;
justify-content: center;
align-items: center;
position: absolute;
right: 10px;
}
.send-text {
font-size: 18px;
}
.recording-indicator {
position: fixed;
top: 20px;
right: 20px;
z-index: 100;
}
.recording-circle {
width: 10px;
height: 10px;
background-color: #ff0000;
border-radius: 50%;
animation: blink 1s infinite;
}
@keyframes blink {
0% {
opacity: 1;
}
50% {
opacity: 0.3;
}
100% {
opacity: 1;
}
}
/deep/ .uni-video-bar {
display: none !important;
}
</style>在这个页面使用
最新发布