AI 内容创作 / 系统课时
实时客户端接入
实时客户端接入
本课把浏览器接入实时数字人服务,完成麦克风分片上行、会话初始化、字幕更新、音视频播放、状态呈现与断线重连,并用一个本地可运行的 WebSocket 示例验证整条客户端链路。
学习目标
- 划分采集、控制、字幕、媒体播放和重连模块的职责。
- 使用 WebSocket 文本帧与二进制帧组织实时会话。
- 通过本地示例验证麦克风、状态、字幕和音视频组件。
客户端职责与链路
实时客户端不是单纯的播放器。它一边把麦克风数据按固定时间片上传,一边接收会话状态、增量字幕和媒体结果,还要维护心跳、播放策略与异常恢复。控制消息适合使用 JSON 文本帧,连续音频适合使用二进制帧,服务端可以在握手后把二进制帧绑定到该连接的会话。

客户端状态可按 DISCONNECTED → CONNECTING → READY → LISTENING 或 GENERATING → READY 组织。异常断线可进入 RECONNECTING。真正的恢复需重新鉴权后交换已确认序号和快照;本例仅重建连接并复用测试 ID,断线前的录制与播报会终止。
消息协议与状态约束
下面的协议示例把文本控制帧和二进制媒体帧分开。<ID>、<TIME> 与 sequence 是动态值;生产服务还应校验鉴权令牌、序号单调性和消息大小。
{
"textFrames": [
{
"type": "session.init",
"sessionId": "<ID>"
},
{
"type": "heartbeat.request",
"sentAt": "<TIME>"
},
{
"type": "command.speak",
"text": "欢迎使用实时数字人"
}
],
"binaryFrames": {
"direction": "client_to_server",
"content": "已绑定连接上的 MediaRecorder 分片;示例只计数"
},
"downstreamTypes": [
"session.ready",
"status.changed",
"subtitle.partial",
"subtitle.final",
"media.audio",
"media.video",
"heartbeat.ack"
]
}收到 session.ready 前不要启动业务指令;真正的跨连接字幕恢复需要用服务端版本或序号去重;音频和视频播放要在新一代 generationId 到达后丢弃旧结果。心跳只证明连接仍可读写,不能替代服务端业务状态确认。
完整可运行示例
示例使用 Node.js 和 ws 在回环地址创建服务。服务器只计数,不解码、不保存录音,每四个分片返回一次计数提示;点击“模拟数字人回复”后,下发最终字幕和一段本地生成的提示音。视频区域使用 Canvas 流模拟,真实服务宜接入已授权的 WebRTC 轨道。
将以下内容保存为 server.cjs。
const http = require("http");
const fs = require("fs");
const path = require("path");
const { WebSocketServer } = require("ws");
const PORT = Number(process.env.PORT ?? 3000);
const indexPath = path.join(__dirname, "public", "index.html");
// 生成 0.25 秒提示音,避免依赖外部音频文件。
function createBeepDataUri() {
const rate = 16000;
const count = rate / 4;
const buffer = Buffer.alloc(44 + count * 2);
buffer.write("RIFF", 0);
buffer.writeUInt32LE(36 + count * 2, 4);
buffer.write("WAVEfmt ", 8);
buffer.writeUInt32LE(16, 16);
buffer.writeUInt16LE(1, 20);
buffer.writeUInt16LE(1, 22);
buffer.writeUInt32LE(rate, 24);
buffer.writeUInt32LE(rate * 2, 28);
buffer.writeUInt16LE(2, 32);
buffer.writeUInt16LE(16, 34);
buffer.write("data", 36);
buffer.writeUInt32LE(count * 2, 40);
for (let i = 0; i < count; i += 1) {
const sample = Math.sin((2 * Math.PI * 440 * i) / rate) * 0.2;
buffer.writeInt16LE(Math.floor(sample * 32767), 44 + i * 2);
}
return `data:audio/wav;base64,${buffer.toString("base64")}`;
}
const server = http.createServer((request, response) => {
if (request.method !== "GET" || request.url !== "/") {
response.writeHead(404).end("Not Found");
return;
}
fs.readFile(indexPath, (error, body) => {
if (error) return response.writeHead(500).end("Missing local HTML");
response.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }).end(body);
});
});
const wss = new WebSocketServer({ noServer: true, maxPayload: 256 * 1024 });
server.on("upgrade", (request, socket, head) => {
const origin = "http://127.0.0.1:" + server.address().port;
if (request.url !== "/control" || request.headers.origin !== origin) {
socket.end("HTTP/1.1 403 Forbidden\r\nConnection: close\r\n\r\n");
return;
}
wss.handleUpgrade(request, socket, head, ws => wss.emit("connection", ws));
});
const beepUrl = createBeepDataUri();
wss.on("connection", (socket) => {
let audioChunks = 0;
let bound = false;
let replyTimer;
socket.on("error", error => console.error(error.message));
socket.on("close", () => clearTimeout(replyTimer));
console.log("客户端已连接");
socket.on("message", (data, isBinary) => {
if (isBinary) {
if (!bound) return socket.close(1008, "session not initialized");
audioChunks += 1;
if (audioChunks % 4 === 0) {
socket.send(JSON.stringify({
type: "subtitle.partial",
text: `已接收 ${audioChunks} 个音频分片`
}));
}
return;
}
let message;
try {
message = JSON.parse(data.toString("utf8"));
} catch {
return socket.close(1007, "invalid JSON");
}
if (!message || typeof message.type !== "string") return socket.close(1008, "invalid message");
if (!bound) {
if (message.type !== "session.init" || typeof message.sessionId !== "string" || !message.sessionId) {
return socket.close(1008, "session not initialized");
}
bound = true;
socket.send(JSON.stringify({ type: "session.ready", sessionId: message.sessionId, state: "READY" }));
} else if (message.type === "heartbeat.request") {
socket.send(JSON.stringify({ type: "heartbeat.ack", sentAt: message.sentAt }));
} else if (message.type === "command.speak") {
if (replyTimer || typeof message.text !== "string" || !message.text.trim() || message.text.length > 240) return;
socket.send(JSON.stringify({ type: "status.changed", state: "GENERATING" }));
replyTimer = setTimeout(() => {
replyTimer = null;
if (socket.readyState !== 1) return;
socket.send(JSON.stringify({ type: "subtitle.final", text: message.text }));
socket.send(JSON.stringify({ type: "media.audio", url: beepUrl }));
socket.send(JSON.stringify({ type: "status.changed", state: "READY" }));
}, 300);
}
});
});
server.listen(PORT, "127.0.0.1", () => {
console.log("服务已启动: http://127.0.0.1:" + server.address().port);
});创建 public 目录,并将以下内容保存为 public/index.html。
<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>实时数字人客户端</title>
<style>
body { font-family: sans-serif; max-width: 720px; margin: 32px auto; line-height: 1.6; }
video { width: 100%; background: #eef3f8; }
button { margin: 0 8px 8px 0; padding: 8px 12px; }
#subtitle { min-height: 32px; font-size: 20px; }
</style>
</head>
<body>
<h1>实时数字人客户端</h1>
<p>状态:<strong id="status">DISCONNECTED</strong></p>
<video id="video" autoplay muted playsinline></video>
<p id="subtitle">等待字幕</p>
<p id="notice" role="status"></p>
<button id="start" disabled>开始采集</button>
<button id="stop" disabled>停止采集</button>
<button id="speak" disabled>模拟数字人回复</button>
<audio id="audio" controls></audio>
<script>
const statusNode = document.querySelector("#status");
const subtitleNode = document.querySelector("#subtitle");
const videoNode = document.querySelector("#video");
const audioNode = document.querySelector("#audio");
const startButton = document.querySelector("#start");
const stopButton = document.querySelector("#stop");
const speakButton = document.querySelector("#speak");
const notice = document.querySelector("#notice");
const sessionId = crypto.randomUUID();
let socket;
let recorder;
let captureStream;
let captureEpoch = 0;
let capturePending = false;
let ready = false;
let generating = false;
let reconnectAttempt = 0;
function updateControls() {
startButton.disabled = !ready || Boolean(recorder) || capturePending;
stopButton.disabled = !recorder && !capturePending;
speakButton.disabled = !ready || generating;
}
function setStatus(value) {
statusNode.textContent = value;
}
function send(message) {
if (socket?.readyState === WebSocket.OPEN) {
socket.send(JSON.stringify(message));
}
}
// 每个连接重新握手;不恢复旧连接未确认的录制或播报。
function connect() {
ready = false;
setStatus("CONNECTING");
const connection = new WebSocket("ws://" + location.host + "/control");
socket = connection;
let lastAck = performance.now();
let heartbeat;
const handshake = setTimeout(() => connection.close(4000, "ready timeout"), 5000);
connection.addEventListener("open", () => send({ type: "session.init", sessionId }));
connection.addEventListener("message", event => {
if (socket !== connection) return;
let message;
try {
message = JSON.parse(event.data);
} catch {
connection.close(4000, "invalid server message");
return;
}
if (!message || typeof message.type !== "string") return;
if (!ready && message.type === "session.ready" && message.sessionId === sessionId) {
ready = true;
reconnectAttempt = 0;
clearTimeout(handshake);
lastAck = performance.now();
setStatus("READY");
updateControls();
heartbeat = setInterval(() => {
if (performance.now() - lastAck > 12000) connection.close(4000, "heartbeat timeout");
else send({ type: "heartbeat.request" });
}, 5000);
} else if (ready && message.type === "heartbeat.ack") {
lastAck = performance.now();
} else if (ready && message.type === "status.changed") {
generating = message.state === "GENERATING";
setStatus(generating ? "GENERATING" : recorder ? "LISTENING" : "READY");
updateControls();
} else if (ready && message.type.startsWith("subtitle.")) {
subtitleNode.textContent = message.text;
} else if (ready && message.type === "media.audio" &&
typeof message.url === "string" && message.url.startsWith("data:audio/wav;base64,")) {
audioNode.src = message.url;
audioNode.play().catch(() => {
if (socket === connection && ready) notice.textContent = "请用音频控件播放";
});
}
});
connection.addEventListener("close", event => {
clearTimeout(handshake);
clearInterval(heartbeat);
if (socket !== connection) return;
ready = false;
generating = false;
stopCapture();
audioNode.pause();
audioNode.removeAttribute("src");
audioNode.load();
if (event.code === 1000 || event.code === 1008 || reconnectAttempt >= 5) {
setStatus("DISCONNECTED");
notice.textContent = "连接结束,请刷新页面重新连接";
return;
}
setStatus("RECONNECTING");
const delay = Math.min(1000 * 2 ** reconnectAttempt, 10000);
reconnectAttempt += 1;
setTimeout(connect, delay);
});
}
async function startCapture() {
if (!ready || recorder || capturePending) return;
const connection = socket;
const epoch = ++captureEpoch;
capturePending = true;
updateControls();
let stream;
try {
stream = await navigator.mediaDevices.getUserMedia({
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true },
});
if (epoch !== captureEpoch || !ready || socket !== connection) {
stream.getTracks().forEach(track => track.stop());
return;
}
const preferred = "audio/webm;codecs=opus";
const active = MediaRecorder.isTypeSupported(preferred)
? new MediaRecorder(stream, { mimeType: preferred })
: new MediaRecorder(stream);
recorder = active;
captureStream = stream;
active.addEventListener("dataavailable", event => {
if (recorder !== active || !ready || socket !== connection ||
connection.readyState !== WebSocket.OPEN || !event.data.size) return;
if (connection.bufferedAmount + event.data.size > 256 * 1024) {
stopCapture();
notice.textContent = "网络发送积压,本次采集已停止,请重新开始";
return;
}
connection.send(event.data);
});
active.start(250);
setStatus(generating ? "GENERATING" : "LISTENING");
} catch (error) {
stream?.getTracks().forEach(track => track.stop());
if (epoch === captureEpoch) {
stopCapture();
notice.textContent = "无法开始采集:" + error.name;
}
} finally {
if (epoch === captureEpoch) {
capturePending = false;
updateControls();
}
}
}
function stopCapture() {
captureEpoch += 1;
capturePending = false;
const active = recorder;
recorder = null;
if (active && active.state !== "inactive") active.stop();
captureStream?.getTracks().forEach(track => track.stop());
captureStream = null;
if (ready) setStatus(generating ? "GENERATING" : "READY");
updateControls();
}
// Canvas 流用于验证视频元素;接入后时由 media.video 替换。
function startVideoPreview() {
const canvas = document.createElement("canvas");
canvas.width = 640;
canvas.height = 360;
const context = canvas.getContext("2d");
let frame = 0;
setInterval(() => {
frame += 1;
context.fillStyle = "#eef3f8";
context.fillRect(0, 0, 640, 360);
context.fillStyle = "#174a7e";
context.beginPath();
context.arc(320, 170, 52 + Math.sin(frame / 5) * 8, 0, Math.PI * 2);
context.fill();
context.fillStyle = "#17324d";
context.font = "24px sans-serif";
context.fillText("数字人视频流预览", 220, 280);
}, 100);
videoNode.srcObject = canvas.captureStream(10);
}
startButton.addEventListener("click", startCapture);
stopButton.addEventListener("click", stopCapture);
speakButton.addEventListener("click", () => {
send({ type: "command.speak", text: "实时客户端接入成功" });
});
startVideoPreview();
connect();
</script>
</body>
</html>安装依赖并启动服务。
npm init -y
npm install ws@8
node server.cjs预期输出:
服务已启动: http://127.0.0.1:3000
客户端已连接运行与验收
浏览器打开 http://127.0.0.1:3000 后,页面状态应从 DISCONNECTED 变为 READY。点击“开始采集”并授权麦克风,状态变为 LISTENING,持续说话后字幕区按四个分片一次更新。点击“模拟数字人回复”,页面先显示 GENERATING,随后播放提示音、显示最终字幕并回到 READY;若采集还在进行,则回到 LISTENING。
关闭服务端进程后,页面应进入 RECONNECTING,延迟按 1、2、4、8、10 秒上限递增。五次尝试内恢复服务即可重新握手。断线会停掉采集,重连后需由用户重新启动。浏览器若阻止自动播放,页面会保留音频控件。
稳定性与安全边界
生产环境必须使用 wss://,并在握手或 session.init 中完成短期令牌鉴权。麦克风分片要限制最大尺寸和发送速率,避免网络变慢时无限积压;重连期间应暂停上传或只保留有限缓冲,是否续传取决于服务器确认、容器分片协议与当前录制代次;示例一律停止旧录制,手动停止时也不上传最后的关闭分片。
MediaRecorder 的容器和编码支持存在浏览器差异,客户端可以用 MediaRecorder.isTypeSupported() 检查优选 MIME;不支持时让浏览器选择,并把实际 mimeType 告知真实接收端。timeslice 只是期望间隔,分片不保证独立可解码。示例的计数提示不代表 ASR 结果。真实视频若使用 WebRTC,应把 WebSocket 保留为控制和状态通道,不要把高码率视频长期塞入同一 WebSocket 队列。
本课总结
- 实时客户端需要同时管理采集、协议、状态、字幕、播放和重连。
- JSON 文本帧承载控制事件,二进制帧承载连续音频,二者由会话标识关联。
- 可观测状态、有限缓冲和指数退避是稳定接入的基础。
下一课预告
下一课将进入“音频与语音链路排障”,从采样率、设备占用、音量、截断、识别延迟、异常发音和播放卡顿等现象出发,建立可重复的定位顺序。