digital_human.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482
  1. """
  2. 数字人路由:多轮问答、题库例题检索、DeepSeek 调用、可选 TTS 与语音资源管理。
  3. """
  4. from pathlib import Path
  5. import base64
  6. import os
  7. import tempfile
  8. import random
  9. import re
  10. from typing import Annotated
  11. from uuid import UUID
  12. from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
  13. from fastapi.responses import FileResponse
  14. import httpx
  15. from sqlmodel import Session, select
  16. from app.core.config import settings
  17. from app.db import get_session
  18. from app.deps import get_current_user, require_role
  19. from app.models import ChatMessage, ChatSession, Exercise, User, UserRole
  20. from app.schemas import AskRequest, AskResponse, VoiceAssetItem
  21. from app.utils.exercise_constants import RESOLUTION_EMPTY_PLACEHOLDER
  22. from app.utils.exercise_knowledge import exercise_matches_knowledge_point
  23. router = APIRouter(prefix="/digital-human", tags=["digital-human"])
  24. VOICE_EXTS = {".wav", ".mp3", ".m4a", ".ogg"}
  25. def _voice_assets_dir() -> Path:
  26. p = Path(settings.voice_assets_dir).expanduser()
  27. p.mkdir(parents=True, exist_ok=True)
  28. return p
  29. def _safe_voice_name(name: str) -> str:
  30. base = Path(name).name.strip()
  31. if not base:
  32. raise HTTPException(status_code=400, detail="文件名不能为空")
  33. ext = Path(base).suffix.lower()
  34. if ext not in VOICE_EXTS:
  35. raise HTTPException(status_code=400, detail="仅支持 wav/mp3/m4a/ogg")
  36. return base
  37. def _resolve_local_tts_paths() -> tuple[Path | None, Path | None, str | None]:
  38. ckpt_raw = settings.local_tts_ckpt_path.strip()
  39. pth_raw = settings.local_tts_pth_path.strip()
  40. ckpt = Path(ckpt_raw).expanduser() if ckpt_raw else None
  41. pth = Path(pth_raw).expanduser() if pth_raw else None
  42. base_dir = Path(settings.local_tts_model_dir).expanduser()
  43. base_dir.mkdir(parents=True, exist_ok=True)
  44. if ckpt is None:
  45. candidates = [p for p in base_dir.rglob("*.ckpt") if p.is_file()]
  46. if candidates:
  47. ckpt = sorted(candidates, key=lambda x: x.stat().st_mtime, reverse=True)[0]
  48. if pth is None:
  49. candidates = [p for p in base_dir.rglob("*.pth") if p.is_file()]
  50. if candidates:
  51. pth = sorted(candidates, key=lambda x: x.stat().st_mtime, reverse=True)[0]
  52. hint = None
  53. if ckpt is None or pth is None:
  54. hint = f"请将模型放到 {base_dir},或在 .env 设置 LOCAL_TTS_CKPT_PATH / LOCAL_TTS_PTH_PATH"
  55. return ckpt, pth, hint
  56. def _is_deepseek_selected(model_name: str | None, model_provider: str | None) -> bool:
  57. provider = (model_provider or "").strip().lower()
  58. if provider:
  59. return provider == "deepseek"
  60. if not model_name:
  61. return False
  62. return "deepseek" in model_name.lower()
  63. def _build_default_answer() -> str:
  64. return (
  65. "我先给出一个解题思路:把题目条件转成方程/不等式,再根据知识点逐步推导。"
  66. "如果你把具体题目发完整(含图形/已知量/求什么),我可以一步步讲解。"
  67. )
  68. def _extract_example_limit(question: str) -> int:
  69. """
  70. 粗略解析用户想要的例题数量:优先识别数字,其次识别“两/二”。
  71. """
  72. q = question or ""
  73. m = re.search(r"([1-9]|10)\s*(道|题|个)?", q)
  74. if m:
  75. return int(m.group(1))
  76. if "两" in q or "二" in q:
  77. return 2
  78. return 3
  79. def _question_wants_examples(question: str) -> bool:
  80. """
  81. 判断用户是否在聊天里提出“要例题/举例/给例子”之类诉求。
  82. """
  83. q = (question or "").strip()
  84. if not q:
  85. return False
  86. keywords = [
  87. "例题",
  88. "例子",
  89. "举例",
  90. "示例",
  91. "给我题",
  92. "给我几道",
  93. "来几道",
  94. "来几题",
  95. "给我几题",
  96. "给出几道",
  97. "练习题",
  98. "习题",
  99. "来一题",
  100. "来一道",
  101. "出一题",
  102. "出几道",
  103. "做几道",
  104. "题库",
  105. "抽题",
  106. "换几道题",
  107. ]
  108. return any(k in q for k in keywords)
  109. def _pick_weighted_without_replacement(candidates: list[Exercise], limit: int) -> list[Exercise]:
  110. """
  111. 按 weight 做不放回加权抽样;weight 高的题更容易被抽到。
  112. """
  113. remaining = list(candidates)
  114. picked: list[Exercise] = []
  115. limit = max(1, int(limit))
  116. while remaining and len(picked) < limit:
  117. weights: list[float] = []
  118. for ex in remaining:
  119. w = int(getattr(ex, "weight", 0) or 0)
  120. weights.append(float(w + 1)) # 避免全为 0 时无法抽样
  121. total = sum(weights)
  122. if total <= 0:
  123. chosen = remaining[0]
  124. else:
  125. r = random.random() * total
  126. acc = 0.0
  127. chosen = remaining[-1]
  128. for ex, w in zip(remaining, weights):
  129. acc += w
  130. if r <= acc:
  131. chosen = ex
  132. break
  133. remaining = [ex for ex in remaining if ex.id != chosen.id]
  134. picked.append(chosen)
  135. return picked
  136. def _build_examples_answer(
  137. question: str,
  138. knowledge_code: str | None,
  139. session: Session,
  140. ) -> tuple[str, list[str]] | None:
  141. """
  142. 若用户请求例题,则从题库抽取同一 knowledge_code 的题并输出题干+解析/答案。
  143. 返回 None 表示不触发该逻辑或题库无匹配题目。
  144. """
  145. if not knowledge_code or not knowledge_code.strip():
  146. return None
  147. if not _question_wants_examples(question):
  148. return None
  149. limit = _extract_example_limit(question)
  150. code = knowledge_code.strip()
  151. all_exs = session.exec(select(Exercise)).all()
  152. candidates = [ex for ex in all_exs if exercise_matches_knowledge_point(ex, code)]
  153. if not candidates:
  154. return None
  155. picked = _pick_weighted_without_replacement(candidates, limit)
  156. sources: list[str] = []
  157. parts: list[str] = []
  158. parts.append(f"根据你选择的知识点(`{code}`),我从题库整理了 {len(picked)} 道例题与解析:")
  159. for i, ex in enumerate(picked, start=1):
  160. title = (ex.title or "").strip() or f"例题 {i}"
  161. body = (ex.body or "").strip()
  162. correct_answer = (ex.correct_answer or "").strip() if getattr(ex, "correct_answer", None) else ""
  163. resolution = ((ex.resolution or "").strip() if getattr(ex, "resolution", None) else "") or RESOLUTION_EMPTY_PLACEHOLDER
  164. sources.append(str(ex.id))
  165. parts.append(f"\n{i}. {title}\n题干:{body}")
  166. if correct_answer:
  167. parts.append(f"参考答案:{correct_answer}")
  168. parts.append(f"解析:{resolution}")
  169. return "\n".join(parts).strip(), sources
  170. def _generate_answer_with_deepseek(question: str, knowledge_code: str | None, model_id: str | None) -> str:
  171. api_key = settings.deepseek_api_key.strip()
  172. if not api_key:
  173. return "DeepSeek 未配置 API Key(请在 backend/.env 设置 DEEPSEEK_API_KEY),当前使用占位回复。"
  174. prompt_context = f"当前知识点编码:{knowledge_code}" if knowledge_code else "当前无指定知识点编码"
  175. messages = [
  176. {
  177. "role": "system",
  178. "content": (
  179. "你是一个面向中学生数学辅导的助教。回答要准确、分步骤、尽量简洁。"
  180. "数学公式请使用 LaTeX:行内用 \\( ... \\),独立成行用 \\[ ... \\] 或 $$ ... $$;"
  181. "不要用 Markdown 代码块(```)包裹公式。"
  182. "避免滥用 Markdown 小标题与加粗,以自然段与编号步骤为主,便于阅读。"
  183. ),
  184. },
  185. {"role": "system", "content": prompt_context},
  186. {"role": "user", "content": question},
  187. ]
  188. payload = {
  189. "model": (model_id or "").strip() or settings.deepseek_model,
  190. "messages": messages,
  191. "temperature": 0.5,
  192. }
  193. base_url = settings.deepseek_base_url.rstrip("/")
  194. url = f"{base_url}/chat/completions"
  195. headers = {
  196. "Authorization": f"Bearer {api_key}",
  197. "Content-Type": "application/json",
  198. }
  199. try:
  200. with httpx.Client(timeout=settings.llm_request_timeout_seconds) as client:
  201. res = client.post(url, json=payload, headers=headers)
  202. res.raise_for_status()
  203. data = res.json()
  204. return data["choices"][0]["message"]["content"].strip()
  205. except Exception as e:
  206. return f"DeepSeek 调用失败({e}),当前使用占位回复。"
  207. def _probe_local_tts_model() -> str:
  208. if not settings.local_tts_enabled:
  209. return "未启用本地TTS模型"
  210. ckpt, pth, hint = _resolve_local_tts_paths()
  211. if ckpt is None:
  212. return f"本地TTS不可用:未找到ckpt文件。{hint or ''}".strip()
  213. if pth is None:
  214. return f"本地TTS不可用:未找到pth文件。{hint or ''}".strip()
  215. if not ckpt.exists():
  216. return f"本地TTS不可用:找不到ckpt文件({ckpt})"
  217. if not pth.exists():
  218. return f"本地TTS不可用:找不到pth文件({pth})"
  219. try:
  220. import torch
  221. except ImportError:
  222. return "本地TTS不可用:未安装 torch(可选依赖)"
  223. try:
  224. _ = torch.load(ckpt, map_location="cpu")
  225. except Exception as e:
  226. return f"本地TTS不可用:ckpt加载失败({e})"
  227. try:
  228. _ = torch.load(pth, map_location="cpu", weights_only=False)
  229. except Exception as e:
  230. return f"本地TTS不可用:pth加载失败({e})"
  231. return f"本地TTS模型加载检查通过(ckpt={ckpt.name}, pth={pth.name};已接入检测,尚未执行真实语音合成)"
  232. def _normalize_tts_api_url(raw_url: str) -> str:
  233. url = (raw_url or "").strip().rstrip("/")
  234. if not url:
  235. return ""
  236. if url.endswith("/tts"):
  237. return url
  238. return f"{url}/tts"
  239. def _synthesize_with_http_tts(text: str) -> tuple[str | None, str | None, str]:
  240. raw_url = settings.local_tts_api_url
  241. url = _normalize_tts_api_url(raw_url)
  242. if not url:
  243. return None, None, "本地TTS HTTP未配置:请在 .env 设置 LOCAL_TTS_API_URL"
  244. payload = {
  245. "text": text,
  246. "text_lang": settings.local_tts_text_lang,
  247. }
  248. if settings.local_tts_ref_wav_path.strip():
  249. payload["ref_audio_path"] = settings.local_tts_ref_wav_path.strip()
  250. if settings.local_tts_prompt_text.strip():
  251. payload["prompt_text"] = settings.local_tts_prompt_text.strip()
  252. if settings.local_tts_prompt_lang.strip():
  253. payload["prompt_lang"] = settings.local_tts_prompt_lang.strip()
  254. try:
  255. with httpx.Client(timeout=settings.local_tts_api_timeout_seconds) as client:
  256. res = client.post(url, json=payload)
  257. res.raise_for_status()
  258. except Exception as e:
  259. return None, None, f"本地TTS HTTP调用失败({e})"
  260. content_type = (res.headers.get("content-type") or "").split(";")[0].strip().lower()
  261. if content_type.startswith("audio/"):
  262. try:
  263. b64 = base64.b64encode(res.content).decode("ascii")
  264. return b64, content_type, f"已使用本地TTS HTTP接口合成语音({url})"
  265. except Exception as e:
  266. return None, None, f"本地TTS HTTP音频解析失败({e})"
  267. try:
  268. data = res.json()
  269. except Exception:
  270. return None, None, f"本地TTS HTTP返回非音频且非JSON(content-type={content_type or 'unknown'})"
  271. audio_b64 = None
  272. audio_mime = "audio/wav"
  273. if isinstance(data, dict):
  274. if isinstance(data.get("audio_base64"), str):
  275. audio_b64 = data["audio_base64"]
  276. audio_mime = data.get("audio_mime") or audio_mime
  277. elif isinstance(data.get("audio"), str):
  278. audio_b64 = data["audio"]
  279. audio_mime = data.get("mime") or audio_mime
  280. elif isinstance(data.get("data"), dict) and isinstance(data["data"].get("audio_base64"), str):
  281. audio_b64 = data["data"]["audio_base64"]
  282. audio_mime = data["data"].get("audio_mime") or audio_mime
  283. if not audio_b64:
  284. return None, None, "本地TTS HTTP返回JSON但未找到 audio_base64/audio 字段"
  285. return audio_b64, audio_mime, f"已使用本地TTS HTTP接口合成语音({url})"
  286. def _synthesize_audio_base64(text: str) -> tuple[str | None, str | None, str]:
  287. provider = settings.local_tts_provider.strip().lower()
  288. http_fail: str | None = None
  289. # GPT-SoVITS HTTP 模式不需要本地 ckpt/pth;原先先探测权重会导致未放模型时永远不请求 HTTP。
  290. if settings.local_tts_enabled and provider in {"gpt_sovits_http", "http", "api"}:
  291. audio_b64, audio_mime, status = _synthesize_with_http_tts(text)
  292. if audio_b64:
  293. return audio_b64, audio_mime, status
  294. http_fail = status
  295. probe = _probe_local_tts_model()
  296. if "加载检查通过" not in probe:
  297. if http_fail:
  298. return None, None, f"{http_fail};{probe}"
  299. return None, None, probe
  300. if http_fail:
  301. probe = f"{probe};{http_fail};已回退到后端系统语音。"
  302. try:
  303. import pyttsx3 # type: ignore
  304. except Exception:
  305. return None, None, f"{probe};未安装pyttsx3,已回退到前端浏览器语音。"
  306. tmp_path = ""
  307. try:
  308. fd, tmp_path = tempfile.mkstemp(suffix=".wav")
  309. os.close(fd)
  310. engine = pyttsx3.init()
  311. engine.save_to_file(text, tmp_path)
  312. engine.runAndWait()
  313. with open(tmp_path, "rb") as f:
  314. payload = base64.b64encode(f.read()).decode("ascii")
  315. return payload, "audio/wav", "本地模型检查通过;当前使用后端系统语音做兼容播放。"
  316. except Exception as e:
  317. return None, None, f"{probe};后端语音生成失败({e}),已回退到前端浏览器语音。"
  318. finally:
  319. if tmp_path and os.path.exists(tmp_path):
  320. try:
  321. os.remove(tmp_path)
  322. except Exception:
  323. pass
  324. @router.get("/voice-assets", response_model=list[VoiceAssetItem])
  325. def list_voice_assets():
  326. base = _voice_assets_dir()
  327. items: list[VoiceAssetItem] = []
  328. for fp in sorted(base.glob("*")):
  329. if not fp.is_file() or fp.suffix.lower() not in VOICE_EXTS:
  330. continue
  331. items.append(
  332. VoiceAssetItem(
  333. name=fp.name,
  334. size=fp.stat().st_size,
  335. url=f"{settings.api_v1_prefix}/digital-human/voice-assets/{fp.name}",
  336. )
  337. )
  338. return items
  339. @router.get("/voice-assets/{name}")
  340. def get_voice_asset(name: str):
  341. safe = _safe_voice_name(name)
  342. fp = _voice_assets_dir() / safe
  343. if not fp.exists():
  344. raise HTTPException(status_code=404, detail="语音文件不存在")
  345. media_type = {
  346. ".wav": "audio/wav",
  347. ".mp3": "audio/mpeg",
  348. ".m4a": "audio/mp4",
  349. ".ogg": "audio/ogg",
  350. }.get(fp.suffix.lower(), "application/octet-stream")
  351. return FileResponse(path=str(fp), media_type=media_type, filename=fp.name)
  352. @router.post("/voice-assets", response_model=VoiceAssetItem)
  353. def upload_voice_asset(
  354. file: Annotated[UploadFile, File(...)],
  355. _teacher: Annotated[User, Depends(require_role(UserRole.teacher))],
  356. ):
  357. safe = _safe_voice_name(file.filename or "")
  358. data = file.file.read()
  359. if not data:
  360. raise HTTPException(status_code=400, detail="上传文件为空")
  361. target = _voice_assets_dir() / safe
  362. target.write_bytes(data)
  363. return VoiceAssetItem(name=target.name, size=target.stat().st_size, url=f"{settings.api_v1_prefix}/digital-human/voice-assets/{target.name}")
  364. @router.delete("/voice-assets/{name}", response_model=dict[str, bool])
  365. def delete_voice_asset(
  366. name: str,
  367. _teacher: Annotated[User, Depends(require_role(UserRole.teacher))],
  368. ):
  369. safe = _safe_voice_name(name)
  370. fp = _voice_assets_dir() / safe
  371. if not fp.exists():
  372. raise HTTPException(status_code=404, detail="语音文件不存在")
  373. fp.unlink()
  374. return {"ok": True}
  375. @router.post("/ask", response_model=AskResponse)
  376. def ask(payload: AskRequest, session: Session = Depends(get_session), current_user: User = Depends(get_current_user)):
  377. if payload.session_id:
  378. chat_session = session.get(ChatSession, payload.session_id)
  379. if not chat_session or chat_session.user_id != current_user.id:
  380. chat_session = None
  381. else:
  382. chat_session = None
  383. if not chat_session:
  384. chat_session = ChatSession(user_id=current_user.id, topic=payload.knowledge_code)
  385. session.add(chat_session)
  386. session.commit()
  387. session.refresh(chat_session)
  388. session.add(ChatMessage(session_id=chat_session.id, sender="user", content=payload.question))
  389. examples_candidate = _build_examples_answer(payload.question, payload.knowledge_code, session)
  390. if examples_candidate:
  391. answer, sources = examples_candidate
  392. else:
  393. sources = []
  394. answer = _build_default_answer()
  395. if _is_deepseek_selected(payload.model_name, payload.model_provider):
  396. candidate = _generate_answer_with_deepseek(payload.question, payload.knowledge_code, payload.model_id)
  397. answer = candidate if candidate else answer
  398. session.add(ChatMessage(session_id=chat_session.id, sender="assistant", content=answer))
  399. session.commit()
  400. audio_base64, audio_mime, tts_status = _synthesize_audio_base64(answer)
  401. return AskResponse(
  402. session_id=UUID(str(chat_session.id)),
  403. answer=answer,
  404. sources=sources,
  405. tts_status=tts_status,
  406. audio_base64=audio_base64,
  407. audio_mime=audio_mime,
  408. )