From f2e6d847e24a10a1ca357905795418d13c780b62 Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 25 Jun 2026 19:18:51 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=85=A8=E5=85=AC=E5=8F=B8=E5=85=B1?= =?UTF-8?q?=E4=BA=AB=E4=B8=89=E8=AF=AD=E6=9C=AF=E8=AF=AD=E8=A1=A8=EF=BC=88?= =?UTF-8?q?=E4=B8=AD/=E8=8B=B1/=E5=8D=B0=E5=B0=BC=EF=BC=89=EF=BC=8C?= =?UTF-8?q?=E4=BF=9D=E5=AD=98=E5=8D=B3=E9=BB=98=E8=AE=A4=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - glossary_store.py: 进程级单例JSON存储，原子写入+asyncio锁，DOCUTRANSLATE_GLOSSARY_PATH可配置 - lang_mapping.py: 三列(zh/en/id)与{src:dst}互转，按目标语言自动取列；兼容旧2列CSV - app.py: 新增GET/POST /service/glossary路由 - environment.py: 新增DOCUTRANSLATE_GLOSSARY_PATH环境变量 - index.html: 弹窗改为可编辑3列表+增删行+保存/导入/导出CSV；页面打开自动加载共享术语表； 翻译时前端按to_lang编译为{src:dst}，payload.glossary_dict契约不变 - i18nData.json: 三语新增术语表相关UI文案 - 翻译管线零改动：MT走translation_options.terms，非MT走system prompt注入 Co-Authored-By: Claude Opus 4.7 --- .gitignore | 2 + docutranslate/app.py | 22 +++ docutranslate/environment.py | 4 + docutranslate/glossary/glossary_store.py | 115 ++++++++++++++++ docutranslate/glossary/lang_mapping.py | 123 +++++++++++++++++ docutranslate/static/i18nData.json | 45 ++++++- docutranslate/static/index.html | 162 ++++++++++++++++++++--- 7 files changed, 449 insertions(+), 24 deletions(-) create mode 100644 docutranslate/glossary/glossary_store.py create mode 100644 docutranslate/glossary/lang_mapping.py diff --git a/.gitignore b/.gitignore index 22be5fe..d834a8b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ wheels/ tests/resource/ tests/ docutranslate/output/ +# Shared glossary runtime data +data/ # Virtual environments .venv #idea diff --git a/docutranslate/app.py b/docutranslate/app.py index 35718ee..3f51f6b 100644 --- a/docutranslate/app.py +++ b/docutranslate/app.py @@ -2206,6 +2206,28 @@ async def service_get_app_version(): return JSONResponse(content={"version": __version__}) +@service_router.get( + "/glossary", tags=["Application"], description="获取全公司共享术语表（三列：中文/English/Bahasa）" +) +async def service_get_glossary(): + from docutranslate.glossary.glossary_store import get_glossary_store + store = get_glossary_store() + return JSONResponse(content={"rows": store.get_rows()}) + + +@service_router.post( + "/glossary", tags=["Application"], description="覆盖保存全公司共享术语表（保存即对所有用户生效）" +) +async def service_save_glossary(body: dict = Body(...)): + from docutranslate.glossary.glossary_store import get_glossary_store + raw_rows = body.get("rows") if isinstance(body, dict) else None + if not isinstance(raw_rows, list): + raise HTTPException(status_code=400, detail="请求体需为 {\"rows\": [...]}") + store = get_glossary_store() + saved = await store.save(raw_rows) + return JSONResponse(content={"rows": saved}) + + @service_router.post( "/flat-translate", summary="translate(sync)", diff --git a/docutranslate/environment.py b/docutranslate/environment.py index b87356a..2788840 100644 --- a/docutranslate/environment.py +++ b/docutranslate/environment.py @@ -24,6 +24,10 @@ DOCUTRANSLATE_PROXY_ENABLED: bool = ( # 任务缓存数量 DOCUTRANSLATE_CACHE_NUM: int = int(os.getenv("DOCUTRANSLATE_CACHE_NUM", "10")) +# --- 共享术语表存储路径 --- +# 留空则使用工作目录下 ./data/glossary.json +DOCUTRANSLATE_GLOSSARY_PATH: str = os.getenv("DOCUTRANSLATE_GLOSSARY_PATH", "") + # --- 翻译 API 默认配置 --- # 默认 API 地址 (自定义接口的 Base URL) DOCUTRANSLATE_BASE_URL: str = os.getenv("DOCUTRANSLATE_BASE_URL", "") diff --git a/docutranslate/glossary/glossary_store.py b/docutranslate/glossary/glossary_store.py new file mode 100644 index 0000000..82031cf --- /dev/null +++ b/docutranslate/glossary/glossary_store.py @@ -0,0 +1,115 @@ +# SPDX-FileCopyrightText: 2025 QinHan +# SPDX-License-Identifier: MPL-2.0 +""" +共享术语表持久化存储。 + +设计要点： +- 进程级单例，所有用户读写同一份 JSON 文件。 +- 原子写入（临时文件 + os.replace），asyncio.Lock 保护并发写。 +- 三列结构 {zh, en, id}，按目标语言编译为单向 {src: dst}（见 lang_mapping.py）。 +""" +import asyncio +import json +import os +import tempfile +from pathlib import Path +from typing import Any + +from docutranslate.logger import global_logger + +_COLS = ("zh", "en", "id") +_VERSION = 1 + + +def _default_path() -> Path: + """解析术语表存储路径：优先环境变量，否则工作目录下 ./data/glossary.json""" + from docutranslate.environment import DOCUTRANSLATE_GLOSSARY_PATH + if DOCUTRANSLATE_GLOSSARY_PATH: + p = Path(DOCUTRANSLATE_GLOSSARY_PATH).expanduser() + else: + p = Path.cwd() / "data" / "glossary.json" + p.parent.mkdir(parents=True, exist_ok=True) + return p + + +def _normalize_row(raw: Any) -> dict: + """规范化一行：确保 zh/en/id 三键均为 str（空串允许）。""" + if not isinstance(raw, dict): + return {c: "" for c in _COLS} + return {c: str(raw.get(c, "") or "") for c in _COLS} + + +class GlossaryStore: + """进程级共享术语表存储（单例）。""" + + _instance = None + + def __new__(cls): + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._initialized = False + return cls._instance + + def __init__(self): + if self._initialized: + return + self._initialized = True + self.logger = global_logger + self._path = _default_path() + self._lock = asyncio.Lock() + self._cache: list[dict] | None = None + self.logger.info(f"术语表存储路径: {self._path}") + + def _read_from_disk(self) -> list[dict]: + if not self._path.exists(): + return [] + try: + with self._path.open("r", encoding="utf-8") as f: + data = json.load(f) + rows = data.get("rows", []) if isinstance(data, dict) else [] + return [_normalize_row(r) for r in rows] + except (OSError, json.JSONDecodeError) as e: + self.logger.error(f"读取术语表失败，回退为空: {e}") + return [] + + def _write_to_disk(self, rows: list[dict]) -> None: + payload = {"version": _VERSION, "rows": rows} + # 原子写：先写临时文件再替换 + fd, tmp_path = tempfile.mkstemp( + prefix="glossary_", suffix=".json", dir=str(self._path.parent) + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(payload, f, ensure_ascii=False, indent=2) + os.replace(tmp_path, self._path) + except Exception: + if os.path.exists(tmp_path): + os.remove(tmp_path) + raise + + def get_rows(self) -> list[dict]: + """同步读取所有行（带进程内缓存）。""" + if self._cache is None: + self._cache = self._read_from_disk() + return [dict(r) for r in self._cache] + + async def save(self, rows: list[dict]) -> list[dict]: + """校验并原子保存全部行，返回保存后的规范化行。""" + normalized = [_normalize_row(r) for r in rows] + # 过滤掉三列全空的行 + normalized = [r for r in normalized if any(r[c] for c in _COLS)] + async with self._lock: + self._write_to_disk(normalized) + self._cache = [dict(r) for r in normalized] + return [dict(r) for r in normalized] + + async def reload(self) -> list[dict]: + """强制从磁盘重新加载（调试/外部编辑后）。""" + async with self._lock: + self._cache = self._read_from_disk() + return [dict(r) for r in self._cache] + + +def get_glossary_store() -> GlossaryStore: + """获取全局单例。""" + return GlossaryStore() diff --git a/docutranslate/glossary/lang_mapping.py b/docutranslate/glossary/lang_mapping.py new file mode 100644 index 0000000..c56c783 --- /dev/null +++ b/docutranslate/glossary/lang_mapping.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: 2025 QinHan +# SPDX-License-Identifier: MPL-2.0 +""" +三语术语表（中文/English/Bahasa）与单向 {src: dst} 字典的互转， +以及与 CSV 的导入/导出。 + +列约定：zh / en / id +""" +import csv +import io +from typing import Any + +# 目标语言 -> 列名 的映射（小写匹配） +_LANG_ALIASES = { + # 中文（简/繁） + "zh": "zh", + "zh-cn": "zh", + "zh-hans": "zh", + "zh-hant": "zh", + "chinese": "zh", + "simplified chinese": "zh", + "traditional chinese": "zh", + "中文": "zh", + "简体中文": "zh", + "繁体中文": "zh", + "汉语": "zh", + # 英文 + "en": "en", + "en-us": "en", + "en-gb": "en", + "english": "en", + "英文": "en", + "英语": "en", + # 印尼语 + "id": "id", + "id-id": "id", + "indonesian": "id", + "bahasa indonesia": "id", + "印尼文": "id", + "印尼语": "id", + "印度尼西亚语": "id", +} + +_COLS = ("zh", "en", "id") + + +def lang_to_col(to_lang: str | None) -> str | None: + """目标语言字符串 -> 列名 'zh'|'en'|'id'，无法识别返回 None。""" + if not to_lang: + return None + key = str(to_lang).strip().lower() + return _LANG_ALIASES.get(key) + + +def build_glossary_dict(rows: list[dict], to_lang: str | None) -> dict[str, str] | None: + """ + 将三列 rows 编译为单向 {src: dst} 字典。 + + 规则：目标列作为 dst，其它两列作为 src，覆盖任意源语言命中文本。 + 返回 None 表示目标语言不受支持（不注入术语表，安全降级）。 + """ + target_col = lang_to_col(to_lang) + if target_col is None or not rows: + return None + result: dict[str, str] = {} + for raw in rows: + if not isinstance(raw, dict): + continue + target = str(raw.get(target_col, "") or "").strip() + if not target: + continue + for col in _COLS: + if col == target_col: + continue + src = str(raw.get(col, "") or "").strip() + if src and src not in result: + result[src] = target + return result or None + + +def rows_to_csv(rows: list[dict]) -> str: + """三列 rows -> CSV 字符串（含 BOM，表头 zh,en,id）。""" + buf = io.StringIO() + buf.write("") # UTF-8 BOM，便于 Excel 正确识别 + writer = csv.writer(buf) + writer.writerow(["zh", "en", "id"]) + for raw in rows: + if not isinstance(raw, dict): + continue + writer.writerow([str(raw.get(c, "") or "") for c in _COLS]) + return buf.getvalue() + + +def csv_to_rows(parsed: list[dict] | list[list]) -> list[dict]: + """ + 解析后的 CSV（PapaParse 结果）-> rows。 + + 支持三种格式： + - 3 列表头 zh/en/id + - 旧 2 列表头 src/dst + - 无表头的 2/3 列数组（兜底按 [zh,en] 或 [zh,en,id] 处理） + """ + rows: list[dict] = [] + for item in parsed: + if isinstance(item, dict): + # 有表头 + if any(c in item for c in _COLS): + rows.append({c: str(item.get(c, "") or "").strip() for c in _COLS}) + elif "src" in item and "dst" in item: + # 旧 2 列格式：无法判断语言，放入 en 列作 src、zh 列作 dst（最常见的英译中场景） + rows.append({"zh": str(item.get("dst", "") or "").strip(), + "en": str(item.get("src", "") or "").strip(), + "id": ""}) + else: + continue + elif isinstance(item, list): + cells = [str(x or "").strip() for x in item] + if len(cells) >= 3: + rows.append({"zh": cells[0], "en": cells[1], "id": cells[2]}) + elif len(cells) == 2: + rows.append({"zh": cells[0], "en": cells[1], "id": ""}) + # 过滤全空行 + return [r for r in rows if any(r[c] for c in _COLS)] diff --git a/docutranslate/static/i18nData.json b/docutranslate/static/i18nData.json index ecf6516..4174786 100644 --- a/docutranslate/static/i18nData.json +++ b/docutranslate/static/i18nData.json @@ -107,7 +107,7 @@ "tpmLabel": "每分钟Token数(估算)", "unlimitedPlaceholder": "留空为无限制", "glossaryGenTitle": "术语表", - "glossaryLabel": "术语表 (可选)", + "glossaryLabel": "术语表（全公司共享）", "glossaryHelp": "选择一个或多个CSV文件。文件需包含'src'和'dst'两列标题，分别代表原文和译文。", "viewGlossaryBtn": "查看术语表", "clearGlossaryBtn": "清空", @@ -174,7 +174,18 @@ "preview_translatedOnly": "仅译文预览", "configImportSuccess": "配置导入成功！", "configImportError": "配置文件解析失败，请检查文件格式。", - "providerLabel": "接口供应商" + "providerLabel": "接口供应商", + "glossarySharedHint": "此术语表全公司共享，保存后对所有用户生效。翻译时按目标语言自动取对应列。", + "glossaryColZh": "中文", + "glossaryColEn": "English", + "glossaryColId": "Bahasa Indonesia", + "glossaryDeleteRow": "删除此行", + "glossaryAddRow": "添加术语", + "glossarySave": "保存", + "glossaryExportCsv": "导出CSV", + "glossaryUnsaved": "未保存", + "glossarySaved": "术语表已保存，对所有用户生效。", + "glossarySaveFailed": "保存失败" }, "en": { "pageTitle": "DocuTranslate - Interactive Document Translation", @@ -284,7 +295,7 @@ "tpmLabel": "Tokens Per Minute(Estimated)", "unlimitedPlaceholder": "Leave empty for unlimited", "glossaryGenTitle": "Glossary", - "glossaryLabel": "Glossary (Optional)", + "glossaryLabel": "Glossary (shared company-wide)", "glossaryHelp": "Select one or more CSV files. Files must contain 'src' and 'dst' headers, representing source and destination terms.", "viewGlossaryBtn": "View Glossary", "clearGlossaryBtn": "Clear", @@ -351,7 +362,18 @@ "preview_translatedOnly": "Translation Only Preview", "configImportSuccess": "Configuration imported successfully!", "configImportError": "Failed to parse config file, please check the file format.", - "providerLabel": "Provider" + "providerLabel": "Provider", + "glossarySharedHint": "This glossary is shared company-wide. Saved changes apply to all users. The target language column is used automatically during translation.", + "glossaryColZh": "中文", + "glossaryColEn": "English", + "glossaryColId": "Bahasa Indonesia", + "glossaryDeleteRow": "Delete row", + "glossaryAddRow": "Add term", + "glossarySave": "Save", + "glossaryExportCsv": "Export CSV", + "glossaryUnsaved": "Unsaved", + "glossarySaved": "Glossary saved and applied to all users.", + "glossarySaveFailed": "Save failed" }, "id": { "pageTitle": "DocuTranslate - Terjemahan Dokumen Interaktif", @@ -461,7 +483,7 @@ "tpmLabel": "Token per Menit (TPM - Estimasi)", "unlimitedPlaceholder": "Kosongkan untuk tanpa batas", "glossaryGenTitle": "Glosarium", - "glossaryLabel": "Glosarium (Opsional)", + "glossaryLabel": "Glosarium (dibagikan ke seluruh perusahaan)", "glossaryHelp": "Pilih file CSV dengan kolom src dan dst.", "viewGlossaryBtn": "Lihat Glosarium", "clearGlossaryBtn": "Hapus", @@ -528,6 +550,17 @@ "preview_translatedOnly": "Pratinjau Hanya Terjemahan", "configImportSuccess": "Konfigurasi berhasil diimpor!", "configImportError": "Gagal memproses file konfigurasi.", - "providerLabel": "Provider" + "providerLabel": "Provider", + "glossarySharedHint": "Glosarium ini dibagikan ke seluruh perusahaan. Perubahan tersimpan berlaku untuk semua pengguna. Kolom bahasa target digunakan otomatis saat menerjemahkan.", + "glossaryColZh": "中文", + "glossaryColEn": "English", + "glossaryColId": "Bahasa Indonesia", + "glossaryDeleteRow": "Hapus baris", + "glossaryAddRow": "Tambah istilah", + "glossarySave": "Simpan", + "glossaryExportCsv": "Ekspor CSV", + "glossaryUnsaved": "Belum disimpan", + "glossarySaved": "Glosarium disimpan, berlaku untuk semua pengguna.", + "glossarySaveFailed": "Gagal menyimpan" } } \ No newline at end of file diff --git a/docutranslate/static/index.html b/docutranslate/static/index.html index b9d0805..761687e 100644 --- a/docutranslate/static/index.html +++ b/docutranslate/static/index.html @@ -848,31 +848,50 @@