Compare commits

...
Author SHA1 Message Date
ila a412e000c0 docs: 记录英汉词典与规范音标的架构、规则与验证 (#40) 2026-09-16 13:27:17 +08:00
ila b1b2e6c6fe feat: 英汉词典与规范音标、多词典并存 (#40)
- ECDICT 常用子集(82,721 条,含中文释义)与 CMUdict ARPAbet→IPA 转写音标,
  由 scripts/dict_prepare.py 从 sha256 pin 的源显式准备,运行时不联网。
- schema v12:放开 lexgo_dictionaries 单行约束并增加 provider 列,英英与英汉词典
  可分别启用;查词合并时中文释义在前、英英释义随后。
- 音标只取可验证来源:优先 CMUdict 转写,否则保留 ECDICT 记法并标注来源,
  两者都不可靠时不显示;含 ^ 等丢失首音的记法整条丢弃。
- 屈折形经 WordNet lemma 解析后回查中文词典,但不把词目音标复制到屈折形上。
- 管理端可区分两种词典并分别启停;学习端查词面板显示音标与中文释义。
2026-09-16 13:17:26 +08:00
24 changed files with 1629 additions and 77 deletions
+25 -6
View File
@@ -6,6 +6,7 @@
<el-alert v-if="resources.notice" :title="resources.notice" type="success" :closable="false" show-icon />
<el-table v-loading="resources.loading" :data="resources.items" border :empty-text="resources.error ? '加载失败,请重试' : '尚未导入词典'">
<el-table-column label="名称" prop="name" min-width="175" />
<el-table-column label="提供者" width="100"><template #default="scope">{{ providerLabel(scope.row.provider) }}</template></el-table-column>
<el-table-column label="语言" width="80"><template #default>英语</template></el-table-column>
<el-table-column label="版本" prop="version" width="80" />
<el-table-column label="来源" prop="source" min-width="160" show-overflow-tooltip />
@@ -18,18 +19,23 @@
<el-card class="import-card">
<h2>导入词典</h2>
<el-form class="dictionary-form" label-position="top" @submit.prevent="upload">
<el-form-item label="词典类型">
<el-select v-model="form.format" :disabled="busy" @change="syncForm">
<el-option v-for="option in supportedOptions" :key="option.format" :label="option.name + ' · ' + option.description" :value="option.format" />
</el-select>
</el-form-item>
<el-form-item label="名称"><el-input v-model="form.name" maxlength="120" :disabled="busy" /></el-form-item>
<div class="form-row">
<el-form-item label="语言"><el-input model-value="英语" disabled /></el-form-item>
<el-form-item label="版本"><el-input v-model="form.version" readonly /></el-form-item>
</div>
<el-form-item label="来源"><el-input v-model="form.source" readonly /></el-form-item>
<el-form-item label="格式"><el-input v-model="form.format" readonly /></el-form-item>
<el-form-item label="词典文件">
<input id="dictionary-file" ref="fileInput" type="file" accept=".zip,application/zip" aria-label="词典文件" :disabled="busy" @change="selectFile">
</el-form-item>
<p class="subtle">WordNet 3.0 ZIP · 上限 32 MiB · 英语释义</p>
<p v-if="resources.supported?.source"><a :href="downloadURL" target="_blank" rel="noopener noreferrer">下载支持的词典文件</a></p>
<p class="subtle">{{ selectedOption?.downloadable ? '官方 ZIP · 上限 32 MiB · 英语释义' : '由 scripts/dict_prepare.py 在本机生成 · 上限 32 MiB · 中文释义与 IPA 音标' }}</p>
<p v-if="selectedOption?.downloadable && form.source"><a :href="form.source" target="_blank" rel="noopener noreferrer">下载支持的词典文件</a></p>
<p v-else class="subtle">先在服务器运行 python scripts/dict_prepare.py,再导入生成的 .local/dictionaries/zh-dict-v1.zip。</p>
<p class="subtle">导入失败时保留当前词典。</p>
<p v-if="fileError" role="alert" class="file-error">{{ fileError }}</p>
<el-button type="primary" native-type="submit" :loading="resources.saving" :disabled="resources.loading || !file">导入并启用</el-button>
@@ -46,17 +52,30 @@ export default {
name: 'DictionariesView',
components: { BasicLayout },
data: () => ({
resources: { items: [], supported: null, loading: false, saving: false, error: '', notice: '' },
resources: { items: [], supported: [], loading: false, saving: false, error: '', notice: '' },
form: { name: 'Princeton WordNet', language: 'en', version: '3.0', source: downloadURL, format: 'wordnet-3.0-zip' },
file: null, fileError: '', downloadURL
}),
computed: { busy() { return this.resources.loading || this.resources.saving } },
computed: {
busy() { return this.resources.loading || this.resources.saving },
supportedOptions() { return Array.isArray(this.resources.supported) ? this.resources.supported : [] },
selectedOption() { return this.supportedOptions.find(option => option.format === this.form.format) || null }
},
watch: { '$store.state.generation': { flush: 'sync', handler() { this.loader.invalidate(); this.clearFile() } } },
created() { this.loader = createDictionaryLoader(session, this.resources) },
mounted() { this.reload() },
beforeUnmount() { this.loader.invalidate(); this.file = null },
methods: {
reload() { return this.loader.load() },
reload() { return this.loader.load().then(() => this.syncForm()) },
providerLabel(provider) { return { wordnet: '英英释义', ecdict: '英汉释义' }[provider] || provider || '英英释义' },
// The server owns which formats are reviewable, so the form mirrors the selected spec instead
// of letting an operator type a version or source that the import would reject.
syncForm() {
const option = this.selectedOption || this.supportedOptions[0]
if (!option) return
this.form = { name: option.name, language: 'en', version: option.version, source: option.source, format: option.format }
this.clearFile()
},
statusLabel(status) { return { ready: '可用', disabled: '已停用', unavailable: '不可用' }[status] || '不可用' },
clearFile() { this.file = null; this.fileError = ''; if (this.$refs.fileInput) this.$refs.fileInput.value = '' },
selectFile(event) {
+27
View File
@@ -55,3 +55,30 @@ test('page exit or session change discards late responses', async () => {
await read
assert.deepEqual(state.items, [])
})
test('both dictionaries stay independently switchable and keep their provider', async () => {
const { createDictionaryLoader } = await import('../src/dictionaries.mjs')
const state = { items: [], supported: null, loading: false, saving: false, error: '', notice: '' }
const supported = [
{ provider: 'wordnet', name: 'Princeton WordNet', version: '3.0', format: 'wordnet-3.0-zip', source: 'https://example.invalid/wordnet.zip', downloadable: true },
{ provider: 'ecdict', name: '简明英汉字典增强版', version: 'ecdict-1.0.28+cmudict-0.7b', format: 'lexgo-zh-dict-v1', source: 'https://example.invalid/ecdict-stardict-28.zip', downloadable: false }
]
const rows = [
{ id: 1, provider: 'wordnet', name: 'Princeton WordNet', enabled: true, status: 'ready' },
{ id: 2, provider: 'ecdict', name: '简明英汉字典增强版', enabled: true, status: 'ready' }
]
const session = {
state: { generation: 1 },
listDictionaries: async () => ({ items: rows, supported }),
importDictionary: async () => ({ resource: rows[1], duplicate: false }),
setDictionaryEnabled: async (id, enabled) => ({ resource: { ...rows.find(row => row.id === id), enabled } })
}
const loader = createDictionaryLoader(session, state)
assert.equal(await loader.load(), true)
assert.deepEqual(state.supported, supported)
assert.equal(state.items.length, 2)
assert.equal(await loader.toggle(2, false), true)
assert.equal(state.items[1].enabled, false)
assert.equal(state.items[0].enabled, true)
assert.match(state.notice, /停用/)
})
+4 -2
View File
@@ -2,8 +2,8 @@
generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件)
wiki_page: Project-Profile
wiki_url: https://git.ilapage.cn/OPC/lexgo/wiki/Project-Profile.-
wiki_revision: 39adde482a8729fcf303ef7ca24cd8bf136c58a6
synchronized_at: 2026-09-16T02:25:55Z
wiki_revision: e61817b949693c82e74ad3d1a8adb39983a90fab
synchronized_at: 2026-09-16T05:27:04Z
<!-- gitea-wiki-mirror:end -->
# LexGo 项目档案
@@ -194,3 +194,5 @@ server 是 go-admin 的选用模块接入:原样保留 SysUser、SysDept、必
本单期间还发现并修复了一个从 #10 起就存在的真实缺陷:`main.ts` 漏引入 Element Plus 的 `dialog` 与 `message-box` 样式,导致全站所有对话框与确认框退化成文档流块;修复后补充了断言浮层定位属性的 `expectModalOverlay`/`expectMessageBoxOverlay` 用例,并把「只看可见性、不看是否浮层」这一验证盲区写回文档。未实现的确认项:书库卡片显示书级作者、章节作者继承书级作者;真机听感与浏览器真实解码播放仍需人工确认。
至此 MVP 原范围 #2~#15 与增补 #21、#37 全部实施并验收;仍待处理 #24 书库列表优化与缺陷 #32。#16 的 #37 复选框已勾选。
- 2026-09-16 新增第二本英语词典(#40):ECDICT 常用子集提供中文释义、CMUdict 提供 IPA 转写音标,二者由 sha256 pin 的源显式准备;schema v12 放开 `lexgo_dictionaries` 单行约束并增加 `provider` 列,英英与英汉词典可分别启停。音标来源必须可追溯(IPA 转写 / ECDICT 记法),不确定时不显示。许可:ECDICT 仓库 MIT(数据由公开资料整理,再分发前需自行评估)、CMUdict BSD-2,许可证随仓库保存,词典归档不提交进 Git。
+59 -2
View File
@@ -2,8 +2,8 @@
generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件)
wiki_page: Architecture-and-Code-Map
wiki_url: https://git.ilapage.cn/OPC/lexgo/wiki/Architecture-and-Code-Map.-
wiki_revision: 3cf18f98cf824dd3bbd5e8c300a09a321e47b357
synchronized_at: 2026-09-16T02:25:55Z
wiki_revision: 8cc11b2fa972babeb4756b68c0b071282cdb1960
synchronized_at: 2026-09-16T05:27:05Z
<!-- gitea-wiki-mirror:end -->
# 架构与代码地图
@@ -516,3 +516,60 @@ schema v8 新增两张表,都只用可重放的 `CREATE TABLE IF NOT EXISTS`
复用不变:`sniffAttachment`(magic bytes 判定)、`validateAttachment`(音频 ≤20 MiB;图片 ≤2 MiB 且 ≤4096×4096)、`coverDimensions`(JPEG/PNG 用标准库、WebP 读容器头)、`binaryResponse` + `http.ServeContent`(Range/206、416、ETag/304)、上传「先校验后写入、失败保留旧文件」的流程,以及 `chapterAttachmentViews`/`chapterPlaybackSeconds` 的批量读取(一次查询喂整个章节列表)。
学习端:`stores/library.ts` 用 `uploadChapterFile`/`deleteChapterFile`/`saveChapterPlayback`/`reportChapterPlayback` 替换了书级音频动作,并新增 `illustrationUrls`(按章节)与 `audioChapterId`(记录当前加载的音频属于哪一章,切章不会复用上一章的文件);`views/BookView.vue` 的书籍页把封面压成**一行紧凑控件**(`.cover-row`:预览 120px +「上传/替换封面」「移除」,不含标题、字段名与规格提示文字),把高度让给章节列表;标题行用 `.title-line` 把**书级作者**显示在书名右侧(`data-testid="book-author"`,未设置则不显示);「编辑书名」改为**「编辑书籍」**,对话框内用同一套 `.field-row` 同行排版编辑**书名与作者**。**章节列表每行只有一个「编辑」按钮**,章节编辑对话框(`data-testid="chapter-dialog"`)同时编辑**章节标题、作者、正文、插图与音频**:标题与作者用「标签在左、输入框在右」的同一行排版(`.field-row`),正文编辑框加高(18 行),插图与音频压缩成各一两行(标签+状态+上传/替换/移除+「JPG/PNG/WebP · ≤2 MiB · ≤4096×4096」「MP3 · ≤20 MiB · 替换或移除会重置位置」),底部一行写明时机(`data-testid="attachment-timing"`),`views/ReaderView.vue` 在正文上方渲染本章插图的**缩略图**(高 120px 的按钮,`data-testid="chapter-illustration"`),点击后在对话框(`data-testid="illustration-dialog"`)里按原图显示(最大 `min(88vw,1200px)` × `78vh`,保持比例不裁切),播放器只在**本章**有音频时出现,离开或切换章节时上报一次位置;**书籍页的章节列表不显示也不预取插图**(2026-09-15 按用户要求修订两次:先改为缩略图+弹窗,再把附件并入章节编辑对话框)。
## #32 编辑任务按内容版本复用(2026-09-16,已修复并合入 main)
**没有 schema 变化**:修复的是编辑路径的任务编排,不是数据模型。
`server/app/lexgo/edit.go` 的 `stageEditJob` 取代了原先「直接创建编辑任务」的写法。一章的一个内容摘要就是一个版本,而 `lexgo_ingest_jobs.request_key` 是唯一的,编辑请求派生的键又是「章节+内容摘要」,因此同一版本第二次成为当前版本时,原写法会撞唯一键、整笔事务回滚并冒泡为通用 500。
| 情况 | 处理 |
|---|---|
| 该章还没有描述这个内容版本的任务行 | 用派生键创建新任务(`edit:<章节>:<内容摘要>`,pending) |
| 已经有描述这个版本的行(改回曾经用过的正文、或该版本上次处理失败) | **复用该行的身份**,重置为 `pending`、`attempts = 0`、清空 `error_reason` 与 `finished_at`,刷新 `updated_at`;不新增行 |
| 复用到的行可能是粘贴版本的任务(键是粘贴请求号) | 正确:一行描述一个内容版本,键只是历史标识;worker 只按 `status = pending` 且「任务内容摘要 = 章节当前内容」认领 |
不变的门控:`RetryIngestJob` 仍拒绝「任务内容摘要 ≠ 章节当前内容」的旧任务(409),所以旧版本任务不会把新版本章节拉回处理;同一版本重复处理是幂等的;未改动的正文仍然不产生新版本、不新增任务。
## 英汉词典与规范音标(#40,schema v12)
一本自托管英语学习工具需要回答"这个词是什么意思",英语学习者往往先要中文。WordNet 只给英英释义,
因此 #40 增加第二本词典,并让两本可以分别启停。
### 资源与准备
- 数据源:ECDICT(`skywind3000/ECDICT`,Release 1.0.28 的 `ecdict-stardict-28.zip`,仓库 MIT)提供中文释义;
CMUdict(`cmusphinx/cmudict`,BSD-2)提供 ARPAbet 音素;WordNet 词表(与 `server/wordnet-resource.json` 同一 pin)
用于裁剪子集。
- `server/zh-dictionary-resource.json` 记录三者的 URL、sha256、字节数与许可证文件名;**镜像只用于传输,
身份是 sha256**,准备脚本逐个校验,不匹配即失败。
- `scripts/dict_prepare.py`(Python,仅工具链,不是运行时代码)执行:下载并校验三个源 → 解析 StarDict `.ifo/.idx/.dict`
与 CMUdict → 裁剪子集(小写单词,且存在于 WordNet 词表或带 ECDICT 考试标签)→ 生成音标 → 写 ZIP
(`manifest.json` + `entries.jsonl.gz`)。产物默认落在 `.local/dictionaries/`,**不入库**。
- 产出前强制校验:条目数与 pin 容差(±2000)、音标字符集必须全部落在 IPA 字母表内,否则拒绝产出;
`manifest.json` 记录三类音标计数与丢弃计数。
### 音标来源分层(不做语义猜测)
1. 词在 CMUdict 中 → 按固定 ARPAbet→IPA 映射转写(`1`→`ˈ`、`2`→`ˌ`、`0` 不标;`AH0`→`ə`、`IY0`→`i`、
`ER0`→`ɚ`;单音节不标重音,`cat` 是 `/kæt/` 而不是 `/kˈæt/`)。
2. 否则用 ECDICT 记法做**字符级**规范化:`'`→`ˈ`、`,`/`.`→`ˌ`、`:`→`ː`、`ә`(U+04D9)→`ə`(U+0259)、
`ε`→`e`、空格/`-`/`=`/`;`/`^` 删除;来源标记为 `ecdict`。
3. 音标以 `^` 开头说明原记法已丢失首音(`grok` 存成 `^rɔk`),整条丢弃——宁可没有,也不给错的转写;
出现其它非 IPA 字符同样丢弃。两个来源都没有音标就不显示。
导入时 `provider` 与 `phoneticSource` 一起进入条目,前端据此区分"IPA 转写"与"ECDICT 记法"。
### 存储与查询
- schema v12:`addDictionaryProvider` 增加 `provider VARCHAR(32) NOT NULL DEFAULT 'wordnet'`,
`dropDictionarySingleSlotCheck` 从 `information_schema` 查到 `CHECK (id = 1)` 后删除。
两步都是条件执行、可重放;版本行只在全部成功后推进,旧二进制看到 v12 会拒绝启动并要求显式迁移。
- 槽位固定:1 = WordNet(英英),2 = ECDICT(英汉 + 音标)。归档整包存进 `lexgo_dictionaries.archive`,
所以普通 dump 就是完整备份;`scripts/dict_prepare.py` 的产物约 3.0 MB。
- `parseResource` 按 `format` 分派解析器;`dictionaryCache` 按资源 id 缓存已解析词典(键含 sha256)。
- 查词合并(`mergeLookups`):中文释义条目在前、英英释义随后,`Resources` 列出全部启用词典,
`Resource` 保持为 WordNet 以兼容既有界面;任一本缺失/损坏/停用时只用剩下那本,都不可用时
仍是 `resource_missing`。
- 屈折形:WordNet 的规则解析给出词目后,再用该词目回查中文词典,因此 `dogs` 也能看到"狗";
但音标属于词目,**不复制**到屈折形上(`dogs` 不显示 `/dɔɡ/`)。
+31 -2
View File
@@ -2,8 +2,8 @@
generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件)
wiki_page: Business-Rules-and-Glossary
wiki_url: https://git.ilapage.cn/OPC/lexgo/wiki/Business-Rules-and-Glossary.-
wiki_revision: b688ef278f83c8055e16f91843a239f9ea2dd492
synchronized_at: 2026-09-15T15:10:08Z
wiki_revision: ba03226b60eee1c44062f4206b970a2047b8ddbc
synchronized_at: 2026-09-16T05:27:06Z
<!-- gitea-wiki-mirror:end -->
# 业务规则与术语
@@ -416,3 +416,32 @@ exact优先;未命中再按WordNet异常表/词尾规则查候选,词性顺
**删除与备份**:删除章节(#10 的事务级联)连带删除该章附件与该章所有账号的位置记录;删除书籍通过章节级联清空。附件存于数据库,因此 #15 的备份与恢复自动覆盖,恢复后按字节相同并可继续拖动。
**范围外**:多张插图、正文中间图文混排、缩略图墙式独立浏览界面、多音轨与分章音轨、字幕与时间戳同步、逐句跟读、转写、TTS、转码、自动搜图、在线音频地址、公开分享与跨用户共享。
## #32 版本回退规则(2026-09-16)
**正文版本可以来回切换**:把一章的正文改回曾经用过的文本,属于发布该内容版本,必须正常成功——不再出现无解释的错误,也不会因此产生重复任务行。规则如下:
- 一个内容版本对应一条任务记录:初次成为当前版本时创建,**再次成为当前版本时复用**(把该记录重置为待处理、重试次数归零、清空上次失败原因),因此反复来回切换不会累积任务行。
- **旧版本的任务不会复活新版本章节**:只有「任务内容摘要 = 章节当前内容」的任务可以被重试或处理,否则拒绝。
- 同一版本重复提交**没有副作用**:正文没变时不产生新版本,也不新增任务。
- 上次处理失败的版本在被重新发布时**重试次数归零**,因为「人再次要求」不应被此前的失败预算挡住。
- 回退到早期版本**不会恢复**当时的词条状态、复习排期或章节完成记录:正文版本与个人学习状态是分开的(#10/#11 已确认的口径)。
## 词典、音标与个人释义的边界(#40)
- **词典内容不是个人数据**:WordNet(英英)与英汉词典(ECDICT 子集 + CMUdict 音标)是全局资源,
对所有学习账号相同,管理员可分别启用或停用;停用只影响查询时的可用来源,不删除个人已保存内容。
- **个人释义与状态优先**:用户自己写下的释义、状态与等级仍按 #7/#8 的规则独立保存,不因词典新增中文释义
而被改写、合并或自动填充;面板上中文释义与英英释义只是参考内容。
- **音标来源必须可见**:IPA 转写(CMUdict)与 ECDICT 记法(仅做字符级规范化)是两种不同的可信度,
系统记录 `phoneticSource`;不确定的记法直接不显示,不用猜测结果冒充实测音标。`grok` 属此类。
- **屈折形**:`dogs` 通过 WordNet 的规则解析看到词目 `dog` 的释义属允许行为,但音标不随屈折形显示,
避免把词目的发音说成屈折形的发音。
- **子集口径**:英汉词典只收常用词(约 8.2 万条:小写单词且存在于 WordNet 词表或带考试标签),
不导入 ECDICT 全量 340 万条;漏收的词表现为"只有英英释义",不是错误。
- **许可**:ECDICT 仓库为 MIT,词典数据由公开资料整理,再分发前需自行评估来源;CMUdict 为 BSD-2;
两份许可证随仓库保存(`server/ECDICT-LICENSE.txt`、`server/CMUDICT-LICENSE.txt`),词典归档本身不提交进 Git。
术语:**英英词典**(WordNet,英语释义)、**英汉词典**(ECDICT 子集,中文释义 + 音标)、
**IPA 转写**(由 CMUdict 音素按固定映射转成)、**ECDICT 记法**(老式音标,仅做字符级规范化)、
**子集口径**(决定哪些词进入英汉词典的规则)。
+53 -2
View File
@@ -2,8 +2,8 @@
generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件)
wiki_page: Local-Development-and-Verification
wiki_url: https://git.ilapage.cn/OPC/lexgo/wiki/Local-Development-and-Verification.-
wiki_revision: 7c41d62a131bfcc8c00a3b42a9830600a31f9c6a
synchronized_at: 2026-09-16T02:25:55Z
wiki_revision: afcd40ee530a24f924a39209c23a9dfd090114a2
synchronized_at: 2026-09-16T05:27:07Z
<!-- gitea-wiki-mirror:end -->
# 本地开发与验证
@@ -1095,3 +1095,54 @@ schema v8 → v9:新增 `lexgo_chapter_attachments` 与 `lexgo_chapter_playbac
- Safari/Firefox 未测;渐进式流式播放仍未做(服务端 Range 已实现)。
- 迁移会清空书级音频测试数据且回退到 v8 找不回;当前无真实用户,工单已记录。
- 已有书籍/章节的存量内容不受影响,但**没有做「把既有书级音频迁到章级」的转换**(无真实数据)。
## #32 修复与验证(2026-09-16)
### 根因与修复
编辑路径用 `contentSHA("edit:<章节>:<内容摘要>")` 派生任务键,与 `lexgo_ingest_jobs.uq_job_request (owner_id, request_key)` 相撞:把正文改回曾经用过的版本时派生出已存在的键,`Create` 触发 MySQL 1062,事务整体回滚并冒泡为通用 500。新增 `stageEditJob`:先按 `(owner_id, chapter_id, content_sha256)` 锁定查询该版本的任务行,找到就复用(重置为 pending、attempts=0、清空失败原因),找不到才用派生键创建。**无 schema 变化**,因此不需要迁移;回退件 `.local/lexgo-pre-issue32.exe`。
### 验证
| 项 | 结果 |
|---|---|
| `go vet ./...` / `gofmt -l` | 通过、无输出 |
| `LEXGO_TEST_DB_NAME=lexgo_test_issue13 python scripts/server.py test-integration` | **92 项顶层用例通过、0 跳过**;新增 `TestMySQLChapterEditBackToAPreviousVersion` 覆盖 A→B→A→B(此前第三步 500)、「一章一版本一行任务」、被复用的行干净重启(pending、attempts=0、无 finished_at/error_reason)、失败版本再次成为当前版本时被复活、章节最终为 ready 且正文为最后一次编辑的内容、读者按该内容返回、正文未改动时不新增任务 |
| 真实 API+MySQL | **21 项检查通过**(`.local/issue32-api-evidence.json`):在开发实例上对一本 fixture 书籍执行 A→B→A→B,三步全部 200(此前第三步 500)、每步都 report versionChanged 并最终 ready、该章只有两条任务行且都以 ready 收尾、没有任务停在 pending、读者返回最后一次编辑的正文、章节内容摘要等于某个任务版本、再切回另一版本仍为 200 且仍是两行、删除 fixture 书籍后任务级联清空 |
| 治理 | `python -m unittest discover -s tests` 65 项与 `harness.py check --strict` 通过 |
### 未验证
- 只验证了「单章来回切换」;多章并发编辑、超大文本反复切换与移动端上的编辑体验未单独压测。
- 与编辑器交互无关:本单只改后端编排,前端在 #10 已实现「正文改版会重新处理」的提示。
## 英汉词典的准备、导入与验证(#40)
```powershell
# 1) 显式准备(需要网络,只在此步骤下载;产物不入库)
python scripts/dict_prepare.py
# 可用 --cache-dir .local/dictionaries/cache 复用已下载的源,--keep-going 在计数漂移时仍产出
# 打印条目数、三类音标计数、产物字节数与 sha256
# 2) 显式迁移(v12 增加 provider 并放开单行约束)
python scripts/server.py migrate
# 3) 导入:管理端「英语词典」页选择"英汉释义"后上传 .local/dictionaries/zh-dict-v1.zip
# 命令行等价:POST /api/v1/dictionaries/import(multipart: name/language/version/source/format + file)
# 导入失败或校验不通过时保留原有词典,不会替换成半成品。
# 4) 查询验证
curl -s http://127.0.0.1:8000/api/v1/dictionaries # items[].provider 应为 wordnet / ecdict
python .local/verify-issue40-api.py # 28 项真实 API 检查
```
- 资源路径:准备好的归档 `.local/dictionaries/zh-dict-v1.zip`(约 3.0 MB);WordNet 夹具沿用
`.local/nlp-resources/wordnet.zip`。测试在夹具缺失时跳过,不会静默通过。
- 测试命令:
`LEXGO_TEST_DB_NAME=lexgo_test_issue40 python scripts/server.py test-integration`(Go 96 项)、
`cd learner && npx vitest --run`(159 项)、`npx playwright test`(26 项,含 390×844 触摸视口)、
`cd admin && npx pnpm test`(32 项)。
- 迁移回退:把 `lexgo_schema.version` 写回 11、删除 id=2 的词典行即可;旧二进制缺少 `provider` 字段的
读语句不受影响(新列有默认值),但旧二进制见到 v12 会拒绝启动并要求显式迁移,这是有意的保护。
- 排错:`provider` 不存在 → 未执行 migrate;导入 400 且提示版本不符 → 归档不是当前 pin 生成的
(重新运行 `dict_prepare.py`);查词只有英英释义 → 英汉词典未导入或被停用。
+13 -2
View File
@@ -2,8 +2,8 @@
generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件)
wiki_page: Product-Requirements-Overview
wiki_url: https://git.ilapage.cn/OPC/lexgo/wiki/Product-Requirements-Overview.-
wiki_revision: daf31e0d00508a7d0cf504a62072ed69322a5644
synchronized_at: 2026-09-16T02:25:57Z
wiki_revision: 5c7b0b4942b2c10edadf2f2b84686675267d323e
synchronized_at: 2026-09-16T05:27:09Z
<!-- gitea-wiki-mirror:end -->
# 产品需求总览
@@ -335,3 +335,14 @@ F 范围之外的 MVP 增补项「书籍音频与封面附件」已实现,待
用户的真实场景「多篇文章合并成一本书、每章各有音频,并为每章加插图」已实现,待用户验收:音频与插图改为**章节级**(一章各一张),封面维持**书级**;阅读页在正文上方显示本章插图缩略图、点击后在弹窗里查看原图(章节列表不显示章节图片),播放器只在本章有音频时出现;书籍页章节列表每行只有一个「编辑」入口,标题、作者、正文、插图与音频在同一对话框内编辑(标题/作者/正文点保存后生效,文件选中后立即上传);**书籍作者**与**章节作者**是两份互相独立的可选自由文本(各 ≤120 字符,留空即清空),书级作者显示在书籍页书名右侧并由「编辑书籍」维护,章节作者显示在阅读页标题下;schema 为 v11;播放位置粒度改为**账号+章节**,切换章节时上报一次。类型按文件内容判定(MP3、JPG/PNG/WebP),音频 ≤20 MiB、图片 ≤2 MiB 且 ≤4096×4096,读取需会话并支持 HTTP Range 与 ETag。schema 升到 v9(两张新表 + 幂等清空书级音频历史行,需显式迁移);**书级音频接口已下线**。附件仍存于数据库,因此 #15 的备份恢复自动覆盖,第三次恢复演练验证了附件按字节恢复且 Range 可用。
**已知代价**:#21 交付的书级音频功能整体作废(其测试期数据在迁移中清空且回退无法找回);前端实现与播放器组件被复用,未推倒重做。下次再调整附件挂载粒度前应先确认真实使用场景。**真机听感与浏览器真实解码播放仍未验证**。其余待办:#24 书库列表优化与缺陷 #32。
## F06 词典范围扩展:英汉释义与规范音标(#40)
用户 2026-09-16 提出「找个英中词典,单词有发音」,并澄清"发音"指**音标**(举例「grok 的 /gra:k/」),
随后确认三项决策:约 5 万词量级的常用子集、规范 IPA 音标、允许多词典并存(schema v12)。
已交付范围:点词后同时看到**中文释义**(ECDICT 子集,约 8.2 万条)与**音标**
(优先 CMUdict IPA 转写,其余保留 ECDICT 记法并标注来源),英英释义仍作为补充;
两本词典可分别启停,个人释义与学习状态规则不变。
非目标:发音**音频**、整句翻译、汉英方向(中→英)、词频统计界面。F 系列其余功能与 X 系列范围不受影响。
+13 -2
View File
@@ -2,8 +2,8 @@
generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件)
wiki_page: Deployment-and-Operations
wiki_url: https://git.ilapage.cn/OPC/lexgo/wiki/Deployment-and-Operations.-
wiki_revision: b9eda78d3e71dab02dbf2e3af700c81d9c725f8d
synchronized_at: 2026-09-15T13:07:36Z
wiki_revision: 2fa9c30e08fc14346d9960178a425a8ee3564a3a
synchronized_at: 2026-09-16T05:27:09Z
<!-- gitea-wiki-mirror:end -->
# 部署与运维
@@ -217,3 +217,14 @@ python scripts/ops.py smoke --api http://127.0.0.1:<端口> --admin-user <管理
- 恢复演练是单机顺序执行,未验证大库恢复耗时与磁盘空间上限。
- 接口级的两账号闭环与越权验证需要 HTTP 客户端,目前只在 Python 工具通道提供;Go 二进制提供数据库层的备份、恢复与校验。
- 备份仍调用 `mysqldump`:自己实现一致性导出风险更高,因此部署机需要 MySQL 客户端而不只是服务端。
## 备份体积与词典资源(#40)
英汉词典归档(约 3.0 MB)与 WordNet(约 10.3 MB)都以整包形式存在 `lexgo_dictionaries.archive`,
因此 `lexgo backup` 的 dump 天然包含两本词典,恢复后无需重新下载或重新准备词典。
- 备份/恢复流程与 #15 相同,不需要新增参数;`lexgo verify` 检查的仍是同一套清单。
- 词典归档由 `scripts/dict_prepare.py` 在本机从 sha256 pin 的源生成,**产物不入库**;
恢复演练时若需要重新准备,必须重新下载并在准备阶段校验 sha256。
- 迁移到 v12 后旧二进制会被拒绝启动(见本地开发页),因此回退二进制必须先回退 schema 版本标记。
- 未做:定时备份、多主机、HTTPS 的缺口沿用 #15 记录,不因本单变化。
+4 -2
View File
@@ -2,8 +2,8 @@
generated: true (请先修改 Gitea Wiki,禁止直接编辑本文件)
wiki_page: Home
wiki_url: https://git.ilapage.cn/OPC/lexgo/wiki/Home
wiki_revision: 3af7d77905777a9b0725e5be755eb4a7f461cad7
synchronized_at: 2026-09-16T02:25:55Z
wiki_revision: bbea927c73e51b6b5490de3cb52d42a88dfbb31d
synchronized_at: 2026-09-16T05:27:04Z
<!-- gitea-wiki-mirror:end -->
# LexGo 文档入口
@@ -100,3 +100,5 @@ Quant-UX 原型 v1 已通过用户验收。[桌面预览](https://qux.ilapage.cn
#37 章节级音频与章节插图已于 2026-09-15 实施完成,待用户验收:每章可以上传一段 MP3(≤20 MiB)与一张插图(JPG/PNG/WebP,≤2 MiB、≤4096×4096),封面仍按书;阅读页在正文上方显示本章插图的缩略图、点击后在弹窗里查看原图(书籍页章节列表不显示章节图片),播放器只在本章有音频时出现且不与文本同步,播放位置按「账号+章节」保存并在切换章节时上报;书籍页章节列表每行只有一个「编辑」,点开后在同一对话框里编辑标题、作者、正文与本章插图、音频(标题/作者/正文点保存后生效,文件选中后立即上传;阅读页在标题下显示作者)。类型按文件内容判定,读取需会话并支持 HTTP Range 与 ETag;附件存于数据库,包含在 #15 的备份恢复范围内(第三次恢复演练已验证按字节恢复)。schema v9 需显式迁移,书级音频接口已下线,其实测试期数据在迁移中清空。真机听感与浏览器真实解码播放仍需人工确认。
#37 章节级音频与章节插图已于 2026-09-15 实施完成,待用户验收:每章可以上传一段 MP3(≤20 MiB)与一张插图(JPG/PNG/WebP,≤2 MiB、≤4096×4096),封面仍按书;阅读页在正文上方显示本章插图的缩略图、点击后在弹窗里查看原图(书籍页章节列表不显示章节图片),播放器只在本章有音频时出现且不与文本同步,播放位置按「账号+章节」保存并在切换章节时上报;书籍页的封面压成一行紧凑控件(预览+上传/替换/移除),书名右侧显示书级作者,「编辑书籍」可改书名与作者,章节列表每行只有一个「编辑」,点开后在同一对话框里编辑标题、作者、正文与本章插图、音频(标题/作者/正文点保存后生效,文件选中后立即上传;阅读页在标题下显示作者)。类型按文件内容判定,读取需会话并支持 HTTP Range 与 ETag;附件存于数据库,包含在 #15 的备份恢复范围内(第三次恢复演练已验证按字节恢复)。schema v9 需显式迁移,书级音频接口已下线,其实测试期数据在迁移中清空。真机听感与浏览器真实解码播放仍需人工确认。
#40 英汉词典与规范音标已于 2026-09-16 实现并进入待验收:新增第二本英语词典——ECDICT 常用子集(约 8.2 万条中文释义,3.0 MB)与 CMUdict 转写的 IPA 音标。音标分三层来源:能由 CMUdict 音素按固定映射转写的用真 IPA(35,799 条),否则保留 ECDICT 记法仅做字符级规范化并标注来源(30,580 条),两者都不可靠就不显示(16,342 条,例如 `grok`);含 `^` 等丢失首音的记法整条丢弃,不做猜测。schema 升到 v12(放开 `lexgo_dictionaries` 单行 CHECK、增加 `provider` 列,两步都带条件守卫、可重放),英英与英汉词典可分别启停,查词时中文释义在前、英英释义随后,屈折形经 WordNet lemma 解析后也能看到中文释义但不会复制词目音标。归档由 `scripts/dict_prepare.py` 从 sha256 pin 的三个源显式准备(运行时不联网,产物不入库,仓库只保存 pin 与两份许可证),整包存进 MySQL 因而包含在 #15 的备份范围内。测试:Go 96 项、学习端 159 单测与 26 项 e2e(含 390×844 触摸视口)、管理端 32 项、真实 API 28 项检查;PR #41 待合入,尚未验收。发音音频、整句翻译与汉英方向不在本单范围。
+18 -1
View File
@@ -76,7 +76,15 @@ test('paste English text, watch a chapter finish processing, then read it verbat
} else if (path === '/api/v1/terms/9') data = { term: savedTerm }
else if (path === '/api/v1/lookup') {
expect(route.request().postDataJSON()).toEqual({ chapterId: 55, start: 0, end: 5 })
data = { status: 'exact', query: 'first', matchedForm: 'first', candidates: [], entries: [{ lemma: 'first', pos: 'adjective', definition: 'Coming before all others.', examples: ['The first fictional chapter.'] }] }
data = {
status: 'exact', query: 'first', matchedForm: 'first', candidates: [], phonetic: 'fɜːst', phoneticSource: 'cmudict',
resources: [{ name: '简明英汉字典增强版(ECDICT)+ CMUdict 音标', version: 'ecdict-1.0.28+cmudict-0.7b' }, { name: 'Princeton WordNet', version: '3.0' }],
resource: { name: 'Princeton WordNet', version: '3.0' },
entries: [
{ lemma: 'first', pos: '', definition: '', examples: [], translation: 'a. 第一的, 最早的\nn. 开始, 第一', provider: 'ecdict' },
{ lemma: 'first', pos: 'adjective', definition: 'Coming before all others.', examples: ['The first fictional chapter.'], provider: 'wordnet' },
],
}
}
await route.fulfill({ status: statusCode, json: { code: 200, data } })
})
@@ -119,12 +127,21 @@ test('paste English text, watch a chapter finish processing, then read it verbat
await word.focus()
await word.press('Enter')
await expect(page.getByText('Coming before all others.')).toBeVisible()
// The merged dictionary answers with the IPA phonetic and the Chinese gloss first, then the
// English detail from WordNet; both directories are named in the footnote.
await expect(page.getByTestId('lookup-phonetic')).toHaveText('/fɜːst/')
await expect(page.getByTestId('lookup-translation')).toContainText('第一的')
const senses = page.locator('.lookup-senses li')
await expect(senses.nth(0)).toHaveAttribute('data-provider', 'ecdict')
await expect(senses.nth(1)).toHaveAttribute('data-provider', 'wordnet')
await expect(page.locator('.lookup-panel')).toContainText('Princeton WordNet')
expect(await readerText.evaluate(element => element.textContent)).toBe(pasted)
// Browser narrow viewport check only; this is not real-device acceptance.
await page.setViewportSize({ width: 390, height: 844 })
await word.click()
await expect(page.getByText('Coming before all others.')).toBeVisible()
await expect(page.getByTestId('lookup-phonetic')).toHaveText('/fɜːst/')
const panelBounds = await page.locator('.lookup-panel').boundingBox()
expect(panelBounds!.y + panelBounds!.height).toBeLessThanOrEqual(845)
expect(panelBounds!.height).toBeLessThanOrEqual(844 * 0.46)
+32
View File
@@ -69,6 +69,38 @@ describe('reader word lookup', () => {
expect(view.text()).toContain('Definition of cat')
expect(view.text()).toContain('A fictional example.')
})
it('shows the IPA phonetic and the Chinese gloss ahead of the English detail', async () => {
const merged = {
status: 'exact', query: 'cat', matchedForm: 'cat', candidates: ['cat'], phonetic: 'kæt', phoneticSource: 'cmudict',
resources: [{ name: '简明英汉字典增强版', version: 'ecdict-1.0.28+cmudict-0.7b' }, { name: 'Princeton WordNet', version: '3.0' }],
resource: { name: 'Princeton WordNet', version: '3.0' },
entries: [
{ lemma: 'cat', pos: '', definition: '', examples: [], translation: 'n. 猫, 恶妇\nvi. 呕吐', phonetic: 'kæt', phoneticSource: 'cmudict', provider: 'ecdict' },
{ lemma: 'cat', pos: 'n', definition: 'feline animal', examples: [], provider: 'wordnet' },
],
}
const { view } = await open({ lookup: async () => ok(merged) })
await view.get('.reader-word').trigger('click'); await flushPromises()
expect(view.get('[data-testid="lookup-phonetic"]').text()).toBe('/kæt/')
expect(view.get('[data-testid="lookup-translation"]').text()).toContain('猫')
expect(view.text()).toContain('feline animal')
const senses = view.findAll('.lookup-senses li')
expect(senses[0]!.attributes('data-provider')).toBe('ecdict')
expect(senses[1]!.attributes('data-provider')).toBe('wordnet')
expect(view.text()).toContain('简明英汉字典增强版')
expect(view.text()).toContain('Princeton WordNet')
})
it('shows no phonetic for an inflected form resolved to a headword', async () => {
const inflected = {
status: 'lemma', query: 'cats', matchedForm: 'cat', candidates: ['cat'],
entries: [{ lemma: 'cat', pos: '', definition: '', examples: [], translation: 'n. 猫', provider: 'ecdict' }],
}
const { view } = await open({ lookup: async () => ok(inflected) })
await view.get('.reader-word').trigger('click'); await flushPromises()
expect(view.find('[data-testid="lookup-phonetic"]').exists()).toBe(false)
expect(view.get('[data-testid="lookup-translation"]').text()).toContain('猫')
expect(view.text()).toContain('词形匹配')
})
it('discards older word responses and clears the selection form', async () => {
let finish!: (response: Response) => void
const { view } = await open({ lookup: body => body.start === 2 ? new Promise(resolve => { finish = resolve }) : Promise.resolve(ok(result('cafe', 'not_found'))) })
+11 -5
View File
@@ -48,7 +48,10 @@ onUnmounted(() => { observer?.disconnect(); window.removeEventListener('resize',
<template>
<aside ref="panel" class="lookup-panel" aria-labelledby="lookup-heading" @keydown.esc.stop.prevent="$emit('close')">
<header class="lookup-heading">
<h2 id="lookup-heading" ref="heading" tabindex="-1">{{ word }}</h2>
<h2 id="lookup-heading" ref="heading" tabindex="-1">
{{ word }}
<span v-if="result?.phonetic" class="lookup-phonetic" data-testid="lookup-phonetic" lang="en">/{{ result.phonetic }}/</span>
</h2>
<ElButton text aria-label="关闭释义" @click="$emit('close')">关闭</ElButton>
</header>
<div v-if="phrase" class="lookup-range" data-testid="phrase-range">
@@ -68,13 +71,16 @@ onUnmounted(() => { observer?.disconnect(); window.removeEventListener('resize',
<p v-else-if="result.status === 'not_found'" class="lookup-message">未找到释义。</p>
<p v-if="result.status === 'lemma'" class="subtle">词形匹配:{{ result.matchedForm }}(规则候选)</p>
<ol v-if="result.entries.length" class="lookup-senses">
<li v-for="(entry, index) in result.entries" :key="index">
<p class="sense-heading"><strong>{{ entry.lemma }}</strong> <span>{{ entry.pos }}</span></p>
<p lang="en">{{ entry.definition }}</p>
<li v-for="(entry, index) in result.entries" :key="index" :data-provider="entry.provider || ''">
<p class="sense-heading"><strong>{{ entry.lemma }}</strong> <span v-if="entry.pos">{{ entry.pos }}</span></p>
<p v-if="entry.translation" class="lookup-translation" data-testid="lookup-translation">{{ entry.translation }}</p>
<p v-if="entry.definition" lang="en">{{ entry.definition }}</p>
<blockquote v-for="(example, exampleIndex) in entry.examples" :key="exampleIndex" lang="en">{{ example }}</blockquote>
</li>
</ol>
<p v-if="result.resource" class="subtle">{{ result.resource.name }} · {{ result.resource.version }}</p>
<p v-if="result.resources?.length || result.resource" class="subtle">
{{ (result.resources?.length ? result.resources : [result.resource!]).map(item => `${item.name} · ${item.version}`).join(';') }}
</p>
</template>
<ElButton v-if="!loading && (error || result?.status === 'resource_missing' || result?.status === 'not_found')" data-testid="lookup-retry" @click="$emit('retry')">重试查询</ElButton>
</div>
+17 -1
View File
@@ -17,13 +17,29 @@ export interface ReaderToken {
}
/** One saved phrase occurrence inside the chapter, in token indices. */
export interface PhraseSpan { id: number; status: TermStatus; wordCount: number; startToken: number; endToken: number }
export interface LookupEntry {
lemma: string
pos: string
definition: string
examples: string[]
/** Chinese gloss and phonetic, present when the 英汉 dictionary is enabled. */
translation?: string
phonetic?: string
/** "cmudict" for a CMUdict IPA transcription, "ecdict" for the ECDICT notation. */
phoneticSource?: string
provider?: string
}
export interface LookupResult {
status: 'exact' | 'lemma' | 'not_found' | 'resource_missing'
query: string
matchedForm: string | null
candidates: string[]
entries: { lemma: string; pos: string; definition: string; examples: string[] }[]
entries: LookupEntry[]
resource?: { name: string; version: string }
resources?: { name: string; version: string }[]
/** The headword phonetic, repeated at the top level so the panel can show it next to the word. */
phonetic?: string
phoneticSource?: string
}
export interface SavedTerm {
id: number
+2
View File
@@ -254,6 +254,8 @@ a.chapter-name:hover { color: var(--accent); text-decoration: underline; }
.lookup-senses li { padding-left: 3px; margin-bottom: 20px; }
.lookup-senses p { margin: 8px 0; }
.sense-heading span { color: var(--text-muted); font-size: 13px; }
.lookup-phonetic { color: var(--text-muted); font-size: 15px; font-weight: 400; margin-left: 8px; }
.lookup-translation { white-space: pre-line; }
.lookup-senses blockquote { border-left: 2px solid var(--status-ready-border); margin: 10px 0; padding-left: 12px; color: var(--text-faint); font-style: italic; }
.lookup-range { border-bottom: 1px solid var(--border); padding-bottom: 14px; }
.lookup-range p { margin: 0 0 10px; }
+350
View File
@@ -0,0 +1,350 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Build the prepared 英汉词典 resource for LexGo (#40).
This is tooling, not product runtime: the Go server only imports the prepared ZIP.
Two pinned sources are combined:
* ECDICT StarDict (MIT) -> Chinese glosses and a fallback phonetic notation
* CMUdict (BSD-2) -> ARPAbet phonemes transcribed to IPA
The subset is limited to lowercase single-word headwords that either exist in the
WordNet lemma list (pinned in server/wordnet-resource.json) or carry an ECDICT
exam tag, which keeps the resource at a few MiB instead of the full 340 万条.
Everything is verified: source sha256, subset size within the pinned tolerance and
the phonetic character set (no character outside the IPA alphabet may survive).
Usage:
python scripts/dict_prepare.py # download + build into .local/dictionaries
python scripts/dict_prepare.py --cache-dir DIR # reuse already downloaded sources
python scripts/dict_prepare.py --keep-going # still write the artifact if counts drift
"""
from __future__ import annotations
import argparse
import gzip
import hashlib
import json
import re
import struct
import sys
import urllib.error
import urllib.parse
import urllib.request
import zipfile
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
PIN_FILE = ROOT / "server" / "zh-dictionary-resource.json"
DEFAULT_OUT = ROOT / ".local" / "dictionaries" / "zh-dict-v1.zip"
DEFAULT_CACHE = ROOT / ".local" / "dictionaries" / "cache"
EXAM_TAG_RE = re.compile(r"\((?:[^)]*?(研|四|六|托|专|雅|高|中|初))[^)]*?\)")
PHONETIC_RE = re.compile(r"^\*?\[([^\]\u4e00-\u9fff]*)\]")
CJK_RE = re.compile(r"[\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]")
SPACE_RE = re.compile(r"\s+")
# ARPAbet (CMUdict, American English) -> IPA, a fixed documented convention.
ARPABET_CONSONANTS = {
"B": "b", "CH": "t͡ʃ", "D": "d", "DH": "ð", "F": "f", "G": "ɡ", "HH": "h",
"JH": "d͡ʒ", "K": "k", "L": "l", "M": "m", "N": "n", "NG": "ŋ", "P": "p",
"R": "ɹ", "S": "s", "SH": "ʃ", "T": "t", "TH": "θ", "V": "v", "W": "w",
"Y": "j", "Z": "z", "ZH": "ʒ",
}
ARPABET_VOWELS = {
"AA": ("ɑ", "ː"), "AE": ("æ", ""), "AH": ("ʌ", ""), "AO": ("ɔ", "ː"),
"AW": ("a", "ʊ"), "AY": ("a", "ɪ"), "EH": ("e", ""), "ER": ("ɜ", "ː"),
"EY": ("e", "ɪ"), "IH": ("ɪ", ""), "IY": ("i", "ː"), "OW": ("o", "ʊ"),
"OY": ("ɔ", "ɪ"), "UH": ("ʊ", ""), "UW": ("u", "ː"),
}
STRESS_MARKS = {"1": "ˈ", "2": "ˌ", "0": ""}
# Only character-level substitutions whose meaning is verifiable from the data.
ECDICT_CHAR_MAP = {
"'": "ˈ", ",": "ˌ", ".": "ˌ", "ˊ": "ˈ", ":": "ː", "ә": "ə", "ε": "e", # ә is U+04D9, ε U+03B5
" ": "", "-": "", "=": "", ";": "", "^": "", # ^ is a stray separator mid-word
}
IPA_ALPHABET = set(
"abcdefghijklmnopqrstuvwxyz"
"æɑɒɔəɜɛɪʊʌʃʒθðŋɹɡɚɝt͡ʃd͡ʒ"
"ːˈˌ"
)
def log(message: str) -> None:
print(message, flush=True)
def digest(path: Path) -> str:
hasher = hashlib.sha256()
with path.open("rb") as handle:
for block in iter(lambda: handle.read(1024 * 1024), b""):
hasher.update(block)
return hasher.hexdigest()
def download(source: dict, cache_dir: Path) -> Path:
"""Download a pinned source, trying every mirror, and verify sha256 + size."""
name = source["name"]
target = cache_dir / source_filename(source["source"])
urls = [source["source"]] + list(source.get("mirrors", []))
if target.exists() and digest(target) == source["sha256"]:
log(f"[cache] {name}: {target.name} ({target.stat().st_size} 字节, sha256 匹配)")
return target
last_error = None
for url in urls:
log(f"[download] {name}: {url}")
try:
request = urllib.request.Request(url, headers={"User-Agent": "lexgo-dict-prepare/1"})
with urllib.request.urlopen(request, timeout=300) as response, target.open("wb") as handle:
while True:
block = response.read(1024 * 256)
if not block:
break
handle.write(block)
except (urllib.error.URLError, TimeoutError, OSError) as error:
last_error = error
log(f"[download] {name}: 失败 {type(error).__name__}: {error}")
continue
found = digest(target)
if found == source["sha256"]:
log(f"[download] {name}: 完成 {target.stat().st_size} 字节, sha256 匹配")
return target
log(f"[download] {name}: sha256 不匹配 ({found[:16]}… != {source['sha256'][:16]}…),尝试下一个通道")
raise SystemExit(f"{name}: 所有通道都无法取得通过校验的资源(最后错误 {last_error})")
def source_filename(url: str) -> str:
return Path(urllib.parse.urlparse(url).path).name
def wordnet_lemmas(archive: Path) -> set:
lemmas = set()
with zipfile.ZipFile(archive) as bundle:
for name in bundle.namelist():
if "/index." not in name:
continue
for line in bundle.read(name).decode("latin-1").splitlines():
if not line.strip() or line.startswith(" "):
continue
lemmas.add(line.split(" ", 1)[0].lower().replace("_", " "))
return lemmas
def cmudict_entries(path: Path) -> dict:
entries = {}
for line in path.read_text(encoding="latin-1").splitlines():
if line.startswith(";;;") or not line.strip():
continue
head, _, rest = line.partition(" ")
head = head.split("(", 1)[0].lower()
if head and rest.strip() and head not in entries:
entries[head] = rest.split()
return entries
def stardict_entries(archive: Path):
with zipfile.ZipFile(archive) as bundle:
names = bundle.namelist()
ifo_name = next((n for n in names if n.endswith(".ifo")), None)
idx_name = next((n for n in names if n.endswith(".idx")), None)
dict_name = next((n for n in names if n.endswith(".dict")), None)
if not (ifo_name and idx_name and dict_name):
raise SystemExit("ECDICT 归档缺少 .ifo/.idx/.dict")
ifo = bundle.read(ifo_name).decode("utf-8", "replace")
sequence = ""
for line in ifo.splitlines():
if line.startswith("sametypesequence="):
sequence = line.split("=", 1)[1].strip()
if sequence != "m":
raise SystemExit(f"未预期的 StarDict sametypesequence: {sequence!r}(只处理纯文本 m)")
index = bundle.read(idx_name)
payload = bundle.read(dict_name)
offset = 0
while offset < len(index):
end = index.index(b"\x00", offset)
word = index[offset:end].decode("utf-8", "replace")
start, size = struct.unpack(">II", index[end + 1:end + 9])
yield word, payload[start:start + size].decode("utf-8", "replace")
offset = end + 9
def arpabet_to_ipa(phonemes) -> str:
vowel_count = sum(1 for phoneme in phonemes if phoneme.rstrip("012").strip() in ARPABET_VOWELS)
parts = []
for phoneme in phonemes:
stress = ""
base = phoneme
if phoneme and phoneme[-1].isdigit():
base, digit = phoneme[:-1], phoneme[-1]
stress = STRESS_MARKS.get(digit, "")
if base in ARPABET_VOWELS and vowel_count == 1:
# A monosyllable carries no stress mark in IPA (cat is /kæt/, not /kˈæt/).
stress = ""
if base in ARPABET_CONSONANTS:
parts.append(ARPABET_CONSONANTS[base])
elif base in ARPABET_VOWELS:
head, tail = ARPABET_VOWELS[base]
if stress == "":
# Unstressed vowels are short: curiosity ends /əti/, not /ətiː/.
tail = ""
if base == "AH":
head = "ə"
elif base == "ER":
head = "ɚ"
parts.append(stress + head + tail)
else:
return ""
return "".join(parts)
def normalise_ecdict_phonetic(raw: str) -> str:
text = raw.strip()
# A phonetic that starts with the stray caret has lost its first sound (grok is stored as
# "^rɔk"): showing "rɔk" would be a wrong transcription, so the whole value is dropped.
if text.startswith("^"):
return ""
for old, new in ECDICT_CHAR_MAP.items():
text = text.replace(old, new)
return SPACE_RE.sub("", text)
def build_subset(lemmas: set, exam_entries) -> dict:
subset = {}
for word, body in exam_entries:
lower = word.lower()
if " " in word or not lower.isalpha() or not lower.islower() or lower in subset:
continue
if lower not in lemmas and not EXAM_TAG_RE.search(body):
continue
subset[lower] = body
return subset
def translation_of(body: str) -> str:
lines = []
for line in body.splitlines():
stripped = PHONETIC_RE.sub("", line).strip()
if not stripped or not CJK_RE.search(stripped):
continue
# "(研四六托 4518/5059)" is an exam-list rank and "[时态] guarded, guarding" is a word-form
# list: neither is a meaning, so they stay out of the gloss.
if stripped.startswith("(") and stripped.endswith(")"):
continue
if stripped.startswith("[时态]"):
continue
lines.append(stripped)
return "\n".join(lines)
def main() -> int:
parser = argparse.ArgumentParser(description="构建 LexGo 英汉词典资源 (#40)")
parser.add_argument("--out", default=str(DEFAULT_OUT))
parser.add_argument("--cache-dir", default=str(DEFAULT_CACHE))
parser.add_argument("--keep-going", action="store_true", help="计数漂移时仍产出,只在 manifest 标记")
args = parser.parse_args()
pin = json.loads(PIN_FILE.read_text(encoding="utf-8"))
sources = {source["role"]: source for source in pin["sources"]}
cache_dir = Path(args.cache_dir)
cache_dir.mkdir(parents=True, exist_ok=True)
ecdict_zip = download(sources["chinese-gloss"], cache_dir)
cmudict_file = download(sources["ipa-phonetic"], cache_dir)
wordnet_zip = download(sources["subset-wordlist"], cache_dir)
log("[parse] WordNet lemma 词表")
lemmas = wordnet_lemmas(wordnet_zip)
log(f"[parse] WordNet lemma: {len(lemmas)}")
log("[parse] CMUdict 音素")
cmu = cmudict_entries(cmudict_file)
log(f"[parse] CMUdict 词条: {len(cmu)}")
log("[build] 裁剪 ECDICT 子集")
subset = build_subset(lemmas, stardict_entries(ecdict_zip))
log(f"[build] 子集词条: {len(subset)}")
records = []
stats = {"cmudict": 0, "ecdict": 0, "none": 0, "droppedPhonetic": 0, "empty": 0}
offenders = {}
for word in sorted(subset):
body = subset[word]
translation = translation_of(body)
phonetic = ""
source_name = ""
if word in cmu:
phonetic = arpabet_to_ipa(cmu[word])
source_name = "cmudict" if phonetic else ""
if not phonetic:
match = PHONETIC_RE.match(body)
candidate = normalise_ecdict_phonetic(match.group(1)) if match else ""
if candidate:
phonetic = candidate
source_name = "ecdict"
if phonetic:
unknown = sorted({char for char in phonetic if char not in IPA_ALPHABET})
if unknown:
stats["droppedPhonetic"] += 1
for char in unknown:
offenders[char] = offenders.get(char, 0) + 1
phonetic = ""
source_name = ""
if not translation and not phonetic:
# An entry that lost both its gloss and its phonetic answers nothing, so it is dropped
# instead of occupying a lookup slot with a bare headword.
stats["empty"] += 1
continue
stats[source_name or "none"] += 1
records.append({
"w": word,
"t": translation,
"p": phonetic,
"ps": source_name,
})
if offenders:
log(f"[check] 丢弃的非 IPA 字符: {sorted((c, n) for c, n in offenders.items())}")
if stats["droppedPhonetic"]:
log(f"[check] 因字符不合法丢弃音标 {stats['droppedPhonetic']} 条(不做语义猜测)")
if stats["empty"]:
log(f"[check] 既无释义也无音标丢弃 {stats['empty']} 条")
expected = pin["subset"]["expectedEntries"]
tolerance = pin["subset"]["toleranceEntries"]
drift = abs(len(records) - expected)
log(f"[check] 条目数 {len(records)}(pin 期望 {expected} ± {tolerance})")
if drift > tolerance and not args.keep_going:
raise SystemExit("子集条目数超出 pin 容差,拒绝产出;确认数据源变化后用 --keep-going 重新评估")
if len(records) == 0:
raise SystemExit("子集为空,拒绝产出")
manifest = {
"format": pin["format"],
"version": pin["version"],
"language": pin["language"],
"provider": pin["provider"],
"entries": len(records),
"phonetic": {"cmudict": stats["cmudict"], "ecdict": stats["ecdict"], "none": stats["none"]},
"droppedPhonetic": stats["droppedPhonetic"],
"droppedEmpty": stats["empty"],
"subsetRule": pin["subset"]["rule"],
"sources": [
{"name": s["name"], "role": s["role"], "sha256": s["sha256"], "license": s["license"]}
for s in pin["sources"]
],
"preparedBy": "scripts/dict_prepare.py",
}
destination = Path(args.out)
destination.parent.mkdir(parents=True, exist_ok=True)
with zipfile.ZipFile(destination, "w", zipfile.ZIP_DEFLATED) as bundle:
payload = "\n".join(json.dumps(record, ensure_ascii=False, sort_keys=True) for record in records)
bundle.writestr("entries.jsonl.gz", gzip.compress(payload.encode("utf-8")))
bundle.writestr("manifest.json", json.dumps(manifest, ensure_ascii=False, indent=2))
log(f"[write] {destination} ({destination.stat().st_size} 字节)")
log(f"[write] sha256 {digest(destination)}")
log(f"[stats] CMUdict IPA {stats['cmudict']} / ECDICT 记法 {stats['ecdict']} / 无音标 {stats['none']}")
return 0
if __name__ == "__main__":
sys.exit(main())
+33
View File
@@ -0,0 +1,33 @@
Copyright (C) 1993-2015 Carnegie Mellon University. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
The contents of this file are deemed to be source code.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
This work was supported in part by funding from the Defense Advanced
Research Projects Agency, the Office of Naval Research and the National
Science Foundation of the United States of America, and by member
companies of the Carnegie Mellon Sphinx Speech Consortium. We acknowledge
the contributions of many volunteers to the expansion and improvement of
this dictionary.
THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2025 Linwei
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+253
View File
@@ -0,0 +1,253 @@
package lexgo
import (
"archive/zip"
"bufio"
"bytes"
"compress/gzip"
"encoding/json"
"errors"
"io"
"path"
"strings"
"unicode"
)
// The prepared 英汉 resource is built by scripts/dict_prepare.py from two pinned sources and is
// imported as a whole; the server only validates and reads it. Chinese glosses and IPA phonetics
// are additive to WordNet, so they live in their own corpus with their own provider identity.
const (
ChineseDictionaryFormat = "lexgo-zh-dict-v1"
ChineseDictionaryVersion = "ecdict-1.0.28+cmudict-0.7b"
ChineseDictionarySource = "https://github.com/skywind3000/ECDICT/releases/download/1.0.28/ecdict-stardict-28.zip"
ChineseDictionarySHA = "" // verified from the archive manifest, see chineseDictionarySources
chineseEntryLimit = 200000
)
// Pinned source checksums: an imported archive must declare exactly these, so a hand-edited or
// unknown resource cannot be loaded as if it were the reviewed one.
var chineseDictionarySources = map[string]string{
"chinese-gloss": "c707d0f3ded6ec79b96466da4a1574e074703da5af9c120fbad97f9cb08c6f2c",
"ipa-phonetic": "81917843c7f44ce2b094ac63873c2c7a4cf802040792c455ba3ca406891c3d22",
"subset-wordlist": "cbda5ea6eef7f36a97a43d4a75f85e07fccbb4f23657d27b4ccbc93e2646ab59",
}
type chineseManifest struct {
Format string `json:"format"`
Version string `json:"version"`
Language string `json:"language"`
Provider string `json:"provider"`
Entries int `json:"entries"`
PreparedBy string `json:"preparedBy"`
Sources []struct {
Name string `json:"name"`
Role string `json:"role"`
SHA256 string `json:"sha256"`
License string `json:"license"`
} `json:"sources"`
}
type chineseRecord struct {
Word string `json:"w"`
Translation string `json:"t"`
Phonetic string `json:"p"`
Source string `json:"ps"`
}
// ChineseDictionary is an immutable lookup table for one imported resource.
type ChineseDictionary struct {
entries map[string]DictionaryEntry
EntryCount int
}
func (d *ChineseDictionary) Lookup(query string) LookupResult {
key := normalizeWord(query)
result := LookupResult{Status: "not_found", Query: query, Candidates: []string{}, Entries: []DictionaryEntry{}}
entry, ok := d.entries[key]
if !ok {
return result
}
form := key
result.Status = "exact"
result.MatchedForm = &form
result.Candidates = []string{form}
result.Entries = []DictionaryEntry{entry}
result.Phonetic = entry.Phonetic
result.PhoneticSource = entry.PhoneticSource
return result
}
// readChineseArchive bounds every declared and inflated size before reading, mirroring the WordNet
// loader so a hostile archive cannot exhaust memory.
func readChineseArchive(raw []byte) (map[string][]byte, error) {
if len(raw) > maxDictionaryZip {
return nil, errors.New("archive too large")
}
reader, err := zip.NewReader(bytes.NewReader(raw), int64(len(raw)))
if err != nil {
return nil, err
}
if len(reader.File) > 8 {
return nil, errors.New("too many archive members")
}
files := map[string][]byte{}
total := int64(0)
for _, f := range reader.File {
name := f.Name
if strings.Contains(name, "\\") || path.IsAbs(name) || strings.Contains(name, ":") || path.Clean(name) != strings.TrimSuffix(name, "/") || (name != "entries.jsonl.gz" && name != "manifest.json") {
return nil, errors.New("unsafe archive member")
}
if !f.FileInfo().IsDir() && !f.Mode().IsRegular() {
return nil, errors.New("unsupported archive member")
}
if _, seen := files[name]; seen {
return nil, errors.New("duplicate archive member")
}
if f.UncompressedSize64 > maxDictionaryInflated || total+int64(f.UncompressedSize64) > maxDictionaryInflated {
return nil, errors.New("inflated archive too large")
}
rc, err := f.Open()
if err != nil {
return nil, err
}
data, err := io.ReadAll(io.LimitReader(rc, maxDictionaryInflated-total+1))
rc.Close()
if err != nil {
return nil, err
}
total += int64(len(data))
if total > maxDictionaryInflated {
return nil, errors.New("inflated archive too large")
}
files[name] = data
}
if len(files["entries.jsonl.gz"]) == 0 || len(files["manifest.json"]) == 0 {
return nil, errors.New("missing required member")
}
return files, nil
}
// ParseChineseDictionary accepts only an archive whose manifest matches the reviewed format,
// version and source checksums. No member is ever extracted to disk.
func ParseChineseDictionary(raw []byte) (*ChineseDictionary, error) {
if len(raw) == 0 || len(raw) > maxDictionaryZip {
return nil, errors.New("archive size out of range")
}
files, err := readChineseArchive(raw)
if err != nil {
return nil, err
}
var manifest chineseManifest
if err = json.Unmarshal(files["manifest.json"], &manifest); err != nil {
return nil, errors.New("invalid manifest")
}
if manifest.Format != ChineseDictionaryFormat || manifest.Version != ChineseDictionaryVersion || manifest.Language != "en" || manifest.Provider != "ecdict" {
return nil, errors.New("unexpected dictionary manifest")
}
if len(manifest.Sources) != len(chineseDictionarySources) {
return nil, errors.New("unexpected dictionary sources")
}
for _, source := range manifest.Sources {
if expected, ok := chineseDictionarySources[source.Role]; !ok || expected != source.SHA256 {
return nil, errors.New("unreviewed dictionary source")
}
}
reader, err := gzip.NewReader(bytes.NewReader(files["entries.jsonl.gz"]))
if err != nil {
return nil, errors.New("invalid entry payload")
}
defer reader.Close()
dictionary := &ChineseDictionary{entries: map[string]DictionaryEntry{}}
scanner := bufio.NewScanner(reader)
scanner.Buffer(make([]byte, 4096), 1<<20)
for scanner.Scan() {
line := scanner.Bytes()
if len(bytes.TrimSpace(line)) == 0 {
continue
}
var record chineseRecord
if err = json.Unmarshal(line, &record); err != nil {
return nil, errors.New("invalid entry record")
}
word := strings.TrimSpace(record.Word)
if word == "" || word != normalizeWord(word) || !isDictionaryHeadword(word) || record.Translation == "" && record.Phonetic == "" {
return nil, errors.New("invalid entry")
}
if _, exists := dictionary.entries[word]; exists {
return nil, errors.New("duplicate entry")
}
if len(dictionary.entries) >= chineseEntryLimit {
return nil, errors.New("too many entries")
}
dictionary.entries[word] = DictionaryEntry{
Lemma: word,
Translation: record.Translation,
Phonetic: record.Phonetic,
PhoneticSource: record.Source,
Provider: "ecdict",
}
}
if err = scanner.Err(); err != nil {
return nil, err
}
if len(dictionary.entries) == 0 || manifest.Entries != len(dictionary.entries) {
return nil, errors.New("entry count mismatch")
}
dictionary.EntryCount = len(dictionary.entries)
return dictionary, nil
}
// mergeLookups keeps every provider's senses in caller order: the Chinese gloss first because it
// answers "what does this mean" for the learner, WordNet afterwards as English detail.
func mergeLookups(query string, results []LookupResult) LookupResult {
merged := LookupResult{Status: "not_found", Query: query, Candidates: []string{}, Entries: []DictionaryEntry{}}
for _, result := range results {
if result.Status == "exact" {
merged.Status = "exact"
} else if result.Status == "lemma" && merged.Status == "not_found" {
merged.Status = "lemma"
}
for _, candidate := range result.Candidates {
if !containsString(merged.Candidates, candidate) {
merged.Candidates = append(merged.Candidates, candidate)
}
}
if merged.MatchedForm == nil && result.MatchedForm != nil {
form := *result.MatchedForm
merged.MatchedForm = &form
}
for _, entry := range result.Entries {
if len(merged.Entries) >= 12 {
break
}
// A phonetic belongs to the headword: an inflected form resolved through WordNet's
// morphology shows the gloss but not the headword's transcription.
if entry.Phonetic != "" && merged.Phonetic == "" && entry.Lemma == normalizeWord(query) {
merged.Phonetic = entry.Phonetic
merged.PhoneticSource = entry.PhoneticSource
}
merged.Entries = append(merged.Entries, entry)
}
}
return merged
}
// isDictionaryHeadword keeps the stored key shape predictable: letters with the apostrophe and
// hyphen that appear in real headwords, nothing that could smuggle in markup or a sentence.
func isDictionaryHeadword(word string) bool {
for _, r := range word {
if !unicode.IsLetter(r) && r != '\'' && r != '-' {
return false
}
}
return true
}
func containsString(values []string, wanted string) bool {
for _, value := range values {
if value == wanted {
return true
}
}
return false
}
+351
View File
@@ -0,0 +1,351 @@
package lexgo
import (
"archive/zip"
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"mime/multipart"
"net/http/httptest"
"os"
"strings"
"testing"
"time"
"github.com/gin-gonic/gin"
admin "go-admin/app/admin/models"
)
const chineseFixturePath = "../../../.local/dictionaries/zh-dict-v1.zip"
func chineseFixture(t *testing.T) []byte {
t.Helper()
raw, err := os.ReadFile(chineseFixturePath)
if os.IsNotExist(err) {
t.Skip("prepare the 英汉 fixture with python scripts/dict_prepare.py")
}
if err != nil {
t.Fatal(err)
}
return raw
}
// buildChineseArchive writes a synthetic archive so rejection paths can be exercised without a
// hand-edited copy of the real 3 MiB resource.
func buildChineseArchive(t *testing.T, manifest map[string]any, entries []map[string]any) []byte {
t.Helper()
if _, ok := manifest["format"]; !ok {
manifest["format"] = ChineseDictionaryFormat
}
if _, ok := manifest["version"]; !ok {
manifest["version"] = ChineseDictionaryVersion
}
if _, ok := manifest["language"]; !ok {
manifest["language"] = "en"
}
if _, ok := manifest["provider"]; !ok {
manifest["provider"] = "ecdict"
}
if _, ok := manifest["entries"]; !ok {
manifest["entries"] = len(entries)
}
if _, ok := manifest["sources"]; !ok {
manifest["sources"] = []map[string]string{
{"name": "ECDICT StarDict", "role": "chinese-gloss", "sha256": chineseDictionarySources["chinese-gloss"], "license": "ECDICT-LICENSE.txt"},
{"name": "CMUdict", "role": "ipa-phonetic", "sha256": chineseDictionarySources["ipa-phonetic"], "license": "CMUDICT-LICENSE.txt"},
{"name": "Princeton WordNet", "role": "subset-wordlist", "sha256": chineseDictionarySources["subset-wordlist"], "license": "WORDNET-LICENSE.txt"},
}
}
lines := make([]string, 0, len(entries))
for _, entry := range entries {
encoded, err := json.Marshal(entry)
if err != nil {
t.Fatal(err)
}
lines = append(lines, string(encoded))
}
var payload bytes.Buffer
writer := gzip.NewWriter(&payload)
writer.Write([]byte(strings.Join(lines, "\n")))
writer.Close()
var archive bytes.Buffer
bundle := zip.NewWriter(&archive)
for name, content := range map[string][]byte{
"manifest.json": mustJSON(t, manifest),
"entries.jsonl.gz": payload.Bytes(),
} {
part, err := bundle.Create(name)
if err != nil {
t.Fatal(err)
}
part.Write(content)
}
if err := bundle.Close(); err != nil {
t.Fatal(err)
}
return archive.Bytes()
}
func mustJSON(t *testing.T, value any) []byte {
t.Helper()
encoded, err := json.Marshal(value)
if err != nil {
t.Fatal(err)
}
return encoded
}
func TestChineseDictionaryParseAndLookup(t *testing.T) {
dictionary, err := ParseChineseDictionary(chineseFixture(t))
if err != nil {
t.Fatal(err)
}
if dictionary.EntryCount != 82721 {
t.Fatalf("entry count %d, want the pinned 82721", dictionary.EntryCount)
}
got := dictionary.Lookup("curiosity")
if got.Status != "exact" || got.Phonetic != "kjˌʊɹiˈɑːsəti" || got.PhoneticSource != "cmudict" {
t.Fatalf("curiosity %+v", got)
}
if len(got.Entries) != 1 || got.Entries[0].Provider != "ecdict" || !strings.Contains(got.Entries[0].Translation, "好奇心") {
t.Fatalf("curiosity entry %+v", got.Entries)
}
if got.Entries[0].Definition != "" || len(got.Entries[0].Examples) != 0 {
t.Fatal("the Chinese resource must not invent English senses")
}
// The reader sends the selected word, so a different case must still resolve.
if upper := dictionary.Lookup("Curiosity"); upper.Status != "exact" || upper.Phonetic != "kjˌʊɹiˈɑːsəti" {
t.Fatalf("case-insensitive query %+v", upper)
}
// A word whose phonetic is unreliable keeps its gloss and shows no transcription.
if grok := dictionary.Lookup("grok"); grok.Status != "exact" || grok.Phonetic != "" || len(grok.Entries) != 1 {
t.Fatalf("grok %+v", grok)
}
if grok := dictionary.Lookup("grok"); !strings.Contains(grok.Entries[0].Translation, "神入") {
t.Fatalf("grok gloss %q", grok.Entries[0].Translation)
}
if miss := dictionary.Lookup("zzzznotaword"); miss.Status != "not_found" || len(miss.Entries) != 0 {
t.Fatalf("miss %+v", miss)
}
}
func TestChineseDictionaryRejectsUnreviewedArchive(t *testing.T) {
entry := []map[string]any{{"w": "grok", "t": "v. 神入", "p": "ɡrɒk", "ps": "cmudict"}}
for name, manifest := range map[string]map[string]any{
"format": {"format": "wordnet-3.0-zip"},
"version": {"version": "ecdict-9.9.9"},
"count": {"entries": 5},
"source": {"sources": []map[string]string{{"name": "ECDICT StarDict", "role": "chinese-gloss", "sha256": strings.Repeat("0", 64), "license": "ECDICT-LICENSE.txt"}}},
"role": {"sources": []map[string]string{{"name": "ECDICT StarDict", "role": "unknown-role", "sha256": chineseDictionarySources["chinese-gloss"], "license": "ECDICT-LICENSE.txt"}}},
} {
if _, err := ParseChineseDictionary(buildChineseArchive(t, manifest, entry)); err == nil {
t.Fatalf("%s: unreviewed archive accepted", name)
}
}
// The reviewed shape still parses, so the rejection cases above are not vacuous.
if _, err := ParseChineseDictionary(buildChineseArchive(t, map[string]any{}, entry)); err != nil {
t.Fatalf("reviewed archive rejected: %v", err)
}
for name, entrySet := range map[string][]map[string]any{
"duplicate": {{"w": "grok", "t": "a", "p": "", "ps": ""}, {"w": "grok", "t": "b", "p": "", "ps": ""}},
"empty": {{"w": "grok", "t": "", "p": "", "ps": ""}},
"notAWord": {{"w": "grok!", "t": "a", "p": "", "ps": ""}},
"upperCase": {{"w": "Grok", "t": "a", "p": "", "ps": ""}},
} {
if _, err := ParseChineseDictionary(buildChineseArchive(t, map[string]any{}, entrySet)); err == nil {
t.Fatalf("%s: invalid entry accepted", name)
}
}
}
func TestMergeLookupsOrdersGlossBeforeEnglishDetail(t *testing.T) {
chinese := LookupResult{Status: "exact", Query: "dog", Candidates: []string{"dog"}, MatchedForm: form("dog"), Phonetic: "dɔɡ", PhoneticSource: "cmudict", Entries: []DictionaryEntry{{Lemma: "dog", Translation: "n. 狗", Phonetic: "dɔɡ", PhoneticSource: "cmudict", Provider: "ecdict"}}}
english := LookupResult{Status: "exact", Query: "dog", Candidates: []string{"dog"}, MatchedForm: form("dog"), Entries: []DictionaryEntry{{Lemma: "dog", POS: "n", Definition: "a domesticated canine", Provider: "wordnet"}}}
merged := mergeLookups("dog", []LookupResult{chinese, english})
if merged.Status != "exact" || len(merged.Entries) != 2 || merged.Entries[0].Provider != "ecdict" || merged.Entries[1].Provider != "wordnet" {
t.Fatalf("merged %+v", merged)
}
if merged.Phonetic != "dɔɡ" || merged.PhoneticSource != "cmudict" {
t.Fatalf("phonetic %+v", merged)
}
// A headword phonetic must not be shown for an inflected form resolved by the other provider.
inflected := mergeLookups("dogs", []LookupResult{{Status: "lemma", Query: "dogs", MatchedForm: form("dog"), Candidates: []string{"dog"}, Entries: []DictionaryEntry{{Lemma: "dog", Translation: "n. 狗", Phonetic: "dɔɡ", Provider: "ecdict"}}}})
if inflected.Phonetic != "" || inflected.Status != "lemma" {
t.Fatalf("inflected %+v", inflected)
}
lemma := mergeLookups("dogs", []LookupResult{
{Status: "lemma", Query: "dogs", MatchedForm: form("dog"), Candidates: []string{"dog"}, Entries: []DictionaryEntry{{Lemma: "dog", POS: "n", Definition: "canine", Provider: "wordnet"}}},
{Status: "not_found", Query: "dogs"},
})
if lemma.Status != "lemma" || len(lemma.Entries) != 1 || lemma.MatchedForm == nil || *lemma.MatchedForm != "dog" {
t.Fatalf("lemma only %+v", lemma)
}
empty := mergeLookups("dogs", []LookupResult{{Status: "not_found", Query: "dogs"}})
if empty.Status != "not_found" || len(empty.Entries) != 0 {
t.Fatalf("empty %+v", empty)
}
}
func form(value string) *string { return &value }
func importResourceAPI(t *testing.T, r *gin.Engine, token string, fields map[string]string, raw []byte, filename string) (int, DictionaryImportResult) {
t.Helper()
var body bytes.Buffer
w := multipart.NewWriter(&body)
for key, value := range fields {
if err := w.WriteField(key, value); err != nil {
t.Fatal(err)
}
}
part, err := w.CreateFormFile("file", filename)
if err != nil {
t.Fatal(err)
}
part.Write(raw)
w.Close()
req := httptest.NewRequest("POST", "/api/v1/dictionaries/import", &body)
req.Header.Set("Content-Type", w.FormDataContentType())
req.Header.Set("Authorization", "Bearer "+token)
response := httptest.NewRecorder()
r.ServeHTTP(response, req)
var envelope struct {
Data json.RawMessage `json:"data"`
}
json.Unmarshal(response.Body.Bytes(), &envelope)
var result DictionaryImportResult
json.Unmarshal(envelope.Data, &result)
if response.Code != 200 {
t.Logf("import %s response %s", filename, response.Body.String())
}
return response.Code, result
}
// TestMergedDictionaryLookupAPI imports both resources into the dedicated test database and checks
// the learner-visible merge: Chinese gloss and phonetic first, English detail after, with each
// dictionary independently switchable.
func TestMergedDictionaryLookupAPI(t *testing.T) {
db := testDB(t)
wordnetRaw, err := os.ReadFile("../../../.local/nlp-resources/wordnet.zip")
if os.IsNotExist(err) {
t.Skip("prepare WordNet fixture")
}
if err != nil {
t.Fatal(err)
}
chineseRaw := chineseFixture(t)
if err := db.Exec("DELETE FROM lexgo_dictionaries").Error; err != nil {
t.Fatal(err)
}
t.Cleanup(func() { db.Exec("DELETE FROM lexgo_dictionaries") })
r := Router(db, time.Now)
adminUser := admin.SysUser{Username: randomName("zhdict"), Password: fixturePassword, RoleId: 1, Status: "2"}
learner := admin.SysUser{Username: randomName("zhlearner"), Password: fixturePassword, RoleId: 2, Status: "2"}
for _, user := range []*admin.SysUser{&adminUser, &learner} {
if err := db.Create(user).Error; err != nil {
t.Fatal(err)
}
}
adminToken := loginToken(t, r, adminUser.Username, fixturePassword)
learnerToken := loginToken(t, r, learner.Username, fixturePassword)
if code, result := importResourceAPI(t, r, adminToken, map[string]string{"name": "Princeton WordNet", "language": "en", "version": WordNetVersion, "source": WordNetSource, "format": "wordnet-3.0-zip"}, wordnetRaw, "wordnet.zip"); code != 200 || result.Resource.Provider != "wordnet" || result.Resource.ID != WordNetSlot || result.Resource.EntryCount == 0 {
t.Fatalf("wordnet import %d %+v", code, result)
}
code, imported := importResourceAPI(t, r, adminToken, map[string]string{"name": "简明英汉字典增强版", "language": "en", "version": ChineseDictionaryVersion, "source": ChineseDictionarySource, "format": ChineseDictionaryFormat}, chineseRaw, "zh-dict-v1.zip")
if code != 200 || imported.Resource.Provider != "ecdict" || imported.Resource.ID != ChineseSlot || imported.Resource.EntryCount != 82721 {
t.Fatalf("chinese import %d %+v", code, imported)
}
// Wrong metadata must not occupy the slot.
if code, _ := importResourceAPI(t, r, adminToken, map[string]string{"name": "unknown", "language": "en", "version": "1.0", "source": ChineseDictionarySource, "format": ChineseDictionaryFormat}, chineseRaw, "zh-dict-v1.zip"); code != 400 {
t.Fatalf("unreviewed metadata accepted: %d", code)
}
book := Book{OwnerID: learner.UserId, Title: "Fictional", Language: "en", CreatedAt: stamp(time.Now()), UpdatedAt: stamp(time.Now())}
if err := db.Create(&book).Error; err != nil {
t.Fatal(err)
}
original := "Dogs want curiosity."
chapter := Chapter{BookID: book.ID, OwnerID: learner.UserId, Ordinal: 1, Title: "Fictional", OriginalText: original, ContentSHA256: contentSHA(original), CharCount: len([]rune(original)), Status: statusReady, CreatedAt: stamp(time.Now()), UpdatedAt: stamp(time.Now())}
if err := db.Create(&chapter).Error; err != nil {
t.Fatal(err)
}
_, data := callAPI(t, r, "GET", fmt.Sprintf("/api/v1/chapters/%d/tokens", chapter.ID), learnerToken, nil)
var tokens ChapterTokens
json.Unmarshal(data, &tokens)
ranges := map[string][2]int{}
for _, token := range tokens.Tokens {
if token.Text == "Dogs" || token.Text == "curiosity" || token.Text == "went" {
ranges[token.Text] = [2]int{token.StartUtf16, token.EndUtf16}
}
}
if _, ok := ranges["Dogs"]; !ok {
t.Fatalf("token ranges missing: %s", string(data))
}
lookup := func(word string) (int, LookupResult) {
span := ranges[word]
code, payload := callAPI(t, r, "POST", "/api/v1/lookup", learnerToken, map[string]any{"chapterId": chapter.ID, "start": span[0], "end": span[1]})
var got LookupResult
json.Unmarshal(payload, &got)
return code, got
}
code, got := lookup("curiosity")
if code != 200 || got.Status != "exact" || got.Phonetic != "kjˌʊɹiˈɑːsəti" || got.PhoneticSource != "cmudict" {
t.Fatalf("curiosity %d %+v", code, got)
}
if len(got.Entries) < 2 || got.Entries[0].Provider != "ecdict" || !strings.Contains(got.Entries[0].Translation, "好奇心") || got.Entries[len(got.Entries)-1].Provider != "wordnet" {
t.Fatalf("curiosity merge %+v", got.Entries)
}
if len(got.Resources) != 2 || got.Resource == nil || got.Resource.Name != "Princeton WordNet" {
t.Fatalf("curiosity resources %+v", got.Resources)
}
if got.Entries[0].Definition != "" {
t.Fatal("the Chinese sense must not carry an English definition")
}
// "Dogs" is not a headword: WordNet resolves the inflection and the gloss follows it, without
// copying the headword's transcription onto the inflected form.
code, plural := lookup("Dogs")
if code != 200 || plural.Status != "lemma" || plural.Phonetic != "" {
t.Fatalf("Dogs %d %+v", code, plural)
}
if len(plural.Entries) == 0 || plural.Entries[0].Provider != "ecdict" || !strings.Contains(plural.Entries[0].Translation, "狗") {
t.Fatalf("Dogs gloss %+v", plural.Entries)
}
// Each dictionary is switchable on its own.
if code, _ := callAPI(t, r, "PATCH", fmt.Sprintf("/api/v1/dictionaries/%d", ChineseSlot), adminToken, map[string]bool{"enabled": false}); code != 200 {
t.Fatalf("disable chinese %d", code)
}
if code, english := lookup("curiosity"); code != 200 || english.Phonetic != "" || hasProviderEntry(english.Entries, "ecdict") || len(english.Resources) != 1 {
t.Fatalf("english only %d %+v", code, english)
}
// Bring the Chinese dictionary back before switching the other one off, so each step proves
// that exactly one provider is answering.
if code, _ := callAPI(t, r, "PATCH", fmt.Sprintf("/api/v1/dictionaries/%d", ChineseSlot), adminToken, map[string]bool{"enabled": true}); code != 200 {
t.Fatalf("re-enable chinese %d", code)
}
if code, _ := callAPI(t, r, "PATCH", fmt.Sprintf("/api/v1/dictionaries/%d", WordNetSlot), adminToken, map[string]bool{"enabled": false}); code != 200 {
t.Fatalf("disable wordnet %d", code)
}
if code, only := lookup("curiosity"); code != 200 || !hasProviderEntry(only.Entries, "ecdict") || only.Phonetic != "kjˌʊɹiˈɑːsəti" || only.Resource == nil || only.Resource.Name == "Princeton WordNet" {
t.Fatalf("chinese only %d %+v", code, only)
}
if code, _ := callAPI(t, r, "PATCH", fmt.Sprintf("/api/v1/dictionaries/%d", ChineseSlot), adminToken, map[string]bool{"enabled": false}); code != 200 {
t.Fatalf("disable both %d", code)
}
if code, none := lookup("curiosity"); code != 200 || none.Status != "resource_missing" || len(none.Resources) != 0 {
t.Fatalf("both disabled %d %+v", code, none)
}
// Re-enabling after a cold restart proves the archive is reloaded from MySQL, not from memory.
if code, _ := callAPI(t, r, "PATCH", fmt.Sprintf("/api/v1/dictionaries/%d", ChineseSlot), adminToken, map[string]bool{"enabled": true}); code != 200 {
t.Fatalf("re-enable %d", code)
}
r = Router(db, time.Now)
if code, cold := lookup("curiosity"); code != 200 || cold.Phonetic != "kjˌʊɹiˈɑːsəti" {
t.Fatalf("cold chinese %d %+v", code, cold)
}
if code, listed := callAPI(t, r, "GET", "/api/v1/dictionaries", adminToken, nil); code != 200 || !bytes.Contains(listed, []byte(`"provider":"ecdict"`)) {
t.Fatalf("list %d %s", code, string(listed))
}
}
+62 -4
View File
@@ -114,17 +114,75 @@ func Migrate(db *gorm.DB) error {
return err
}
}
// v12 lets a second English dictionary live beside WordNet: the single-slot CHECK on
// lexgo_dictionaries is dropped and a provider column names each resource. Both steps are
// conditional because MySQL has no DROP CHECK IF EXISTS and an older database lacks the column,
// and the version row still advances only after everything succeeded.
if current < 12 {
if err = addDictionaryProvider(ctx, conn); err != nil {
return err
}
if err = dropDictionarySingleSlotCheck(ctx, conn); err != nil {
return err
}
}
_, err = conn.ExecContext(ctx, fmt.Sprintf("UPDATE lexgo_schema SET version=%d WHERE id=1", SchemaVersion))
return err
}
// SchemaVersion is the version an explicit migration leaves behind, and the
// version the server requires before it starts.
const SchemaVersion = 11
const SchemaVersion = 12
// addAuthorColumn adds an optional author column when it is missing, so replaying the migration
// stays safe just like the CREATE TABLE IF NOT EXISTS statements around it. The table name is a
// constant from this file, never user input.
// addDictionaryProvider adds the provider column when it is missing, so replaying the migration
// stays safe. Existing rows keep the WordNet slot through the default value.
func addDictionaryProvider(ctx context.Context, conn *sql.Conn) error {
var exists int
if err := conn.QueryRowContext(ctx,
"SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'lexgo_dictionaries' AND COLUMN_NAME = 'provider'").Scan(&exists); err != nil {
return err
}
if exists > 0 {
return nil
}
_, err := conn.ExecContext(ctx, "ALTER TABLE lexgo_dictionaries ADD COLUMN provider VARCHAR(32) NOT NULL DEFAULT 'wordnet'")
return err
}
// dropDictionarySingleSlotCheck removes the CHECK (id = 1) constraint of v1..v11 so more than one
// dictionary row can exist. MySQL names such a constraint automatically, so it is looked up and
// skipped when it is already gone.
func dropDictionarySingleSlotCheck(ctx context.Context, conn *sql.Conn) error {
rows, err := conn.QueryContext(ctx,
"SELECT tc.CONSTRAINT_NAME, cc.CHECK_CLAUSE FROM information_schema.TABLE_CONSTRAINTS tc "+
"JOIN information_schema.CHECK_CONSTRAINTS cc ON cc.CONSTRAINT_SCHEMA = tc.CONSTRAINT_SCHEMA AND cc.CONSTRAINT_NAME = tc.CONSTRAINT_NAME "+
"WHERE tc.TABLE_SCHEMA = DATABASE() AND tc.TABLE_NAME = 'lexgo_dictionaries' AND tc.CONSTRAINT_TYPE = 'CHECK'")
if err != nil {
return err
}
names := []string{}
for rows.Next() {
var name, clause string
if err = rows.Scan(&name, &clause); err != nil {
rows.Close()
return err
}
compact := strings.ReplaceAll(strings.ReplaceAll(clause, "`", ""), " ", "")
if strings.Contains(compact, "id=1") {
names = append(names, name)
}
}
rows.Close()
if err = rows.Err(); err != nil {
return err
}
for _, name := range names {
if _, err = conn.ExecContext(ctx, "ALTER TABLE lexgo_dictionaries DROP CHECK `"+name+"`"); err != nil {
return err
}
}
return nil
}
func addAuthorColumn(ctx context.Context, conn *sql.Conn, table string) error {
var exists int
if err := conn.QueryRowContext(ctx,
+161 -43
View File
@@ -4,6 +4,7 @@ import (
"errors"
"io"
"net/http"
"sort"
"strconv"
"strings"
"sync"
@@ -17,10 +18,12 @@ import (
"gorm.io/gorm/clause"
)
// ID 1 is the sole English corpus slot. Keeping the validated ZIP in MySQL
// makes ordinary database backups include the resource needed after a restart.
// Slot 1 holds the reviewed WordNet corpus and slot 2 the prepared 英汉 resource. The provider
// column names what each row is, because a second English dictionary now lives beside the first.
// Keeping both in MySQL means ordinary database backups include every resource needed at restart.
type DictionaryResource struct {
ID int64 `gorm:"primaryKey;autoIncrement:false"`
Provider string
Name string
Language string
Version string
@@ -37,6 +40,7 @@ func (DictionaryResource) TableName() string { return "lexgo_dictionaries" }
type DictionaryView struct {
ID int64 `json:"id"`
Provider string `json:"provider"`
Name string `json:"name"`
Language string `json:"language"`
Version string `json:"version"`
@@ -50,7 +54,7 @@ type DictionaryView struct {
}
func dictionaryView(r DictionaryResource, status string) DictionaryView {
return DictionaryView{r.ID, r.Name, r.Language, r.Version, r.Source, r.Format, status, r.Enabled, r.SHA256, r.EntryCount, r.UpdatedAt}
return DictionaryView{r.ID, r.Provider, r.Name, r.Language, r.Version, r.Source, r.Format, status, r.Enabled, r.SHA256, r.EntryCount, r.UpdatedAt}
}
type DictionaryImportResult struct {
@@ -63,34 +67,54 @@ type ChapterTokens struct {
Phrases []PhraseSpan `json:"phrases"`
}
// Each router keeps at most one immutable parsed corpus; no private chapter or
// Each router keeps at most one immutable parsed corpus per resource slot; no private chapter or
// lookup data enters the cache. The mutex also coalesces simultaneous cold loads.
type dictionaryCache struct {
mu sync.Mutex
sha string
engine *WordNet
type corpus interface {
Lookup(query string) LookupResult
}
func (cache *dictionaryCache) load(tx *gorm.DB, r DictionaryResource) (*WordNet, error) {
type cachedCorpus struct {
sha string
corpus corpus
}
type dictionaryCache struct {
mu sync.Mutex
loaded map[int64]cachedCorpus
}
// parseResource dispatches on the stored format, so WordNet and the 英汉 resource can coexist
// without either parser knowing about the other's layout.
func parseResource(resource DictionaryResource, archive []byte) (corpus, error) {
switch resource.Format {
case ChineseDictionaryFormat:
return ParseChineseDictionary(archive)
case "wordnet-3.0-zip":
return ParseWordNet(archive)
default:
return nil, errors.New("unsupported resource format")
}
}
func (cache *dictionaryCache) load(tx *gorm.DB, r DictionaryResource) (corpus, error) {
cache.mu.Lock()
defer cache.mu.Unlock()
if r.SHA256 != WordNetSHA {
return nil, errors.New("unsupported resource checksum")
}
if cache.sha == r.SHA256 && cache.engine != nil {
return cache.engine, nil
if cached, ok := cache.loaded[r.ID]; ok && cached.sha == r.SHA256 && cached.corpus != nil {
return cached.corpus, nil
}
var stored DictionaryResource
if err := tx.Select("id", "archive").Where("id = ? AND sha256 = ?", r.ID, r.SHA256).First(&stored).Error; err != nil {
return nil, err
}
engine, err := ParseWordNet(stored.Archive)
parsed, err := parseResource(r, stored.Archive)
if err != nil {
return nil, err
}
cache.sha = r.SHA256
cache.engine = engine
return engine, nil
if cache.loaded == nil {
cache.loaded = map[int64]cachedCorpus{}
}
cache.loaded[r.ID] = cachedCorpus{sha: r.SHA256, corpus: parsed}
return parsed, nil
}
func readyOwnedChapter(tx *gorm.DB, owner int, id int64) (Chapter, error) {
@@ -109,7 +133,7 @@ func readyOwnedChapter(tx *gorm.DB, owner int, id int64) (Chapter, error) {
}
func readDictionaryUpload(c *gin.Context) (DictionaryResource, error) {
bad := failure(400, "词典上传无效,请使用指定的 WordNet 3.0 ZIP 和完整资源信息")
bad := failure(400, "词典上传无效,请使用指定的词典 ZIP 和完整资源信息")
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxDictionaryZip+(64<<10))
reader, err := c.Request.MultipartReader()
if err != nil {
@@ -154,11 +178,50 @@ func readDictionaryUpload(c *gin.Context) (DictionaryResource, error) {
}
part.Close()
}
name := strings.TrimSpace(fields["name"])
if name == "" || utf8.RuneCountInString(name) > 120 || strings.IndexFunc(name, unicode.IsControl) >= 0 || fields["language"] != "en" || fields["version"] != "3.0" || fields["format"] != "wordnet-3.0-zip" || fields["source"] != WordNetSource || len(archive) == 0 {
return DictionaryResource{}, bad
return newDictionaryResource(fields, archive)
}
// supportedDictionaries lists every reviewed resource an operator may import. The 英汉 archive is
// not downloaded from the internet at import time: scripts/dict_prepare.py builds it from the
// pinned sources and the operator imports that ZIP.
func supportedDictionaries() []gin.H {
return []gin.H{
{"provider": "wordnet", "name": "Princeton WordNet", "language": "en", "version": WordNetVersion, "format": "wordnet-3.0-zip", "source": WordNetSource, "sha256": WordNetSHA, "downloadable": true, "description": "英语释义"},
{"provider": "ecdict", "name": "简明英汉字典增强版(ECDICT)+ CMUdict 音标", "language": "en", "version": ChineseDictionaryVersion, "format": ChineseDictionaryFormat, "source": ChineseDictionarySource, "sha256": "", "downloadable": false, "description": "中文释义与 IPA 音标,由 scripts/dict_prepare.py 生成"},
}
}
// newDictionaryResource validates the multipart fields against the reviewed specification of the
// declared format, so an unreviewed archive can never occupy a dictionary slot.
func newDictionaryResource(fields map[string]string, archive []byte) (DictionaryResource, error) {
name := strings.TrimSpace(fields["name"])
if name == "" || utf8.RuneCountInString(name) > 120 || strings.IndexFunc(name, unicode.IsControl) >= 0 || fields["language"] != "en" || len(archive) == 0 {
return DictionaryResource{}, failure(400, "词典上传无效,请使用指定的词典 ZIP 和完整资源信息")
}
switch fields["format"] {
case "wordnet-3.0-zip":
if fields["version"] != WordNetVersion || fields["source"] != WordNetSource {
return DictionaryResource{}, failure(400, "词典上传无效,请使用指定的词典 ZIP 和完整资源信息")
}
resource := DictionaryResource{ID: WordNetSlot, Provider: "wordnet", Name: name, Language: "en", Version: WordNetVersion, Source: WordNetSource, Format: "wordnet-3.0-zip", SHA256: WordNetSHA, Enabled: true, Archive: archive}
parsed, err := ParseWordNet(archive)
if err != nil {
return DictionaryResource{}, failure(400, "词典文件校验失败,请选择指定的 WordNet 3.0 ZIP")
}
resource.EntryCount = parsed.EntryCount
return resource, nil
case ChineseDictionaryFormat:
if fields["version"] != ChineseDictionaryVersion || fields["source"] != ChineseDictionarySource {
return DictionaryResource{}, failure(400, "词典上传无效,请使用 dict_prepare.py 生成的英汉词典 ZIP(版本 "+ChineseDictionaryVersion+")")
}
parsed, err := ParseChineseDictionary(archive)
if err != nil {
return DictionaryResource{}, failure(400, "英汉词典文件校验失败,请使用 dict_prepare.py 生成的 ZIP")
}
return DictionaryResource{ID: ChineseSlot, Provider: "ecdict", Name: name, Language: "en", Version: ChineseDictionaryVersion, Source: ChineseDictionarySource, Format: ChineseDictionaryFormat, SHA256: contentSHA(string(archive)), EntryCount: parsed.EntryCount, Enabled: true, Archive: archive}, nil
default:
return DictionaryResource{}, failure(400, "词典上传无效,请使用指定的词典 ZIP 和完整资源信息")
}
return DictionaryResource{ID: 1, Name: name, Language: "en", Version: "3.0", Source: WordNetSource, Format: "wordnet-3.0-zip", SHA256: WordNetSHA, Enabled: true, Archive: archive}, nil
}
func registerDictionaryRoutes(v *gin.RouterGroup, protect func(bool, func(*gin.Context, *gorm.DB, admin.SysUser) (any, error)) gin.HandlerFunc, now func() time.Time) {
@@ -183,7 +246,7 @@ func registerDictionaryRoutes(v *gin.RouterGroup, protect func(bool, func(*gin.C
}
items = append(items, dictionaryView(row, status))
}
return gin.H{"items": items, "supported": gin.H{"name": "Princeton WordNet", "language": "en", "version": "3.0", "format": "wordnet-3.0-zip", "source": WordNetSource, "sha256": WordNetSHA}}, nil
return gin.H{"items": items, "supported": supportedDictionaries()}, nil
}))
v.POST("/dictionaries/import", protect(true, func(c *gin.Context, tx *gorm.DB, u admin.SysUser) (any, error) {
select {
@@ -196,11 +259,6 @@ func registerDictionaryRoutes(v *gin.RouterGroup, protect func(bool, func(*gin.C
if err != nil {
return nil, err
}
engine, err := ParseWordNet(resource.Archive)
if err != nil {
return nil, failure(400, "词典文件校验失败,请选择指定的 WordNet 3.0 ZIP")
}
resource.EntryCount = engine.EntryCount
resource.UpdatedAt = stamp(now())
// INSERT ... ON CONFLICT followed by a locking read serializes even the first
// concurrent import. A validated replacement and its metadata commit together.
@@ -209,12 +267,12 @@ func registerDictionaryRoutes(v *gin.RouterGroup, protect func(bool, func(*gin.C
return nil, created.Error
}
var existing DictionaryResource
if err = tx.Omit("archive").Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = 1").First(&existing).Error; err != nil {
if err = tx.Omit("archive").Clauses(clause.Locking{Strength: "UPDATE"}).Where("id = ?", resource.ID).First(&existing).Error; err != nil {
return nil, err
}
duplicate := created.RowsAffected == 0 && existing.SHA256 == resource.SHA256
if created.RowsAffected == 0 {
if err = tx.Model(&DictionaryResource{}).Where("id = 1").Select("name", "language", "version", "source", "format", "sha256", "entry_count", "enabled", "archive", "updated_at").Updates(&resource).Error; err != nil {
if err = tx.Model(&DictionaryResource{}).Where("id = ?", resource.ID).Select("provider", "name", "language", "version", "source", "format", "sha256", "entry_count", "enabled", "archive", "updated_at").Updates(&resource).Error; err != nil {
return nil, err
}
existing = resource
@@ -227,7 +285,7 @@ func registerDictionaryRoutes(v *gin.RouterGroup, protect func(bool, func(*gin.C
}))
v.PATCH("/dictionaries/:id", protect(true, func(c *gin.Context, tx *gorm.DB, u admin.SysUser) (any, error) {
id, err := strconv.ParseInt(c.Param("id"), 10, 64)
if err != nil || id != 1 {
if err != nil || id <= 0 {
return nil, failure(404, "词典不存在")
}
var input struct {
@@ -310,20 +368,80 @@ func registerDictionaryRoutes(v *gin.RouterGroup, protect func(bool, func(*gin.C
return nil, err
}
missing := LookupResult{Status: "resource_missing", Query: query, Candidates: []string{}, Entries: []DictionaryEntry{}}
var resource DictionaryResource
err = tx.Omit("archive").Where("id = 1 AND enabled = ?", true).First(&resource).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return missing, nil
}
if err != nil {
var resources []DictionaryResource
if err = tx.Omit("archive").Where("language = ? AND enabled = ?", "en", true).Order("id").Find(&resources).Error; err != nil {
return nil, err
}
engine, err := cache.load(tx, resource)
if err != nil {
if len(resources) == 0 {
return missing, nil
}
result := engine.Lookup(query)
result.Resource = &LookupResource{Name: resource.Name, Version: resource.Version}
return result, nil
// The Chinese gloss answers "what does this mean" first; WordNet follows as English detail.
// A disabled or unreadable resource is skipped, so one missing dictionary degrades to the
// other instead of failing the whole lookup.
sort.SliceStable(resources, func(i, j int) bool { return providerRank(resources[i].Provider) < providerRank(resources[j].Provider) })
results := make([]LookupResult, 0, len(resources))
loaded := make([]LookupResource, 0, len(resources))
loadedCorpora := make([]struct {
resource DictionaryResource
corpus corpus
}, 0, len(resources))
var primary *LookupResource
for _, resource := range resources {
engine, loadErr := cache.load(tx, resource)
if loadErr != nil {
continue
}
results = append(results, engine.Lookup(query))
loaded = append(loaded, LookupResource{Name: resource.Name, Version: resource.Version})
loadedCorpora = append(loadedCorpora, struct {
resource DictionaryResource
corpus corpus
}{resource, engine})
if resource.Provider == "wordnet" && primary == nil {
primary = &LookupResource{Name: resource.Name, Version: resource.Version}
}
}
if len(results) == 0 {
return missing, nil
}
merged := mergeLookups(query, results)
merged.Resources = loaded
if primary == nil {
primary = &loaded[0]
}
merged.Resource = primary
// WordNet resolves inflections (dogs -> dog); the Chinese gloss is stored per headword, so
// the resolved form is looked up there too. The gloss is shown, but its phonetic is not
// copied to the inflection because it belongs to the headword.
if merged.MatchedForm != nil && *merged.MatchedForm != merged.Query && !hasProviderEntry(merged.Entries, "ecdict") {
for _, item := range loadedCorpora {
if item.resource.Provider != "ecdict" {
continue
}
lemma := item.corpus.Lookup(*merged.MatchedForm)
if lemma.Status == "exact" && len(lemma.Entries) > 0 {
lemma.Entries[0].Lemma = *merged.MatchedForm
merged.Entries = append(lemma.Entries, merged.Entries...)
}
}
}
return merged, nil
}))
}
func hasProviderEntry(entries []DictionaryEntry, provider string) bool {
for _, entry := range entries {
if entry.Provider == provider {
return true
}
}
return false
}
// providerRank orders providers for display: the Chinese gloss first, then English detail.
func providerRank(provider string) int {
if provider == "ecdict" {
return 0
}
return 1
}
+5 -3
View File
@@ -169,8 +169,10 @@ func TestMigrationFromV4KeepsLibraryAndAddsPersonalTerms(t *testing.T) {
if err := db.Create(&chapter).Error; err != nil {
t.Fatal(err)
}
resource := DictionaryResource{ID: 1, Name: "Fixture WordNet", Language: "en", Version: "3.0", Source: WordNetSource, Format: "wordnet-3.0-zip", SHA256: strings.Repeat("a", 64), EntryCount: 5, Enabled: true, Archive: []byte("fixture archive"), UpdatedAt: now}
if err := db.Create(&resource).Error; err != nil {
// A v4 database has no provider column, so the fixture is written with the columns that
// existed then; the migration adds provider and the default names it as the WordNet slot.
if err := db.Exec("INSERT INTO lexgo_dictionaries (id,name,language,version,source,format,sha256,entry_count,enabled,archive,updated_at) VALUES (1,?,?,?,?,?,?,?,?,?,?)",
"Fixture WordNet", "en", "3.0", WordNetSource, "wordnet-3.0-zip", strings.Repeat("a", 64), 5, true, []byte("fixture archive"), now).Error; err != nil {
t.Fatal(err)
}
if err := CheckSchema(db); err == nil {
@@ -187,7 +189,7 @@ func TestMigrationFromV4KeepsLibraryAndAddsPersonalTerms(t *testing.T) {
t.Fatal("migration changed original chapter", err)
}
var restoredResource DictionaryResource
if err := db.Omit("archive").First(&restoredResource, 1).Error; err != nil || restoredResource.SHA256 != resource.SHA256 || restoredResource.EntryCount != 5 {
if err := db.Omit("archive").First(&restoredResource, 1).Error; err != nil || restoredResource.SHA256 != strings.Repeat("a", 64) || restoredResource.EntryCount != 5 || restoredResource.Provider != "wordnet" {
t.Fatal("migration changed the shared dictionary resource", err)
}
var terms int64
+19
View File
@@ -16,6 +16,13 @@ import (
const WordNetSHA = "cbda5ea6eef7f36a97a43d4a75f85e07fccbb4f23657d27b4ccbc93e2646ab59"
const WordNetSource = "https://raw.githubusercontent.com/nltk/nltk_data/96f9b3252457a2b97e52aec64c3dfceeb5c312d5/packages/corpora/wordnet.zip"
const WordNetVersion = "3.0"
// Dictionary slots are fixed so an operator always knows which row holds which resource.
const (
WordNetSlot = 1
ChineseSlot = 2
)
const maxDictionaryZip = 32 << 20
const maxDictionaryInflated = 128 << 20
@@ -76,6 +83,11 @@ type DictionaryEntry struct {
POS string `json:"pos"`
Definition string `json:"definition"`
Examples []string `json:"examples"`
// The Chinese gloss and phonetic come from the 英汉 resource and are empty for WordNet senses.
Translation string `json:"translation,omitempty"`
Phonetic string `json:"phonetic,omitempty"`
PhoneticSource string `json:"phoneticSource,omitempty"`
Provider string `json:"provider,omitempty"`
}
type LookupResult struct {
Status string `json:"status"`
@@ -84,6 +96,12 @@ type LookupResult struct {
Candidates []string `json:"candidates"`
Entries []DictionaryEntry `json:"entries"`
Resource *LookupResource `json:"resource,omitempty"`
Resources []LookupResource `json:"resources,omitempty"`
// Phonetic repeats the first phonetic found among the entries so a reader can show it next to
// the word without walking the senses; PhoneticSource records whether it is a CMUdict IPA
// transcription or the ECDICT notation kept with character-level normalisation only.
Phonetic string `json:"phonetic,omitempty"`
PhoneticSource string `json:"phoneticSource,omitempty"`
}
type LookupResource struct {
Name string `json:"name"`
@@ -300,6 +318,7 @@ func (wn *WordNet) Lookup(query string) LookupResult {
}
entry := part.data[offset]
entry.Lemma = strings.ReplaceAll(form, "_", " ")
entry.Provider = "wordnet"
result.Entries = append(result.Entries, entry)
}
}
+65
View File
@@ -0,0 +1,65 @@
{
"name": "简明英汉字典增强版常用子集 (ECDICT) + CMUdict 音标转写",
"language": "en",
"provider": "ecdict",
"format": "lexgo-zh-dict-v1",
"version": "ecdict-1.0.28+cmudict-0.7b",
"sources": [
{
"name": "ECDICT StarDict",
"role": "chinese-gloss",
"source": "https://github.com/skywind3000/ECDICT/releases/download/1.0.28/ecdict-stardict-28.zip",
"mirrors": [],
"sha256": "c707d0f3ded6ec79b96466da4a1574e074703da5af9c120fbad97f9cb08c6f2c",
"bytes": 70415834,
"license": "ECDICT-LICENSE.txt",
"licenseUrl": "https://github.com/skywind3000/ECDICT/blob/master/LICENSE",
"note": "仓库为 MIT;词典数据由公开资料整理,随项目再分发前需自行评估来源。子集按下方 subset 规则裁剪,不导入全量 340 万条。"
},
{
"name": "CMUdict",
"role": "ipa-phonetic",
"source": "https://raw.githubusercontent.com/cmusphinx/cmudict/master/cmudict.dict",
"mirrors": [
"https://cdn.jsdelivr.net/gh/cmusphinx/cmudict@master/cmudict.dict"
],
"sha256": "81917843c7f44ce2b094ac63873c2c7a4cf802040792c455ba3ca406891c3d22",
"bytes": 3618488,
"license": "CMUDICT-LICENSE.txt",
"licenseUrl": "https://github.com/cmusphinx/cmudict/blob/master/LICENSE",
"note": "BSD-2。ARPAbet 音素按固定映射表转写为 IPA;仅用于需要音标的词条。"
},
{
"name": "Princeton WordNet",
"role": "subset-wordlist",
"source": "https://raw.githubusercontent.com/nltk/nltk_data/96f9b3252457a2b97e52aec64c3dfceeb5c312d5/packages/corpora/wordnet.zip",
"mirrors": [
"https://cdn.jsdelivr.net/gh/nltk/nltk_data@96f9b3252457a2b97e52aec64c3dfceeb5c312d5/packages/corpora/wordnet.zip"
],
"sha256": "cbda5ea6eef7f36a97a43d4a75f85e07fccbb4f23657d27b4ccbc93e2646ab59",
"license": "WORDNET-LICENSE.txt",
"note": "与 server/wordnet-resource.json 同一 pin,仅用于取 lemma 词表决定子集范围,不复制其释义。"
}
],
"subset": {
"rule": "lowcase single alphabetic headword, unique, present in the WordNet lemma list or carrying an ECDICT exam tag (研/四/六/托/专/雅/高/中/初)",
"expectedEntries": 82721,
"toleranceEntries": 2000,
"measuredAt": "2026-09-16",
"measured": {
"entries": 82721,
"withCmudictIpa": 35799,
"withEcdictPhoneticOnly": 30580,
"withoutPhonetic": 16342,
"droppedEmpty": 45,
"rawPayloadMiB": 8.1
}
},
"phoneticRules": "prefer CMUdict ARPAbet->IPA mapping; otherwise keep the ECDICT notation with character-level normalisation only and mark the source; no phonetic rather than a guess",
"runtime": "Go only; the prepared ZIP is explicitly imported and persisted in MySQL; no runtime download and no Python process",
"preparedArtifact": {
"bytes": 3003208,
"sha256": "a4f4ab972c6e233e39909a861015c99ad7943e8db24353afc9cc9d8160dded24",
"note": "由 scripts/dict_prepare.py 在本机生成,不入库;每次重建都会打印自己的 sha256"
}
}