1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
| var bdictSm = []string{
"c", "d", "b", "f", "g", "h", "ch", "j", "k", "l", "m", "n",
"", "p", "q", "r", "s", "t", "sh", "zh", "w", "x", "y", "z",
}
var bdictYm = []string{
"uang", "iang", "iong", "ang", "eng", "ian", "iao", "ing", "ong",
"uai", "uan", "ai", "an", "ao", "ei", "en", "er", "ua", "ie", "in", "iu",
"ou", "ia", "ue", "ui", "un", "uo", "a", "e", "i", "o", "u", "v",
}
func (BaiduBdict) Parse(filename string) Dict {
data, _ := os.ReadFile(filename)
r := bytes.NewReader(data)
ret := make(Dict, 0, r.Len()>>8)
var tmp []byte
r.Seek(0x350, 0)
for r.Len() > 4 {
// 拼音长
pyLen := ReadUint16(r)
// 词频
freq := ReadUint16(r)
// 判断下两个字节
tmp = make([]byte, 2)
r.Read(tmp)
// 编码和词不等长，全按 utf-16le
if tmp[0] == 0 && tmp[1] == 0 {
wordLen := ReadUint16(r)
// 读编码
tmp = make([]byte, pyLen*2)
r.Read(tmp)
code, _ := util.Decode(tmp, "UTF-16LE")
// 读词
tmp = make([]byte, wordLen*2)
r.Read(tmp)
word, _ := util.Decode(tmp, "UTF-16LE")
ret = append(ret, Entry{
Word: word,
Pinyin: []string{code},
Freq: freq,
})
continue
}
// 全英文的词，编码和词是一样的
if int(tmp[0]) >= len(bdictSm) && tmp[0] != 0xff {
r.Seek(-2, 1)
eng := make([]byte, pyLen)
r.Read(eng)
ret = append(ret, Entry{
Word: string(eng),
Pinyin: []string{string(eng)},
Freq: freq,
})
continue
}
// 一般格式
r.Seek(-2, 1)
pinyin := make([]string, 0, pyLen)
for i := 0; i < pyLen; i++ {
smIdx, _ := r.ReadByte()
ymIdx, _ := r.ReadByte()
// 带英文的词组
if smIdx == 0xff {
pinyin = append(pinyin, string(ymIdx))
continue
}
pinyin = append(pinyin, bdictSm[smIdx]+bdictYm[ymIdx])
}
// 读词
tmp = make([]byte, pyLen*2)
r.Read(tmp)
word, _ := util.Decode(tmp, "UTF-16LE")
ret = append(ret, Entry{
Word: word,
Pinyin: pinyin,
Freq: freq,
})
}
return ret
}
|
