98五笔完美词库

# open wangma data
with open('wmwb98qm.dat', 'rb') as infile:
    wmData = infile.read()

# the header is 256 bytes
# discard the header
wmData = wmData[256:]

# each record is 304 bytes
recordSize = 304

# make sure the file is valid
assert len(wmData) % recordSize == 0
numRecord = len(wmData)//recordSize

print('dictionary size: ', len(wmData))
print('number of record: ', numRecord)

def GetRecord(id):
    return wmData[id * recordSize : (id + 1) * recordSize]

def ParseKeyByte(keyBytes):
    key = keyBytes.decode('ascii').strip(' ')
    return key

def ParseRecord_SingleCharacter(record):
    key = ParseKeyByte(record[4:8])
    try:
        character = record[8:10].decode('gbk')
    except Exception as e:
        return None
    return key, character

def ParseRecord_Word(record):
    key = ParseKeyByte(record[4:8])

    # sequentially scan for the word
    # which starts at the 48th byte of the record
    indStart = 48
    indNow = indStart

    while indNow < recordSize and record[indNow] != 0:
        indNow += 1
    assert indNow < recordSize

    # extract the word
    word = record[indStart : indNow].decode('gbk')
    return key, word

def ParseRecord_ExtendedChrSet(record):
    key = ParseKeyByte(record[4:8])
    character = record[8:12].decode('gb18030')
    return key, character

def ParseRecord_None(record):
    return None

recordParserDict = {
    b'\x01\x00\x00\x00' : ParseRecord_SingleCharacter, # single character
    b'\x08\x00\x00\x00' : ParseRecord_None, # date
    b'\x10\x00\x00\x00' : ParseRecord_None, # special symbols
    b'\x04\x00\x00\x00' : ParseRecord_Word, # words/phrases
    b'\x02\x00\x00\x00' : ParseRecord_ExtendedChrSet, # characters not in GBK standard (encoded by GB18030)
    b'\x00\x00\x00\x00' : ParseRecord_None  # null record
}

result = []


for i in range(numRecord):
    record = GetRecord(i)
    recordType = record[0:4]
    recordFunc = recordParserDict[recordType]

    res = recordFunc(record)
    if res is not None:
        result.append(res)


import codecs
import re
with codecs.open('onetwo.txt', 'r', 'utf8') as infile:
    onetwoData = infile.readlines()

onetwoResult = []

letterSeq = re.sub(' ', '', onetwoData[0].lower().strip())
assert len(letterSeq) == 25

# append one-key characters
oneKeyLine = onetwoData[2]
for i in range(len(letterSeq)):
    onetwoResult.append((letterSeq[i], oneKeyLine[i]))

# append two-key characters
twoKeyGrid = [''] * len(letterSeq)
for i in range(len(letterSeq)):
    twoKeyGrid[i] = onetwoData[i + 4].strip('\n').strip('\r')
    assert len(twoKeyGrid[i]) == len(letterSeq)

for i in range(len(letterSeq)):
    for j in range(len(letterSeq)):
        key = letterSeq[i] + letterSeq[j]
        character = twoKeyGrid[i][j]
        if character != ' ':
            onetwoResult.append((key, character))

result = onetwoResult + result

# reorganize the result for baidu wubi
# when the same key combination appears multiple times in the database,
# the actual order is treated in reverse. so we have to manually reverse
# the occurance of those combinations
allKeys = [item[0] for item in result]
occurrence = dict()
for ind, item in enumerate(result):
    if item[0] in occurrence:
        occurrence[item[0]].append(ind)
    else:
        occurrence[item[0]] = [ind]

# filter out those that only appered once
toDelete = []
for key, val in occurrence.items():
    if len(val) <= 1:
        toDelete.append(key)
for key in toDelete:
    del occurrence[key]

# swap occurrence now
for key, val in occurrence.items():
    left = 0
    right = len(val) - 1

    while right > left and left != right:
        temp = result[val[left]]
        result[val[left]] = result[val[right]]
        result[val[right]] = temp
        left += 1
        right -= 1

# write results
with codecs.open('wb98perfection.txt', 'w', 'utf16') as outfile:
    for item in result:
        outfile.write('\t'.join(item) + '\r\n')