firstname.py
"""
百家姓列表
"""
first_name = [\'赵\' ,\'钱\', \'孙\',\'李\',\'周\',\'吴\',\'郑\',\'王\',\'冯\',\'陈\',\'褚\',\'卫\',\'蒋\',
\'沈\',\'韩\',\'杨\',\'朱\',\'秦\',\'尤\',\'许\',\'何\',\'吕\',\'施\',\'张\',\'孔\',\'曹\'
,\'严\',\'华\',\'金\',\'魏\',\'陶\',\'姜\',\'戚\',\'谢\',\'邹\',\'喻\',\'柏\',\'水\',\'窦\',\'章\',\'云\',\'苏\',
\'潘\',\'葛\',\'奚\',\'范\',\'彭\',\'郎\',\'鲁\',\'韦\',\'昌\',\'马\',\'苗\',\'凤\',\'花\',\'方\',\'俞\',\'任\',\'袁\',\'柳\',
\'酆\',\'鲍\',\'史\',\'唐\',\'费\',\'廉\',\'岑\',\'薛\',\'雷\',\'贺\',\'倪\',\'汤\',\'滕\',\'殷\',\'罗\',\'毕\',\'郝\',\'邬\',
\'安\',\'常\',\'乐\',\'于\',\'时\',\'傅\',\'皮\',\'卞\',\'齐\',\'康\',\'伍\',\'余\',\'元\',\'卜\',\'顾\',\'孟\',\'平\',\'黄\',
\'和\',\'穆\',\'萧\',\'尹\',\'姚\',\'邵\',\'湛\',\'汪\',\'祁\',\'毛\',\'禹\',\'狄\',\'米\',\'贝\',\'明\',\'臧\',\'计\',\'伏\',
\'成\',\'戴\',\'谈\',\'宋\',\'茅\',\'庞\',\'熊\',\'纪\',\'舒\',\'屈\',\'项\',\'祝\',\'董\',\'粱\',\'杜\',\'阮\',\'蓝\',\'闵\',
\'席\',\'季\',\'麻\',\'强\',\'贾\',\'路\',\'娄\',\'危\',\'江\',\'童\',\'颜\',\'郭\',\'梅\',\'盛\',\'林\',\'刁\',\'钟\',\'徐\',
\'邱\',\'骆\',\'高\',\'夏\',\'蔡\',\'田\',\'樊\',\'胡\',\'凌\',\'霍\',\'虞\',\'万\',\'支\',\'柯\',\'昝\',\'管\',\'卢\',\'莫\',
\'经\',\'房\',\'裘\',\'缪\',\'干\',\'解\',\'应\',\'宗\',\'丁\',\'宣\',\'贲\',\'邓\',\'郁\',\'单\',\'杭\',\'洪\',\'包\',\'诸\',
\'左\',\'石\',\'崔\',\'吉\',\'钮\',\'龚\',\'程\',\'嵇\',\'邢\',\'滑\',\'裴\',\'陆\',\'荣\',\'翁\',\'荀\',\'羊\',\'於\',\'惠\',
\'甄\',\'麴\',\'家\',\'封\',\'芮\',\'羿\',\'储\',\'靳\',\'汲\',\'邴\',\'糜\',\'松\',\'井\',\'段\',\'富\',\'巫\',\'乌\',\'焦\',
\'巴\',\'弓\',\'牧\',\'隗\',\'山\',\'谷\',\'车\',\'侯\',\'宓\',\'蓬\',\'全\',\'郗\',\'班\',\'仰\',\'秋\',\'仲\',\'伊\',\'宫\',
\'宁\',\'仇\',\'栾\',\'暴\',\'甘\',\'钭\',\'厉\',\'戎\',\'祖\',\'武\',\'符\',\'刘\',\'景\',\'詹\',\'束\',\'龙\',\'叶\',\'幸\',
\'司\',\'韶\',\'郜\',\'黎\',\'蓟\',\'薄\',\'印\',\'宿\',\'白\',\'怀\',\'蒲\',\'邰\',\'从\',\'鄂\',\'索\',\'咸\',\'籍\',\'赖\',
\'卓\',\'蔺\',\'屠\',\'蒙\',\'池\',\'乔\',\'阴\',\'欎\',\'胥\',\'能\',\'苍\',\'双\',\'闻\',\'莘\',\'党\',\'翟\',\'谭\',\'贡\',
\'劳\',\'逄\',\'姬\',\'申\',\'扶\',\'堵\',\'冉\',\'宰\',\'郦\',\'雍\',\'舄\',\'璩\',\'桑\',\'桂\',\'濮\',\'牛\',\'寿\',\'通\',
\'边\',\'扈\',\'燕\',\'冀\',\'郏\',\'浦\',\'尚\',\'农\',\'温\',\'别\',\'庄\',\'晏\',\'柴\',\'瞿\',\'阎\',\'充\',\'慕\',\'连\',
\'茹\',\'习\',\'宦\',\'艾\',\'鱼\',\'容\',\'向\',\'古\',\'易\',\'慎\',\'戈\',\'廖\',\'庾\',\'终\',\'暨\',\'居\',\'衡\',\'步\',
\'都\',\'耿\',\'满\',\'弘\',\'匡\',\'国\',\'文\',\'寇\',\'广\',\'禄\',\'阙\',\'东\',\'殴\',\'殳\',\'沃\',\'利\',\'蔚\',\'越\',
\'夔\',\'隆\',\'师\',\'巩\',\'厍\',\'聂\',\'晁\',\'勾\',\'敖\',\'融\',\'冷\',\'訾\',\'辛\',\'阚\',\'那\',\'简\',\'饶\',\'空\',
\'曾\',\'毋\',\'沙\',\'乜\',\'养\',\'鞠\',\'须\',\'丰\',\'巢\',\'关\',\'蒯\',\'相\',\'查\',\'後\',\'荆\',\'红\',\'游\',\'竺\',
\'权\',\'逯\',\'盖\',\'益\',\'桓\',\'公\',\'万俟\',\'司马\',\'上官\',\'欧阳\',\'夏侯\',\'诸葛\',\'闻人\',\'东方\',\'赫连\',
\'皇甫\',\'尉迟\',\'公羊\',\'澹台\',\'公冶\',\'宗政\',\'濮阳\',\'淳于\',\'单于\',\'太叔\',\'申屠\',\'公孙\',\'仲孙\',
\'轩辕\',\'令狐\',\'钟离\',\'宇文\',\'长孙\',\'慕容\',\'鲜于\',\'闾丘\',\'司徒\',\'司空\',\'亓官\',\'司寇\',\'仉\',\'督\',
\'子车\',\'颛孙\',\'端木\',\'巫马\',\'公西\',\'漆雕\',\'乐正\',\'壤驷\',\'公良\',\'拓跋\',\'夹谷\',\'宰父\',\'谷梁\',\'晋\',
\'楚\',\'闫\',\'法\',\'汝\',\'鄢\',\'涂\',\'钦\',\'段干\',\'百里\',\'东郭\',\'南门\',\'呼延\',\'归\',\'海\',\'羊舌\',\'微生\',
\'岳\',\'帅\',\'缑\',\'亢\',\'况\',\'后\',\'有\',\'琴\',\'梁丘\',\'左丘\',\'东门\',\'西门\',\'商\',\'牟\',\'佘\',\'佴\',\'伯\',
\'赏\',\'南宫\',\'墨\',\'哈\',\'谯\',\'笪\',\'年\',\'爱\',\'阳\',\'佟\',\'言\',\'福\']
langconv.py
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from copy import deepcopy
import re
try:
import psyco
psyco.full()
except:
pass
try:
from zh_wiki import zh2Hant, zh2Hans
except ImportError:
from zhtools.zh_wiki import zh2Hant, zh2Hans
import sys
py3k = sys.version_info >= (3, 0, 0)
if py3k:
UEMPTY = \'\'
else:
_zh2Hant, _zh2Hans = {}, {}
for old, new in ((zh2Hant, _zh2Hant), (zh2Hans, _zh2Hans)):
for k, v in old.items():
new[k.decode(\'utf8\')] = v.decode(\'utf8\')
zh2Hant = _zh2Hant
zh2Hans = _zh2Hans
UEMPTY = \'\'.decode(\'utf8\')
# states
(START, END, FAIL, WAIT_TAIL) = list(range(4))
# conditions
(TAIL, ERROR, MATCHED_SWITCH, UNMATCHED_SWITCH, CONNECTOR) = list(range(5))
MAPS = {}
class Node(object):
def __init__(self, from_word, to_word=None, is_tail=True,
have_child=False):
self.from_word = from_word
if to_word is None:
self.to_word = from_word
self.data = (is_tail, have_child, from_word)
self.is_original = True
else:
self.to_word = to_word or from_word
self.data = (is_tail, have_child, to_word)
self.is_original = False
self.is_tail = is_tail
self.have_child = have_child
def is_original_long_word(self):
return self.is_original and len(self.from_word)>1
def is_follow(self, chars):
return chars != self.from_word[:-1]
def __str__(self):
return \'<Node, %s, %s, %s, %s>\' % (repr(self.from_word),
repr(self.to_word), self.is_tail, self.have_child)
__repr__ = __str__
class ConvertMap(object):
def __init__(self, name, mapping=None):
self.name = name
self._map = {}
if mapping:
self.set_convert_map(mapping)
def set_convert_map(self, mapping):
convert_map = {}
have_child = {}
max_key_length = 0
for key in sorted(mapping.keys()):
if len(key)>1:
for i in range(1, len(key)):
parent_key = key[:i]
have_child[parent_key] = True
have_child[key] = False
max_key_length = max(max_key_length, len(key))
for key in sorted(have_child.keys()):
convert_map[key] = (key in mapping, have_child[key],
mapping.get(key, UEMPTY))
self._map = convert_map
self.max_key_length = max_key_length
def __getitem__(self, k):
try:
is_tail, have_child, to_word = self._map[k]
return Node(k, to_word, is_tail, have_child)
except:
return Node(k)
def __contains__(self, k):
return k in self._map
def __len__(self):
return len(self._map)
class StatesMachineException(Exception): pass
class StatesMachine(object):
def __init__(self):
self.state = START
self.final = UEMPTY
self.len = 0
self.pool = UEMPTY
def clone(self, pool):
new = deepcopy(self)
new.state = WAIT_TAIL
new.pool = pool
return new
def feed(self, char, map):
node = map[self.pool+char]
if node.have_child:
if node.is_tail:
if node.is_original:
cond = UNMATCHED_SWITCH
else:
cond = MATCHED_SWITCH
else:
cond = CONNECTOR
else:
if node.is_tail:
cond = TAIL
else:
cond = ERROR
new = None
if cond == ERROR:
self.state = FAIL
elif cond == TAIL:
if self.state == WAIT_TAIL and node.is_original_long_word():
self.state = FAIL
else:
self.final += node.to_word
self.len += 1
self.pool = UEMPTY
self.state = END
elif self.state == START or self.state == WAIT_TAIL:
if cond == MATCHED_SWITCH:
new = self.clone(node.from_word)
self.final += node.to_word
self.len += 1
self.state = END
self.pool = UEMPTY
elif cond == UNMATCHED_SWITCH or cond == CONNECTOR:
if self.state == START:
new = self.clone(node.from_word)
self.final += node.to_word
self.len += 1
self.state = END
else:
if node.is_follow(self.pool):
self.state = FAIL
else:
self.pool = node.from_word
elif self.state == END:
# END is a new START
self.state = START
new = self.feed(char, map)
elif self.state == FAIL:
raise StatesMachineException(\'Translate States Machine \'
\'have error with input data %s\' % node)
return new
def __len__(self):
return self.len + 1
def __str__(self):
return \'<StatesMachine %s, pool: "%s", state: %s, final: %s>\' % (
id(self), self.pool, self.state, self.final)
__repr__ = __str__
class Converter(object):
def __init__(self, to_encoding):
self.to_encoding = to_encoding
self.map = MAPS[to_encoding]
self.start()
def feed(self, char):
branches = []
for fsm in self.machines:
new = fsm.feed(char, self.map)
if new:
branches.append(new)
if branches:
self.machines.extend(branches)
self.machines = [fsm for fsm in self.machines if fsm.state != FAIL]
all_ok = True
for fsm in self.machines:
if fsm.state != END:
all_ok = False
if all_ok:
self._clean()
return self.get_result()
def _clean(self):
if len(self.machines):
self.machines.sort(key=lambda x: len(x))
# self.machines.sort(cmp=lambda x,y: cmp(len(x), len(y)))
self.final += self.machines[0].final
self.machines = [StatesMachine()]
def start(self):
self.machines = [StatesMachine()]
self.final = UEMPTY
def end(self):
self.machines = [fsm for fsm in self.machines
if fsm.state == FAIL or fsm.state == END]
self._clean()
def convert(self, string):
self.start()
for char in string:
self.feed(char)
self.end()
return self.get_result()
def get_result(self):
return self.final
def registery(name, mapping):
global MAPS
MAPS[name] = ConvertMap(name, mapping)
registery(\'zh-hant\', zh2Hant)
registery(\'zh-hans\', zh2Hans)
del zh2Hant, zh2Hans
def run():
import sys
from optparse import OptionParser
parser = OptionParser()
parser.add_option(\'-e\', type=\'string\', dest=\'encoding\',
help=\'encoding\')
parser.add_option(\'-f\', type=\'string\', dest=\'file_in\',
help=\'input file (- for stdin)\')
parser.add_option(\'-t\', type=\'string\', dest=\'file_out\',
help=\'output file\')
(options, args) = parser.parse_args()
if not options.encoding:
parser.error(\'encoding must be set\')
if options.file_in:
if options.file_in == \'-\':
file_in = sys.stdin
else:
file_in = open(options.file_in)
else:
file_in = sys.stdin
if options.file_out:
if options.file_out == \'-\':
file_out = sys.stdout
else:
file_out = open(options.file_out, \'wb\')
else:
file_out = sys.stdout
c = Converter(options.encoding)
for line in file_in:
# print >> file_out, c.convert(line.rstrip(\'\n\').decode(
file_out.write(c.convert(line.rstrip(\'\n\').decode(
\'utf8\')).encode(\'utf8\'))
if __name__ == \'__main__\':
run()
