#!usr/bin/env python #-*- coding:utf-8-*- import jieba import jieba.analyse import codecs file_path='./data/000000_0' result_file_path="./data/000000_0_result_textrank" f=open(file_path,'r') f_result=codecs.open(result_file_path,'w+','utf-8') f_result.write('[\n') texts=f.readlines() num_text=0 for text in texts: text_id=texts.split('\t')[0] text_content=texts.split('\t')[3] f_result.write('{\"text_id\": %s\n'% text_id) keywords=jieba.analyse.textrank(text_content,topK=20,withWeight=True,allowPOS=('ns', 'n', 'vn', 'v'),withFlag=True) f_result.write('\"keyword\":{') num_text+=1 for keyword in keywords: (word,flag)=keyword[0] weight=keyword[1] f_result.write('\"word\":%s,\"weight\":%s,\"nature\": %s\n'% (word,weight,flag)) f_result.write('}\n') if num_text%100=0: print "已经处理%s篇文章" % num_text f_result.write(']\n') print "总共处理%s篇文章" num_text f_result.close() f.close()
2.基于IF-IDF的提取文章关键词extract_keywod_ifidf.py:
#!usr/bin/env python #-*- coding:utf-8-*- import jieba import jieba.analyse import codecs file_path='./data/000000_0' result_file_path="./data/000000_0_result_ifidf" f=open(file_path,'r') f_result=codecs.open(result_file_path,'w+','utf-8') f_result.write('[') texts=f.readlines() num_text=0 for text in texts: text_id=texts.split('\t')[0] text_content=texts.split('\t')[3] f_result.write('{\"text_id\": %s\n'% text_id) keywords=jieba.analyse.extract_tags(text_content,topK=20,withWeight=True,allowPOS=('ns', 'n', 'vn', 'v'),withFlag=True) f_result.write('\"keyword\":{') num_text+=1 for keyword in keywords: (word,flag)=keyword[0] weight=keyword[1] f_result.write('\"word\":%s,\"weight\":%s,\"nature\": %s\n'% (word,weight,flag)) f_result.write('}') if num_text%100=0: print "已经处理%s篇文章" % num_text f_result.write(']') print "总共处理%s篇文章" %num_text f_result.close() f.close()