Python提取数据
python提取数据
目前实现的代码:
import re
def read_write_file(in_file_path, out_file_path, reg_expression, line_per_loop):
in_file = open(in_file_path)
out_file = open(out_file_path, "a")
#pattern = (r'.')
pattern = (reg_expression)
while 1:
lines = in_file.readlines(line_per_loop)
if not lines:
break
for line in lines:
#process the line
match = (line)
if match:
print line
out_file.write(line)
in_file.close()
out_file.close()
if __name__ == "__main__":
read_write_file("", "", r'INCLUDE', 100000)
访问目录的部分
import os, sys
# depth: the current dir:depth = 0
def dir_walk(dir, out_file, depth):
out_file.write(dir + '\n')
current_depth = 0
file_num = 0
list = (dir)
for line in list:
file_path = (dir, line)
if (file_path): # is a directory
#(line + '\\' + '\n')
dir_walk(file_path, out_file, depth - 1)
elif : # is a file
#process the line
read_write_file(file_path, out_file, r'warn\.mse\.360\.cn', 100000)
if __name__ == "__main__":
#dir_path = ()
file = open('', 'w')
dir_walk('E:\\git', file, 2)
数据部分在优盘里
最后完善的代码:
import re
import os, sys
# in_file_path: the file will be read that maybe in different path
# out_file: the content that was been extracted from the source file
# reg_expression: the regulation rules that want to extract
# liner_per_loop: the lines that will be loaded per time
def read_write_file(in_file_path, out_file, reg_expression, line_per_loop):
in_file = open(in_file_path)
#pattern = (r'.')
pattern = (reg_expression)
while 1:
lines = in_file.readlines(line_per_loop)
if not lines:
break
for line in lines:
#process the line
match = (line)
if match:
print line
out_file.write(line)
in_file.close()
# dir: the directory that will be search
# out_file: the file that has been opened
# depth: the current dir:depth = 0
def dir_walk(dir, out_file, current_depth, max_depth):
print current_depth
if current_depth > max_depth:
return
list = (dir)
for line in list:
print line
file_path = (dir, line)
if (file_path): # is a directory
dir_walk(file_path, out_file, current_depth + 1, max_depth)
elif : # is a file
if current_depth != 0:
#process the line
read_write_file(file_path, out_file, r'INCLUDE', 100000)
if __name__ == "__main__":
current_dir = ()
out_file = open("extract_log.txt", "a")
dir_walk(current_dir, out_file, 0, 3)
out_file.close()
在第0层中,代码只会对文件夹进行扫描,其他层次扫描所有的文件和目录。