#!/usr/bin/env python3 import pymongo
import datetime
import os yesterday = str(datetime.date.today() - datetime.timedelta(days=1)) def mongo2csv(db_name, ts_string=yesterday, column_handle=None, column_delete=None):
''' :param db_name: 表名
:param ts_string: 日期
:param column_handle: 需要处理的列,
格式{k:v} ,
k:需要处理的列,
v:1.数字,即列表索引,取列表的第几个元素
2.字符串,即字典key,取子字典的某个value
3.join ,即需要将列表转为字符串 :param column_delete: 需要删除的列
:return:
'''
print('starting...')
host = '127.0.0.1'
port = 5600
user = 'root'
password = ''
url = 'mongodb://' + user + ':' + password + '@' + host + ':' + str(port) + '/'
client = pymongo.MongoClient(url)
db = client.rental
collection = db[db_name]
select_condition = {'_id': 0, 'ts': 0}
if not column_delete:
column_delete = []
if not column_handle:
column_handle = {}
if len(column_delete) > 0:
for temp in column_delete:
select_condition[temp] = 0
title_dic = collection.find_one({'ts_string': ts_string}, select_condition)
title = sorted(title_dic.keys())
con = collection.find({'ts_string': ts_string}, select_condition)
file_name = db_name + '_' + yesterday + '.csv'
with open(file_name, 'a', encoding='utf8') as f:
f.write(','.join(title) + '\n')
for item in con:
for k, v in column_handle.items():
item[k] = item[k][v] if v != 'join' else (
'|'.join([str(x) for x in item[k]]) if isinstance(item[k], list) else item[k])
f.write(','.join([str(item[x]) for x in title]) + '\n') if __name__ == '__main__':
mongo2csv('lianjia_detail')
mongo2csv('mogu_detail', column_handle={'metroInfo': 0, 'rentType': 'value'})
mongo2csv('qingke_detail', column_handle={})
mongo2csv('xiangyu_detail', column_delete=['endDate','vacantStartDate','tabList','vacantEndDate'])
mongo2csv('ziru_detail', column_handle={'subway_line_code': 'join', 'subway_station_code': 'join'})
print(os.system('wc -l *.csv'))