import pandas as pd
from numpy import *
import matplotlib.pylab as plt
import copy
def read(filename):
dat=pd.read_csv(filename,iterator=True)
loop = True
chunkSize = 1000000
R=[]
while loop:
try:
data = dat.get_chunk(chunkSize)
data=data.loc[:,'B':'C'] # 切片
data=data[data.B==855] #条件选择
data['C']=pd.to_datetime(data['C']) # 转换成时间格式
data=data.set_index(['C']) # 设置索引
data.loc[:,'D']=array([1]*len(data)) #增加一列
data=data.resample('D').sum() #按天求和
data=data.loc[:,'D'] #截取
data.fillna(0) #填充缺失值
R.append(data)
except StopIteration:
loop = False
print ("Iteration is stopped.")
R.to_csv('855_pay.csv') # 保存
def read2(filename):
reader=pd.read_csv(filename,iterator=True)
loop = True
chunkSize = 100000
chunks = []
while loop:
try:
chunk = reader.get_chunk(chunkSize)
chunks.append(chunk)
except StopIteration:
loop = False
print ("Iteration is stopped.")
df = pd.concat(chunks, ignore_index=True)
return df
def read3save(filename):
dat=pd.read_csv(filename)
#data = dat.get_chunk(chunkSize)
data=dat.loc[:,'B':'C'] # 切片
data=data[data.B==855]#条件选择
print(shape(data))
data['C']=pd.to_datetime(data['C']) # 转换成时间格式
data=data.set_index(['C'])# 设置索引
if len(data)==0:
return
data.loc[:,'D']=array([1]*len(data)) #增加一列
data=data.resample('D').sum() #按天求和
data=data.loc[:,'D'] #截取
data.fillna(0) #填充缺失值
data.to_csv('855_pay.csv',mode='a') # 保存
def loadDataSet(fileName, delim='\t'):
fr = open(fileName)
stringArr = [line.strip().split(delim) for line in fr.readlines()]
datArr = [list(map(float,line)) for line in stringArr]
return mat(datArr)
def getShopData():
fr = open('shopInfo.txt')
shopID = [line.strip().split('\n') for line in fr.readlines()]
# datArr = [list(map(float,line))for line in stringArr]
for i in range(1,9):
name="user_pay.001.00%d"%i
dat=pd.read_csv(name)
#data = dat.get_chunk(chunkSize)
data=dat.loc[:,'B':'C'] # 切片
for factor in shopID:
data=data[data.B==int(str(factor[0]))]#条件选择
print(shape(data))
if len(data)==0: continue
data['C']=pd.to_datetime(data['C']) # 转换成时间格式
data=data.set_index(['C'])# 设置索引
data.loc[:,'D']=array([1]*len(data)) #增加一列
data=data.resample('D').sum() #按天求和
data=data.loc[:,'D'] #截取
data.fillna(0) #填充缺失值
s=str(factor[0])
savename='D:\python\data\%s_pay.csv'%s
data.to_csv(savename,mode='a') # 保存
del dat
print("over")
def tset(filename):
dat=pd.read_csv(filename)
#data = dat.get_chunk(chunkSize)
data=dat.loc[:,'B':'C'] # 切片
data=data[data.B==855]#条件选择
print(shape(data))
data['C']=pd.to_datetime(data['C']) # 转换成时间格式
data=data.set_index(['C'])# 设置索引
if len(data)==0:
return
data.loc[:,'D']=array([1]*len(data)) #增加一列
data=data.resample('D').sum() #按天求和
data=data.loc[:,'D'] #截取
data.fillna(0) #填充缺失值
#data.to_csv('855_pay.csv',mode='a') # 保存
s='my'
savename='D:\python\data\%s_pay.csv'%s
data.to_csv(savename,mode='a') # 保存
def getShopData2(filename):
import csv
# fr = open('shopInfo.txt')
# shopID = [line.strip().split('\n') for line in fr.readlines()]
# datArr = [list(map(float,line))for line in stringArr]
#for i in range(1,9):
#name="user_pay.001.00%d"%i
dat=pd.read_csv(filename)
#data = dat.get_chunk(chunkSize)
data=dat.loc[:,'B':'C'] # 切片
data['C']=pd.to_datetime(data['C']) # 转换成时间格式
data=data.set_index(['C'])# 设置索引
data.loc[:,'D']=array([1]*len(data)) #增加一列
for i in range(1,2001):
d=copy.copy(data)
d=d[data.B==i]#条件选择
#print(shape(d))
print(i)
if len(d)==0: continue
d=d.resample('D').sum() #按天求和
d=d.loc[:,'D'] #截取
d.fillna(0) #填充缺失值
s=str(i)
#print(s)
savename='D:\python\data2\%s_pay.csv'%s
c=open(savename,'a')
writer=csv.writer(c)
writer.writerow(['C','D'])
c.close()
d.to_csv(savename,mode='a') # 保存
# del dat
print("over")
def formatData():
#fr = open('shopInfo.txt')
#shopID = [line.strip().split('\n') for line in fr.readlines()]
# datArr = [list(map(float,line))for line in stringArr]
#data = dat.get_chunk(chunkSize)
for i in range(1,2001):
s=str(i)
print(s)
name='D:\python\data2\%s_pay.csv'%s
dat=pd.read_csv(name)
data['C']=pd.to_datetime(data['C']) # 转换成时间格式
data=data.set_index(['C'])# 设置索引
data=data.resample('D').sum() #按天求和
data.fillna(0) #填充缺失值
savename='D:\python\data3\%s_pay.csv'%s
data.to_csv(savename,mode='w') # 保存
del dat
print("over")
