使用Python对csv文件去重
import pandas as pd
import csv
with open('uid_15.csv','r',encoding='utf-8') as f1:
reader = csv.reader(f1)
tweets_id = [row[0] for row in reader]
f1.close()
for tweet_id in tweets_id:
print(f'正在处理%文件...' % tweet_id)
with open('comments/' + tweet_id + '.csv','r',encoding='utf-8') as f1:
reader = csv.reader(f1)
comments = [row for row in reader]
finished = []
for comment in comments:
if comment not in finished:
finished.append(comment)
with open('comments_new/' + tweet_id + '.csv', 'w',encoding='utf-8') as f2:
write = csv.writer(f2)
write.writerows(finished)