MongoDB封装，实现数据存储
'''
存储文件:MongoClass.py
'''
import pymongo
import traceback
class MongoOpt:
__client = ''
__db = ''
__col = ''
def __init__(self):
try:
self.__client = pymongo.MongoClient("mongodb://127.0.0.1:27017/")
self.__db = self.__client['geng']
self.__col = self.__db['gerenqingkuang']
except:
traceback.print_exc()
def add_collection(self, info):
try:
self.__col.insert_one(info)
except:
traceback.print_exc()
def get_all_collections(self):
try:
value = []
text = ''
for r in self.__col.find({},{"_id": 0, "相片": 0}):
value.extend(r.values())
for v in value:
text += str(v)
with open("./conf/233.txt", 'w', encoding='utf-8') as txt:
txt.write(text)
return text
except:
traceback.print_exc()
if __name__ == '__main__':
mo = MongoOpt()
# mo.add_collection({'name':"geng", 'sex':'m'})
res = mo.get_all_collections()
print( res)
爬虫封装，实现数据爬取
'''
文件目录:SpiderClass.py
'''
import requests
from bs4 import BeautifulSoup as bs
import re
class SpiderOpt:
def __init__(self, method, url, **kwargs):
self.__response = requests.request(method, url, **kwargs)
def get_response(self):
self.__res = self.__response.text
self.__bs = bs(self.__res, 'html.parser')
def get_person_details(self):
details = self.__bs.find_all(class_="newshow2")
keys = []
values = []
for detail in details:
keys.extend(detail.find_all(class_='s1'))
values.extend(detail.find_all(class_='txt'))
# re去除html标签
re_html = re.compile('<[^<]+?>')
for i in range(len(keys)):
keys[i] = re_html.sub('', str(keys[i])).split('：')[0]
for j in range(len(values)):
values[j] = re_html.sub('', str(values[j])).split()[0]
return dict(zip(keys, values))
def get_family_love(self):
details = self.__bs.find_all(class_="newshow3")
keys = []
values = []
for detail in details:
keys.extend(detail.find_all(class_='s1'))
values.extend(detail.find_all(class_='txt'))
# re去除html标签
re_html = re.compile('<[^<]+?>')
for i in range(len(keys)):
keys[i] = re_html.sub('', str(keys[i])).split(':')[0]
for j in range(len(values)):
values[j] = re_html.sub('', str(values[j])).split()[0]
return dict(zip(keys, values))
def get_love_request(self):
table = self.__bs.find(class_="tableB")
tds = table.find_all('td')
keys = []
values = []
# re去除html标签
re_html = re.compile('<[^<]+?>')
for i in range(len(tds)):
s = re_html.sub(www.cjyl1yule.com'', str(tds[i])).split('：')[0]
if i % 2 == 0:
keys.append(s)
else:
values.append(s)
return dict(zip(keys, values))
def get_jpg_src(self):
photo = self.__bs.find(class_=www.oushengyule.com"love_photo")
jpg_src = photo.find('a')
return jpg_src['href']
词云封装，实现词图生成
'''
文件目录：CloudClass.py
'''
from wordcloud import WordCloud
from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
import jieba
class Cloud:
def __init__(self):
pass
def get_cloud(self, text):
# path = r'./'
font = r'./conf/FZSTK.TTF' # 字体path
text = (open('./conf/233.txt', 'r'www.zheshengyuLe.com, encoding='utf-8')).read() # 如果是中文的话encoding换成utf8
cut = jieba.cut(text) # 分词
string = ' '.join(cut)
print(len(string))
img = Image.open('./conf/timg.jpg') # 打开图片
img_array = np.array(img) # 将图片装换为数组
stopwords = {'要求': 0, '汉族': 0, '厘米': 0, '公斤': 0, '父亲': 0, '母亲': 0, '父母': 0,
'随意': 0, '其他': 0, '退休': 0, '对方': 0, '中文': 0, '普通话': 0, '不能': 0,
'以上': 0, '一般': 0, '无神论': 0, '建在': 0, '退休金': 0
} # 噪声词
wc = WordCloud(
scale=4, # 清晰度
background_color='white', # 背景颜色
max_words=400, # 最多单词
width=1000,
height=800,
mask=img_array,
font_path=font,
stopwords=stopwords # 停用词
)
wc.generate_from_text(string) # 绘制图片
plt.imshow(www.chaoyuepint.com)
plt.axis('off')
plt.figure()
plt.show() # 显示图片
wc.to_file('./conf/个人信息.png') # 保存图片
if __name__ == '__main__':
cloud = Cloud()
cloud.get_cloud()
主函数
'''
文件目录:Spider.py
'''
from yc.MongoClass import MongoOpt
from yc.SpiderClass import SpiderOpt
from yc.CloudClass import Cloud
class Spider:
def spider(self, num):
try:
headers = {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
"Connection": "keep-alive",
"Cookie": "PHPSESSID=adgqumos4shf5pj3tkvfuohhs6; yc123_loveyun_loveloginMember=AAIGDhpVVQIHBggHWl1RUVMFAwcHAF8EUgEKUAFSXVZXV1BXVw%3D%3D; yc123_loveyun_showhxtoday=NA%3D%3D; yc123_loveyun_showweixinpushtoday=NA%3D%3D;",
"Host": www.yifayule2d.com"love.yc123.com",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36",
}
spider = SpiderOpt(www.yifa5yl.com'GET', 'https://www.zzhehong.com love.yc123.com/show.php?id=' + str(num), headers=headers)
spider.get_response()
person_details = spider.get_person_details()
# family_love = spider.get_family_love()
love_request = spider.get_love_request()
# jpg_src = spider.get_jpg_src()
result = {}
# result.update(person_details) # 个人情况
# result.update(family_love) # 家庭情况
result.update(love_request) # 择偶要求
# result.update({"相片": jpg_src})
mongo = MongoOpt()
mongo.add_collection(result)
return True
except:
return False
if __name__ == '__main__':
spider = Spider()
mongo = MongoOpt()
cloud = Cloud()
# x, y = 0, 0
# for i in range(3001, 4720):
# if not spider.spider(i):
# x += 1
# print(i, "爬取无效")
# continue
# y += 1
# print(i, "爬取完成")
# print(x, y, sep=" ")
text = mongo.get_all_collections()
cloud.get_cloud(text)