和昨天一样的工作量，时间只用了一半，但还是效率有点低了，因为要把两个网页结合起来，所以在列表操作上用了好多时间
import requests
from lxml import etree headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36'} def get_html(url):
response = requests.get(url, headers=headers)
response.encoding = response.apparent_encoding
html = response.text
return html def parse_html(html):
informations = []
urls = []
html_element = etree.HTML(html)
kinds = html_element.xpath('(//tr[@class="even"]|//tr[@class="odd"])/td[2]/text()')
'''
kinds:
['技术类', '设计类', '技术类', '技术类', '技术类', '技术类', '技术类', '技术类', '技术类', '产品/项目类']
'''
nums = html_element.xpath('(//tr[@class="even"]|//tr[@class="odd"])//td[3]/text()')
'''
nums:
['2', '1', '2', '1', '2', '2', '1', '2', '1', '1']
'''
addresses = html_element.xpath('(//tr[@class="even"]|//tr[@class="odd"])//td[4]/text()')
'''
addresses:
['深圳', '深圳', '深圳', '深圳', '深圳', '深圳', '深圳', '深圳', '深圳', '深圳']
'''
times = html_element.xpath('(//tr[@class="even"]|//tr[@class="odd"])//td[5]/text()')
'''
times:
['2018-08-04', '2018-08-04', '2018-08-04', '2018-08-04', '2018-08-04', '2018-08-04', '2018-08-04', '2018-08-04', '2018-08-04', '2018-08-04']
'''
names = html_element.xpath('(//tr[@class="even"]|//tr[@class="odd"])//a/text()') detail_url = html_element.xpath('(//tr[@class="even"]|//tr[@class="odd"])//a/@href')
for str_url in detail_url: url = 'https://hr.tencent.com/' + str(str_url)
urls.append(url) '''
urls :
['https://hr.tencent.com/position_detail.php?id=42917&keywords=python&tid=0&lid=0',
'https://hr.tencent.com/position_detail.php?id=42908&keywords=python&tid=0&lid=0',
......
'https://hr.tencent.com/position_detail.php?id=42832&keywords=python&tid=0&lid=0',
'https://hr.tencent.com/position_detail.php?id=42628&keywords=python&tid=0&lid=0']
'''
for index, name in enumerate(names):
information = {}
information['name'] = name
information['url'] = urls[index]
information['kind'] = kinds[index]
information['nums_of_need'] = nums[index]
information['address'] = addresses[index]
informations.append(information)
# print(informations)
# print(urls)
return urls, informations def parse_detail_page(url):
#one detail page
html = get_html(url)
return html def get_all_page(page_nums):
for i in range(0, page_nums):
url = 'https://hr.tencent.com/position.php?lid=&tid=&keywords=python&start={0}#a'.format(i*10)
html = get_html(url)
urls, informations = parse_html(html)
# print(informations)
works = []
for i, url in enumerate(urls): html_detail = parse_detail_page(url)
html_element = etree.HTML(html_detail)
work_intro = html_element.xpath('//td[@class="l2"]//text()')
for index, text in enumerate(work_intro):
if text.startswith('工作职责：'):
text = text.replace('工作职责：', '')
works_detail = {}
intros = []
for x in range(index+1, len(work_intro)):
intro = work_intro[x].strip()
if work_intro[x].startswith('工作要求：'):
break
intros.append(intro)
while '' in intros:
intros.remove('')
works_detail['1_____工作职责：'] = intros
works.append(works_detail)
# print(intros)
'''
['负责NLP与深度学习相关技术的研究与实现；',
'负责建设基础的语义分析工具和平台；',
'负责搜索系统、知识图谱系统、问答与对话系统的设计与搭建；',
'结合实际业务需求与数据，研发高效、稳健、完备的NLP解决方案。']
''' if text.startswith('工作要求：'):
text = text.replace('工作要求：', '')
works_detail = {}
requests = []
for x in range(index+1, len(work_intro)):
intro = work_intro[x].strip()
if work_intro[x].startswith('申请岗位'):
break
requests.append(intro)
while '' in requests:
requests.remove('')
works_detail['2_____工作要求：'] = requests
works.append(works_detail)
# print(requests)
'''
['三年以上自然语言处理经验包括语义表示、搜索、知识图谱、对话系统等；',
'扎实的编程基础，至少精通一种编程语言，如C++，Java，python等；',
'熟悉深度学习以及常见机器学习算法的原理与算法，能熟练运用聚类、分类、回归、排序等模型解决有挑战性的问题；',
'对自然语言处理相关的分词、词性标注、实体识别、句法分析、语义分析等有深入的实践经验；',
'有强烈求知欲，对人工智能领域相关技术有热情；', '具有良好的数学基础，良好的英语阅读能力；',
'有项目管理经验，与他人合作良好，能够独立有效推动复杂项目。']
'''
return works, informations def main():
works, informations = get_all_page(1)
for index, information in enumerate(informations):
list = []
list.append(works[index*2])
list.append(works[index*2+1])
information['duty'] = list
print(information) if __name__ == '__main__':
main()
目前sublime还输入不了中文，所以把输出注释上，方便看清格式
运行结果：
红色圈出来的是一个字典，包含第一个网页的信息（职位名称，url，位置）和详情页面的职责（工作职责，工作要求），嵌套的可能有点复杂，但目前还没有想到更简明的方法