这个是我想爬取的链接：http://www.lagou.com/zhaopin/Python/?labelWords=label
页面显示如下：
在Chrome浏览器中审查元素，找到对应的链接：
然后依次针对相应的链接（比如上面显示的第一个，链接为：http://www.lagou.com/jobs/2234309.html），打开之后查看，下面是我想具体爬取的每个公司岗位相关信息：
针对想要爬取的内容信息，找到html代码标签位置：
找到了相关的位置之后，就可以进行爬取的操作了。
以下是代码部分
# -*- coding:utf-8 -*- import urllib
import urllib2
from bs4 import BeautifulSoup
import re
import xlwt # initUrl = 'http://www.lagou.com/zhaopin/Python/?labelWords=label'
def Init(skillName):
totalPage = 30
initUrl = 'http://www.lagou.com/zhaopin/'
# skillName = 'Java'
userAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'
headers = {'User-Agent':userAgent} # create excel sheet
workBook = xlwt.Workbook(encoding='utf-8')
sheetName = skillName + ' Sheet'
bookSheet = workBook.add_sheet(sheetName)
rowStart = 0
for page in range(totalPage):
page += 1
print '##################################################### Page ',page,'#####################################################'
currPage = initUrl + skillName + '/' + str(page) + '/?filterOption=3'
# print currUrl
try:
request = urllib2.Request(currPage,headers=headers)
response = urllib2.urlopen(request)
jobData = readPage(response)
# rowLength = len(jobData)
for i,row in enumerate(jobData):
for j,col in enumerate(row):
bookSheet.write(rowStart + i,j,col)
rowStart = rowStart + i +1
except urllib2.URLError,e:
if hasattr(e,"code"):
print e.code
if hasattr(e,"reason"):
print e.reason
xlsName = skillName + '.xls'
workBook.save(xlsName) def readPage(response):
btfsp = BeautifulSoup(response.read())
webLinks = btfsp.body.find_all('div',{'class':'p_top'})
# webLinks = btfsp.body.find_all('a',{'class':'position_link'})
# print weblinks.text
count = 1
jobData = []
for link in webLinks:
print 'No.',count,'==========================================================================================='
pageUrl = link.a['href']
jobList = loadPage(pageUrl)
# print jobList
jobData.append(jobList)
count += 1
return jobData def loadPage(pageUrl):
currUrl = 'http:' + pageUrl
userAgent = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'
headers = {'User-Agent':userAgent}
try:
request = urllib2.Request(currUrl,headers=headers)
response = urllib2.urlopen(request)
content = loadContent(response.read())
return content
except urllib2.URLError,e:
if hasattr(e,"code"):
print e.code
if hasattr(e,"reason"):
print e.reason def loadContent(pageContent):
# print pageContent
btfsp = BeautifulSoup(pageContent)
# job infomation
job_detail = btfsp.find('dl',{'id':'job_detail'})
jobInfo = job_detail.h1.text
tempInfo = re.split(r'(?:\s*)',jobInfo) # re.split is better than the Python's raw split function
jobTitle = tempInfo[1]
jobName = tempInfo[2]
job_request = job_detail.find('dd',{'class':'job_request'})
reqList = job_request.find_all('p')
jobAttract = reqList[1].text
publishTime = reqList[2].text
itemLists = job_request.find_all('span')
salary = itemLists[0].text
workplace = itemLists[1].text
experience = itemLists[2].text
education = itemLists[3].text
worktime = itemLists[4].text # company's infomation
jobCompany = btfsp.find('dl',{'class':'job_company'})
# companyName = jobCompany.h2
companyName = re.split(r'(?:\s*)',jobCompany.h2.text)[1]
companyInfo = jobCompany.find_all('li')
# workField = companyInfo[0].text.split(' ',1)
workField = re.split(r'(?:\s*)|(?:\n*)',companyInfo[0].text)[2]
# companyScale = companyInfo[1].text
companyScale = re.split(r'(?:\s*)|(?:\n*)',companyInfo[1].text)[2]
# homePage = companyInfo[2].text
homePage = re.split(r'(?:\s*)|(?:\n*)',companyInfo[2].text)[2]
# currStage = companyInfo[3].text
currStage = re.split(r'(?:\s*)|(?:\n*)',companyInfo[3].text)[1]
financeAgent = ''
if len(companyInfo) == 5:
# financeAgent = companyInfo[4].text
financeAgent = re.split(r'(?:\s*)|(?:\n*)',companyInfo[4].text)[1]
workAddress = ''
if jobCompany.find('div',{'class':'work_addr'}):
workAddress = jobCompany.find('div',{'class':'work_addr'})
workAddress = ''.join(workAddress.text.split()) # It's sooooo cool! # workAddress = jobCompany.find('div',{'class':'work_addr'})
# workAddress = ''.join(workAddress.text.split()) # It's sooooo cool! infoList = [companyName,jobTitle,jobName,salary,workplace,experience,education,worktime,jobAttract,publishTime,
workField,companyScale,homePage,workAddress,currStage,financeAgent] return infoList def SaveToExcel(pageContent):
pass if __name__ == '__main__':
# Init(userAgent)
Init('Python')
也是一边摸索一边来进行的，其中的一些代码写的不是很规范和统一。
结果显示如下：
考虑打算下一步可以对相关的信息进行处理分析下，比如统计一下分布、薪资水平等之类的。
原文地址：http://www.cnblogs.com/leonwen/p/5769888.html
欢迎交流，请不要私自转载，谢谢