使用Python，HtmlParser来统计深圳市保障房申请人的原籍省份分布，年龄分布等。从侧面可以反映鹏城人的地域分布。以下python代码增大了每一次获取的记录数，从而少提交几次请求。如果按照WEB主页设定的每一次请求最多50个记录，那就得提交数千次请求，显然费时。另外，也可以使用多线程处理，快速获得数据，解析数据，然后使用pandas，matplotlib等工具进行数据处理和绘制。查询了系统，截止2016年2月，轮候系统的保障房人数大概4万多，公租房轮候人数大概5万，以下数据仅作学习使用，统计结果如下：
毫无疑问，广东本地人申请的占多数。前十名当中，和广东接壤的省份也占了不少比例，特别是两湖，江西，剩下的由人口大省占据。深圳保障房建设速度和规模居全国首位，但是因为人数众多，所以需要排队等候。远离XX的房东，避免年年涨的房租，那就加入排队轮候大军吧。
# -*- coding:utf-8 -*-
import time
import json
import lxml.html
from lxml import etree
from HTMLParser import HTMLParser #使用beautifulsoup也可以 #http://www.crummy.com/software/BeautifulSoup/
#http://blog.csdn.net/my2010sam/article/details/14526223 try:
from urllib.request import urlopen,Request
except:
from urllib2 import urlopen, Request area={"":"北京","":"天津","":"河北","":"山西","":"内蒙古","":"辽宁","":"吉林","":"黑龙江","":"上海",
"":"江苏","":"浙江","":"安徽","":"福建","":"江西","":"山东","":"河南","":"湖北","":"湖南",
"":"广东","":"广西","":"海南","":"重庆","":"四川","":"贵州","":"云南","":"*","":"陕西",
"":"甘肃","":"青海","":"宁夏","":"*","":"*","":"香港","":"澳门","":"国外"}
ages =[0]*11
provinceCnt=[0]*91
RECORD_BY_EACH_PAGE = [10,15,30,50,5000]
currentYear=time.localtime()[0]#get year
URL_BY_PAGESIZE='http://bzflh.szjs.gov.cn/TylhW/lhmcAction.do?pageSize=%s&method=queryYgbLhmcInfo&waittype=2' #http://XXX.cn?pageSize=XXX&page=XXX,waittype=2 公租房，waittype=1 安居房
URL_BY_PAGE_PAGESIZE ='http://bzflh.szjs.gov.cn/TylhW/lhmcAction.do?pageSize=%s&method=queryYgbLhmcInfo&waittype=%s&page=%s' #Social_Housing_Items=[URL_BY_PAGE_PAGESIZE_GongZuFang,URL_BY_PAGE_PAGESIZE_AnJuFang] def getHomePage(url,pagesize):
try:
request = Request(url)
lines=urlopen(request,timeout=10).read()
if len(lines)<20:
return None #no data
except Exception as e:
print e
else:
if pagesize!=10 and pagesize!=15 and pagesize!=30 and pagesize!=50 and pagesize !=5000:
pagesize = 15 #default as 15 record each page
lines=lines.decode('utf-8')
splitLines=lines.split('\r\n')
for line in splitLines:
#if "pageSize" in line:
#print line[:50]
if "pagebanner" in line:
totalPage= line[:50].split('>')[1].split(' ')[0]
totalPage=totalPage.split(',')
if len(totalPage)>1:
pages=(int(totalPage[0])*1000+int(totalPage[1]))/pagesize
return pages def getRawData(url):
try:
request = Request(url)
lines=urlopen(request,timeout=10).read()
if len(lines)<20:
return None #no data
except Exception as e:
print e
else:
return lines.decode('utf-8') def getIdentityInfo(code):
"""
:param code: identity code showing province and date
:return: province,date
"""
provinceCode=code[:2]
cityCode = code[2:6]
date=code[6:10]
return provinceCode,date class Dataparser(HTMLParser):
def __init__(self):
HTMLParser.__init__(self)
self.tr=False
self.td =0
self.data =False
def handle_starttag(self,tag,attrs):
"""
参数tag是标签名,比如td,tr',attrs为标签所有属性(name,value)列表,这里是[('class','para')]
:param tag:
:param attrs:
:return:
"""
if tag=='tr':
self.tr=True
if tag =='td'and self.tr==True:
self.data = True
for name,value in attrs:
print "name and value are",name,value
def handle_endtag(self,tag):
if tag=='td':
self.data = False
#print "a end tag:",tag,self.getpos() def handle_data(self,data):
if self.data and len(data)==18 and '\r\n' not in data:
#print data #ID card NO
provinceCode,date=getIdentityInfo(data)
ageRange=currentYear - int(date)
if ageRange>=100:
print 'test',ageRange
#ages[ageRange/10] +=1
#temp=area[provinceCode].decode('utf-8')
PC=int(provinceCode)
provinceCnt[PC]+=1 if __name__ =='__main__':
#计算总共页数，每页可以自己限定
for type in range(2):
pages=getHomePage(URL_BY_PAGE_PAGESIZE%(RECORD_BY_EACH_PAGE[0],type+1,1),RECORD_BY_EACH_PAGE[4])
parse=Dataparser()
while pages>=1:
#for page in range(pages):
lines=getRawData(URL_BY_PAGE_PAGESIZE%(RECORD_BY_EACH_PAGE[4],type+1,pages))
parse.feed(lines)
#parse.close()
pages-=1
parse.close()
if type==0:
print "深圳安居房申请人全国分布情况统计:"
for i in provinceCnt:
if i>0: #只打印有数据的省份
pIndex=str(provinceCnt.index(i))
print area[pIndex],i
provinceCnt =[0]*91
elif type==1:
print "深圳公租房申请人全国分布情况统计:"
for i in provinceCnt:
if i>0: #只打印有数据的省份
pIndex=str(provinceCnt.index(i))
print area[pIndex],i
provinceCnt =[0]*91