python BeautifulSoup 获取页面多个子节点中的各个节点的内容

页面html格式为

 import httplib

 from BeautifulSoup import BeautifulSoup

 def main():

     f = open('result','a')

     headers = {'Content-Type':'application/x-www-form-urlencoded',

         'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',

         'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',

         'Accept-Encoding': 'gzip, deflate',

         'Referer': 'http://xxx.xxx.com/admin/userlist',

         'Cookie': 'JSESSIONID=9F6F2D03D2C11400B3D6731E90D73117',

         'User-Agent': 'User-Agent: Mozilla/5.0 (Windows NT 6.1; rv:46.0) Gecko/20100101 Firefox/46.0',

     }

     conn = httplib.HTTPConnection('*.*.*.*', timeout=50)

     for p in range(1,1287):

         print p

         conn.request(method='GET',

                         url="/admin/userlist?toPage=%s&sessionID=" % str(p),

                         headers=headers)

         resp = conn.getresponse()

         html_doc = resp.read()

         mainSoup = BeautifulSoup(html_doc)

         for s in mainSoup.findAll('tr', attrs={'bgcolor':'#7bb5de'}):

             if 'style' not in str(s):

                 continue

             for d in s.findAll('td'):

                 print d.getText(),

                 f.write("%s " % d.getText().encode('utf-8'))   #f.write("%s " % d.getText())==> UnicodeEncodeError: 'ascii' codec can't encode characters in position 0-2: ordinal not in range(128)

             f.write("%s\n" % d.getText().encode('utf-8'))

             print

     f.close()

     conn.close()

 if __name__ == '__main__':

     main()

秒客网

python BeautifulSoup 获取页面多个子节点中的各个节点的内容

相关文章