爬遍整个网络
1 当我们访问整个网络的时候，我们不可避免的会访问不同的网站，但是不同的网站会有完全不同的结构和内容...
现在一步一步的构建访问整个网络的脚本
I 从一个网站开始，每一次都爬向不同的网站。如果在一个页面找不到指向其他网站的链接，获取本网站其他界面信息，直到找到其他网站的链接。
# -*- coding:utf-8 -*- from urllib.request import urlopen
from urllib.error import HTTPError
from bs4 import BeautifulSoup
from random import choice
import re basename = "http://en.wikipedia.org"
visitedpages = set() def getInternalLinks(bsObj,includeUrl):
return [eachlink.attrs['href'] for eachlink in bsObj.find_all("a",href=re.compile("^(/|.*" + includeUrl + ")")) if 'href' in eachlink.attrs] def getExternalLinks(bsObj,excludeUrl):
return [eachlink.attrs['href'] for eachlink in bsObj.find_all("a",href=re.compile("^(http|www)((?!" + excludeUrl + ").)*$")) if 'href' in eachlink.attrs] def splitAddress(address):
addressParts = address.replace("http://","").split("/")
return addressParts def getRandomExternalLink(startingPage):
html = urlopen(startingPage)
with html:
bsObj = BeautifulSoup(html,"html.parser")
externalLinks = getExternalLinks(bsObj,splitAddress(startingPage)[0])
if len(externalLinks) == 0:
internalLinks = getInternalLinks(bsObj, splitAddress(startingPage)[0])
return choice(internalLinks)
else:
return choice(externalLinks) def followExternalLink(startingPage):
externalLink = getRandomExternalLink("http://www.oreilly.com/")
if externalLink in visitedpages:
print("visited")
else:
print("the random external link is " + externalLink)
visitedpages.add(externalLink)
followExternalLink(externalLink) if __name__ == "__main__":
#print(splitAddress("http://www.oreilly.com/")[0])
#print(getRandomExternalLink("http://www.oreilly.com/"))
followExternalLink("http://www.oreilly.com/")
II 从一个网站开始，查找这个网站所有界面信息，获取整个网站指向其他网站的链接
# -*- coding:utf-8 -*- from urllib.request import urlopen
from urllib.error import HTTPError
from bs4 import BeautifulSoup
from random import choice
import re def getInternalLinks(bsObj,includeUrl):
return [eachlink.attrs['href'] for eachlink in bsObj.find_all("a",href=re.compile("^(/|.*" + includeUrl + ")")) if 'href' in eachlink.attrs] def getExternalLinks(bsObj,excludeUrl):
return [eachlink.attrs['href'] for eachlink in bsObj.find_all("a",href=re.compile("^(http|www)((?!" + excludeUrl + ").)*$")) if 'href' in eachlink.attrs] def splitAddress(address):
addressParts = address.replace("http://","").split("/")
return addressParts allINlinks = set()
allEXlinks = set()
def getAllexternalLinks(startPage):
try:
with urlopen(startPage) as html:
bsObj = BeautifulSoup(html,"html.parser")
except HTTPError as e:
print(e)
else:
allinternallinks = getInternalLinks(bsObj,splitAddress(startPage)[0])
allexternallinks = getExternalLinks(bsObj,splitAddress(startPage)[0])
print("************external*******************************")
for eachexternallink in allexternallinks:
if eachexternallink not in allEXlinks:
allEXlinks.add(eachexternallink)
print(eachexternallink)
print("************internal*******************************")
for eachinternallink in allinternallinks:
if eachinternallink not in allINlinks:
allINlinks.add(eachinternallink)
print(eachinternallink)
getAllexternalLinks(eachinternallink) if __name__ == "__main__":
getAllexternalLinks("http://www.oreilly.com/")
***************还存在问题的代码***************************