本文实例为大家分享了python爬取网页内容转换为PDF的具体代码，供大家参考，具体内容如下
将廖雪峰的学习教程转换成PDF文件，代码只适合该网站，如果需要其他网站的教程，可靠需要进行稍微的修改。
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
|
# coding=utf-8
import os
import re
import time
import pdfkit
import requests
from bs4 import BeautifulSoup
from PyPDF2 import PdfFileMerger
import sys
reload(sys)
sys.setdefaultencoding('utf8')
html_template = """
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
</head>
<body>
{content}
</body>
</html>
"""
#----------------------------------------------------------------------
def parse_url_to_html(url, name):
"""
解析URL，返回HTML内容
:param url:解析的url
:param name: 保存的html文件名
:return: html
"""
try:
response = requests.get(url)
soup = BeautifulSoup(response.content, 'html.parser')
# 正文
body = soup.find_all(class_="x-wiki-content")[0]
# 标题
title = soup.find('h4').get_text()
# 标题加入到正文的最前面，居中显示
center_tag = soup.new_tag("center")
title_tag = soup.new_tag('h1')
title_tag.string = title
center_tag.insert(1, title_tag)
body.insert(1, center_tag)
html = str(body)
# body中的img标签的src相对路径的改成绝对路径
pattern = "(<img .*?src=\")(.*?)(\")"
def func(m):
if not m.group(3).startswith("http"):
rtn = m.group(1) + "http://www.liaoxuefeng.com" + m.group(2) + m.group(3)
return rtn
else:
return m.group(1)+m.group(2)+m.group(3)
html = re.compile(pattern).sub(func, html)
html = html_template.format(content=html)
html = html.encode("utf-8")
with open(name, 'wb') as f:
f.write(html)
return name
except Exception as e:
print "解析错误！"
#----------------------------------------------------------------------
def get_url_list():
"""
获取所有URL目录列表
:return:
"""
response = requests.get("http://www.liaoxuefeng.com/wiki/0014316089557264a6b348958f449949df42a6d3a2e542c000")
soup = BeautifulSoup(response.content, "html.parser")
menu_tag = soup.find_all(class_="uk-nav uk-nav-side")[1]
urls = []
for li in menu_tag.find_all("li"):
url = "http://www.liaoxuefeng.com" + li.a.get('href')
urls.append(url)
return urls
#----------------------------------------------------------------------
def save_pdf(htmls, file_name):
"""
把所有html文件保存到pdf文件
:param htmls: html文件列表
:param file_name: pdf文件名
:return:
"""
options = {
'page-size': 'Letter',
'margin-top': '0.75in',
'margin-right': '0.75in',
'margin-bottom': '0.75in',
'margin-left': '0.75in',
'encoding': "UTF-8",
'custom-header': [
('Accept-Encoding', 'gzip')
],
'cookie': [
('cookie-name1', 'cookie-value1'),
('cookie-name2', 'cookie-value2'),
],
'outline-depth': 10,
}
pdfkit.from_file(htmls, file_name, options=options)
#----------------------------------------------------------------------
def main():
start = time.time()
file_name = u"liaoxuefeng_Python3_tutorial"
urls = get_url_list()
for index, url in enumerate(urls):
parse_url_to_html(url, str(index) + ".html")
htmls =[]
pdfs =[]
for i in range(0,124):
htmls.append(str(i)+'.html')
pdfs.append(file_name+str(i)+'.pdf')
save_pdf(str(i)+'.html', file_name+str(i)+'.pdf')
print u"转换完成第"+str(i)+'个html'
merger = PdfFileMerger()
for pdf in pdfs:
merger.append(open(pdf,'rb'))
print u"合并完成第"+str(i)+'个pdf'+pdf
output = open(u"廖雪峰Python_all.pdf", "wb")
merger.write(output)
print u"输出PDF成功！"
for html in htmls:
os.remove(html)
print u"删除临时文件"+html
for pdf in pdfs:
os.remove(pdf)
print u"删除临时文件"+pdf
total_time = time.time() - start
print(u"总共耗时：%f 秒" % total_time)
#----------------------------------------------------------------------
def changeDir(dir_name):
"""
目录切换
"""
if not os.path.exists(dir_name):
os.mkdir(dir_name)
os.chdir(dir_name)
#----------------------------------------------------------------------
if __name__ == '__main__':
#存放文件的路径
dir_name = '/home/Python/Html'
changeDir(dir_name)
main()
|
以上就是本文的全部内容，希望对大家的学习有所帮助，也希望大家多多支持服务器之家。
原文链接：https://blog.csdn.net/manjianchao/article/details/71122815