python scrapy 插入数据库的操作

时间：2021-11-24 01:12:36
需要安装这个
pymysql
写法还是很简单的

# -*- coding: utf-8 -*-

# Define your item pipelines here

#

# Don't forget to add your pipeline to the ITEM_PIPELINES setting

# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import pymysql

import douban.settings  as settings

class DoubanPipeline(object):

    def __init__(self):

    # 连接数据库

        self.connect = pymysql.connect(

        host=settings.MYSQL_HOST,

        db=settings.MYSQL_DBNAME,

        user=settings.MYSQL_USER,

        passwd=settings.MYSQL_PASSWD,

        charset='utf8',

        use_unicode=True)

    # 通过cursor执行增删查改

        self.cursor = self.connect.cursor();

    def process_item(self, item, spider):

        #数据库的操作

         try:

            # 插入数据

            self.cursor.execute(

                """insert into course(name,teacher, content, comment )

                value (%s, %s, %s, %s)""",

                (  item['bookName'],

                 item['author'],

                 item['score'],

                 item['comment']))

            # 提交sql语句

            self.connect.commit()

         except Exception as e:

            print("错误在这里>>>>>>>>>>>>>",e,"<<<<<<<<<<<<<错误在这里")

        #写入文件txt

         with open('xuetang-0823.txt','a',encoding='utf-8') as fl:

            fl.write(str(item['xuhao'])+','+item['bookName']+','

                     +item['author']+','

                     +item['score'].replace("\n","").replace("\t","").replace(",","，").replace("\r","")+','+item['comment'].replace("\n","").replace("\t","").replace("\r","")+'\n')

相关文章