springboot+webmagic实现java爬虫jdbc及mysql的方法

时间:2022-03-05 23:36:02

前段时间需要爬取网页上的信息,自己对于爬虫没有任何了解,就了解了一下webmagic,写了个简单的爬虫。

一、首先介绍一下webmagic:

webmagic采用完全模块化的设计,功能覆盖整个爬虫的生命周期(链接提取、页面下载、内容抽取、持久化),支持多线程抓取,分布式抓取,并支持自动重试、自定义ua/cookie等功能。

实现理念:

springboot+webmagic实现java爬虫jdbc及mysql的方法

maven依赖:

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
<dependency>
   <groupid>us.codecraft</groupid>
   <artifactid>webmagic-core</artifactid>
   <version>0.7.3</version>
  </dependency>
  <dependency>
   <groupid>us.codecraft</groupid>
   <artifactid>webmagic-extension</artifactid>
   <version>0.7.3</version>
  </dependency>
 
  <dependency>
   <groupid>us.codecraft</groupid>
   <artifactid>webmagic-extension</artifactid>
   <version>0.7.3</version>
   <exclusions>
    <exclusion>
     <groupid>org.slf4j</groupid>
     <artifactid>slf4j-log4j12</artifactid>
    </exclusion>
   </exclusions>
  </dependency>

jdbc模式:

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
ublic class csdnblogdao {
  private connection conn = null;
  private statement stmt = null;
 
  public csdnblogdao() {
    try {
      class.forname("com.mysql.jdbc.driver");
      string url = "jdbc:mysql://localhost:3306/test?"
          + "user=***&password=***3&useunicode=true&characterencoding=utf8";
      conn = drivermanager.getconnection(url);
      stmt = conn.createstatement();
    } catch (classnotfoundexception e) {
      e.printstacktrace();
    } catch (sqlexception e) {
      e.printstacktrace();
    }
 
  }
 
  public int add(csdnblog csdnblog) {
    try {
      string sql = "insert into `test`.`csdnblog` (`keyes`, `titles`, `content` , `dates`, `tags`, `category`, `views`, `comments`, `copyright`) values (?, ?, ?, ?, ?, ?, ?, ?,?);";
      preparedstatement ps = conn.preparestatement(sql);
      ps.setint(1, csdnblog.getkey());
      ps.setstring(2, csdnblog.gettitle());
      ps.setstring(3,csdnblog.getcontent());
      ps.setstring(4, csdnblog.getdates());
      ps.setstring(5, csdnblog.gettags());
      ps.setstring(6, csdnblog.getcategory());
      ps.setint(7, csdnblog.getview());
      ps.setint(8, csdnblog.getcomments());
      ps.setint(9, csdnblog.getcopyright());
      return ps.executeupdate();
    } catch (sqlexception e) {
      e.printstacktrace();
    }
    return -1;
  }
}

实体类:

  1. public class csdnblog { 
  2.   private int key;// 编号 
  3.  
  4.   private string title;// 标题 
  5.  
  6.   private string dates;// 日期 
  7.  
  8.   private string tags;// 标签 
  9.  
  10.   private string category;// 分类 
  11.  
  12.   private int view;// 阅读人数 
  13.  
  14.   private int comments;// 评论人数 
  15.  
  16.   private int copyright;// 是否原创 
  17.  
  18.   private string content; //文字内容 
  19.  
  20.   public string getcontent() { 
  21.     return content; 
  22.   } 
  23.  
  24.   public void setcontent(string content) { 
  25.     this.content = content; 
  26.   } 
  27.  
  28.   public int getkey() { 
  29.     return key; 
  30.   } 
  31.  
  32.   public void setkey(int key) { 
  33.     this.key = key; 
  34.   } 
  35.  
  36.   public string gettitle() { 
  37.     return title; 
  38.   } 
  39.  
  40.   public void settitle(string title) { 
  41.     this.title = title; 
  42.   } 
  43.  
  44.   public string getdates() { 
  45.     return dates; 
  46.   } 
  47.  
  48.   public void setdates(string dates) { 
  49.     this.dates = dates; 
  50.   } 
  51.  
  52.   public string gettags() { 
  53.     return tags; 
  54.   } 
  55.  
  56.   public void settags(string tags) { 
  57.     this.tags = tags; 
  58.   } 
  59.  
  60.   public string getcategory() { 
  61.     return category; 
  62.   } 
  63.  
  64.   public void setcategory(string category) { 
  65.     this.category = category; 
  66.   } 
  67.  
  68.   public int getview() { 
  69.     return view; 
  70.   } 
  71.  
  72.   public void setview(int view) { 
  73.     this.view = view; 
  74.   } 
  75.  
  76.   public int getcomments() { 
  77.     return comments; 
  78.   } 
  79.  
  80.   public void setcomments(int comments) { 
  81.     this.comments = comments; 
  82.   } 
  83.  
  84.   public int getcopyright() { 
  85.     return copyright; 
  86.   } 
  87.  
  88.   public void setcopyright(int copyright) { 
  89.     this.copyright = copyright; 
  90.   } 
  91.   public string tostring() { 
  92.     return "csdnblog [key=" + key + ", title=" + title + ", content=" + content + ",dates=" + dates + ", tags=" + tags + ", category=" 
  93.         + category + ", view=" + view + ", comments=" + comments + ", copyright=" + copyright + "]"
  94.   } 

启动类:

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
public class csdnblogpageprocessor implements pageprocessor {
 
 
  private static string username="chenyufeng1991"; // 设置csdn用户名
 
  private static int size = 0;// 共抓取到的文章数量
 
  // 抓取网站的相关配置,包括:编码、抓取间隔、重试次数等
  private site site = site.me().setretrytimes(3).setsleeptime(1000);
 
  public site getsite() {
    return site;
  }
 
 
  // process是定制爬虫逻辑的核心接口,在这里编写抽取逻辑
  public void process(page page) {
    // 列表页
    if (!page.geturl().regex("http://blog.csdn.net/" + username + "/article/details/d+").match()) {
      // 添加所有文章页
      page.addtargetrequests(page.gethtml().xpath("//div[@id='article_list']").links()// 限定文章列表获取区域
          .regex("/" + username + "/article/details/d+")
          .replace("/" + username + "/", "http://blog.csdn.net/" + username + "/")// 巧用替换给把相对url转换成绝对url
          .all());
      // 添加其他列表页
      page.addtargetrequests(page.gethtml().xpath("//div[@id='papelist']").links()// 限定其他列表页获取区域
          .regex("/" + username + "/article/list/d+")
          .replace("/" + username + "/", "http://blog.csdn.net/" + username + "/")// 巧用替换给把相对url转换成绝对url
          .all());
      // 文章页
    } else {
      size++;// 文章数量加1
      // 用csdnblog类来存抓取到的数据,方便存入数据库
      csdnblog csdnblog = new csdnblog();
      // 设置编号
      csdnblog.setkey(integer.parseint(
          page.geturl().regex("http://blog.csdn.net/" + username + "/article/details/(d+)").get()));
      // 设置标题
      csdnblog.settitle(
          page.gethtml().xpath("//div[@class='article_title']//span[@class='link_title']/a/text()").get());
 
      //设置内容
      csdnblog.setcontent(
          page.gethtml().xpath("//div[@class='article_content']/alltext()").get());
 
      // 设置日期
      csdnblog.setdates(
          page.gethtml().xpath("//div[@class='article_r']/span[@class='link_postdate']/text()").get());
      // 设置标签(可以有多个,用,来分割)
      csdnblog.settags(listtostring(page.gethtml().xpath("//div[@class='article_l']/span[@class='link_categories']/a/alltext()").all()));
      // 设置类别(可以有多个,用,来分割)
      csdnblog.setcategory(listtostring(page.gethtml().xpath("//div[@class='category_r']/label/span/text()").all()));
      // 设置阅读人数
      csdnblog.setview(integer.parseint(page.gethtml().xpath("//div[@class='article_r']/span[@class='link_view']")
          .regex("(d+)人阅读").get()));
      // 设置评论人数
      csdnblog.setcomments(integer.parseint(page.gethtml()
          .xpath("//div[@class='article_r']/span[@class='link_comments']").regex("((d+))").get()));
      // 设置是否原创
      csdnblog.setcopyright(page.gethtml().regex("bog_copyright").match() ? 1 : 0);
      // 把对象存入数据库
      new csdnblogdao().add(csdnblog);
      // 把对象输出控制台
      system.out.println(csdnblog);
    }
  }
 
  // 把list转换为string,用,分割
  public static string listtostring(list<string> stringlist) {
    if (stringlist == null) {
      return null;
    }
    stringbuilder result = new stringbuilder();
    boolean flag = false;
    for (string string : stringlist) {
      if (flag) {
        result.append(",");
      } else {
        flag = true;
      }
      result.append(string);
    }
    return result.tostring();
  }
 
  public static void main(string[] args) {
    long starttime, endtime;
    system.out.println("【爬虫开始】...");
    starttime = system.currenttimemillis();
    // 从用户博客首页开始抓,开启5个线程,启动爬虫
    spider.create(new csdnblogpageprocessor()).addurl("http://blog.csdn.net/" + username).thread(5).run();
    endtime = system.currenttimemillis();
    system.out.println("【爬虫结束】共抓取" + size + "篇文章,耗时约" + ((endtime - starttime) / 1000) + "秒,已保存到数据库,请查收!");
  }
}

使用mysql类型:

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
public class gamepageprocessor implements pageprocessor {
 
  private static final logger logger = loggerfactory.getlogger(gamepageprocessor.class);
  private static dianjingservice d;
  private static bannerservice bs;
  private static sportservice ss;
  private static yulenewsservice ys;
 
  private static updateservice ud ;
  // 抓取网站的相关配置,包括:编码、抓取间隔、重试次数等
  private site site = site.me().setretrytimes(3).setsleeptime(1000);
 
  public site getsite() {
    return site;
  }
  // process是定制爬虫逻辑的核心接口,在这里编写抽取逻辑
  public static void main(string[] args) {
    configurableapplicationcontext context= springapplication.run(gamepageprocessor.class, args);
     d = context.getbean(dianjingservice.class);
    //spider.create(new gamepageprocessor()).addurl("网址").thread(5).run();
  }
 
  public void process(page page) {
    selectable url = page.geturl();
    if (url.tostring().equals("网址")) {
      dianjingvideo dv = new dianjingvideo();
      list<string> ls = page.gethtml().xpath("//div[@class='v']/div[@class='v-meta va']/div[@class='v-meta-title']/a/text()").all();
      //hrefs
      list<string> ls1 = page.gethtml().xpath("//div[@class='v']/div[@class='v-link']/a/@href").all();//获取a标签的href
 
      list<string> ls2 = page.gethtml().xpath("//div[@class='v']/div[@class='v-meta va']/div[@class='v-meta-entry']/div[@class='v-meta-data']/span[@class='r']/text()").all();
      //photo
      list<string> ls3 = page.gethtml().xpath("//div[@class='v']/div[@class='v-thumb']/img/@src").all();
 
      for (int i = 0; i < 5; i++) {
        dv.settitles(ls.get(i));
        dv.setcategory("");
        dv.setdates(ls2.get(i));
        dv.sethrefs(ls1.get(i));
        dv.setphoto(ls3.get(i));
        dv.setsources("");
 
        d.addvideo(dv);
 
      }
    }
}

controller:

  1. @controller 
  2. @requestmapping(value = "/dianjing"
  3. public class dianjingcontroller { 
  4.   @autowired 
  5.   private dianjingservice s; 
  6.  
  7.  
  8.  
  9.  
  10.     /* 
  11.     手游 
  12.      */ 
  13.   @requestmapping("/dianjing"
  14.   @responsebody 
  15.   public object dianjing(){ 
  16.     list<dianjing> list = s.find2(); 
  17.     jsonobject jo = new jsonobject(); 
  18.     if(list!=null){ 
  19.  
  20.       jo.put("code",0); 
  21.       jo.put("success",true); 
  22.       jo.put("count",list.size()); 
  23.       jo.put("list",list); 
  24.     } 
  25.     return jo; 
  26.   } 

实体类就不展示了

dao层

?
1
2
@insert("insert into dianjing (titles,dates,category,hrefs,photo,sources) values(#{titles},#{dates},#{category},#{hrefs},#{photo},#{sources})")
int adddj(dianjing dj);

以上这篇springboot+webmagic实现java爬虫jdbc及mysql的方法就是小编分享给大家的全部内容了,希望能给大家一个参考,也希望大家多多支持服务器之家。

原文链接:https://www.cnblogs.com/NCL--/p/8608336.html