perl 爬取某理财网站产品信息

时间：2021-09-05 05:14:41
use  LWP::UserAgent;

use utf8;

use DBI;

$user="root";

$passwd="xxxxx";

$dbh="";

$dbh = DBI->connect("dbi:mysql:database=zjzc_vote;host=14.5.6.57;port=3306",$user,$passwd) or die "can't connect to  database ". DBI-errstr;

$dbh->do("SET NAMES utf8");

use POSIX;

use Data::Dumper;

use HTML::TreeBuilder;

open DATAFH,">data.html" || die "open data file failed:$!";

my $ua = LWP::UserAgent->new;

$ua->timeout(10);

$ua->env_proxy;

$ua->agent("Mozilla/8.0");

my $response = $ua->get('https://licai.yingyinglicai.com/product/list.htm');

if ($response->is_success) {

 print DATAFH  $response->decoded_content;  # or whatever

# print   $response->decoded_content;  # or whatever

  use HTML::TreeBuilder::XPath;

  my $tree= HTML::TreeBuilder::XPath->new;

  $tree->parse_file( "data.html");

my @arr1= $tree->find_by_tag_name("tr") ;

shift @arr1;

foreach my $row ( @arr1) {

   my @arr2= $row->content_list;

    my $str1= $arr2[0]->as_text;

    my $str2= $arr2[1]->as_text;

    my $str3= $arr2[2]->as_text;

    print $str1,$str2,$str3."\n";

     my $XDATE = strftime("%Y%m%d%H%M%S",localtime());

    my $sql = "SELECT count(*)   FROM p2p_product  where name='$str1' and cname='yylc'";

    my $sth = $dbh->prepare($sql);

    $sth->execute();

  my  $count = $sth->fetchrow_array();

  if ($count == 0 ){

   $dbh->do("insert into p2p_product  values ('$str1','$str2','$str3','$XDATE','yylc')") or $!;

                  }

    }}

相关文章