• 设为首页
  • 点击收藏
  • 手机版
    手机扫一扫访问
    迪恩网络手机版
  • 关注官方公众号
    微信扫一扫关注
    迪恩网络公众号

[工具开发] Perl 爬虫脚本--从美国国家漏洞数据库抓取实时信息 ...

原作者: [db:作者] 来自: [db:来源] 收藏 邀请

一、简介

美国国家漏洞数据库收集了操作系统,应用软件的大量漏洞信息,当有新的漏洞出现时,它也会及时发布出来.

由于信息量巨大,用户每次都需要到它的网站进行搜索,比较麻烦.如果能有个工具,每天自动分析它发布的漏洞数据库,如果发现有所需要的新漏洞信息,通过邮件自动发送给公司的系统或者安全管理员就好了.

下面我写的这个工具就是起到这个作用的.图片是工具自动发送的邮件截图:

它每天都会根据用户设置的关键字自动抓取 NVD 数据,然后和前一天抓取的数据进行对比分析,当发现今天有新的数据时就发送邮件给用户,否则不发送.

 

二、效果截图

 

三、源代码

#!/usr/bin/perl -w
#[email protected]
use 5.10.1;
use strict;
use LWP::Simple;
use Net::SMTP;
use MIME::Base64;
use Encode qw/ decode encode /;
 
my $REC_DIR = \'/home/hupeng/nvd\';
my @query_keywords = qw/ kernel tomcat apache spring /;
 
my $TO_DAY = `date +%Y-%m-%d`;
my $LAST_DAY = `date +%Y-%m-%d -d \'-1 days\'`;
my $THIS_MONTH = `date +%m -d \'-1 months\'`;
my $NEXT_MONTH = `date +%m`;
my $THIS_YEAR = `date +%Y`;
my $NEXT_YEAR = `date +%Y -d \'+1 months\'`;
 
chomp($TO_DAY);
chomp($LAST_DAY);
chomp($THIS_MONTH);
chomp($THIS_YEAR);
chomp($NEXT_MONTH);
chomp($NEXT_YEAR);
$THIS_MONTH =~ s/^0+//g;
 
my $nvdfile_lastday = "$REC_DIR/nvd_$LAST_DAY.txt";
my $nvdfile_today = "$REC_DIR/nvd_$TO_DAY.txt";
 
my $nvd_url_pre = \'http://web.nvd.nist.gov/view/vuln/detail?vulnId=\';
#my $sev_base = \'MEDIUM_HIGH\';
 
my $theSmtpServer = \'XXXX\';
my $theSmtpUser = \'XXXX\';
my $theSmtpPasswd = \'XXXXX\';
my $theSmtpSend = \'XXXXX\';
my @theSmtpTo = (\'[email protected]\',\'[email protected]\');
my $theSmtpSubject = \'NVD 新记录 \'.$TO_DAY;
my $query_keywords_str = &arr2str0(@query_keywords);
my $theSmtpBody = \'<p>NVD 新记录</p><br><p>关键字:\'.$query_keywords_str.\'</p><br>\';
 
sub str2arr {
        my ($str) = @_;
        $str =~ s/^\n|\n$//g;
        my @arr = split /\n/,$str;
        @arr = sort(@arr);
    #@arr = keys %{{ map { $_ => 1 } @arr }};
        return @arr;
}
 
sub arr2str {
        my @arr = @_;
        my $str = \'\';
        @arr = sort(@arr);
        foreach(@arr){
                $str = $str.$_."\n";
        }
        return $str;
}
 
sub arr2str0 {
        my @arr = @_;
        my $str = \'\';
        @arr = sort(@arr);
        foreach(@arr){
                $str = $str.$_.\', \';
        }
    $str =~ s/,\ $//g;
        return $str;
}
 
sub getContent {
    my ($query_keywords) = @_;
    my @content = ();
 
    foreach my $query_keyword (@query_keywords){
        #my $url = "http://web.nvd.nist.gov/view/vuln/search-results?adv_search=true\&cves=on\&query=$query_keyword\&pub_date_start_month=$start_month\&pub_date_start_year=$start_year\&cvss_sev_base=$sev_base\&cve_id=";
        #my $url = "http://web.nvd.nist.gov/view/vuln/search-results?adv_search=true\&cves=on\&query=$query_keyword";
        my $url = "http://web.nvd.nist.gov/view/vuln/search-results?adv_search=true\&cves=on\&query=$query_keyword\&pub_date_start_month=$THIS_MONTH\&pub_date_start_year=$THIS_YEAR\&cve_id=";
 
        my $tmpStr = get($url);
        my @tmpArr = &str2arr($tmpStr);
        $tmpStr = \'\';
        foreach(@tmpArr){
            my $str = $_;
            chomp($str);
            $str =~ s/\s+//g;
            if( $str =~ m/BodyPlaceHolder_cplPageContent_plcZones_lt_zoneCenter_VulnerabilitySearchResults_VulnResultsRepeater_[\w]+(Anchor_.*$)/ ){
                push(@content,$query_keyword.$1."\n");
            }
        }
        @content = keys %{{ map { $_ => 1 } @content }};
        @content = sort(@content);
        @tmpArr = ();
    }
    return @content;
}
 
sub getNvd {
        my ($nvd_file) = @_;
    my $maxnvd = \'0\';
    my @nvds = ();
    my %result = (\'maxnvd\'=>\'0\',\'nvds\'=>[]);
    if( open(FILE, "$nvd_file") ){
        while(<FILE>){
            push(@nvds, $_);
        }
        close FILE;
        foreach(@nvds){
            if( $_ gt $maxnvd ){
                $maxnvd = $_;
            }
        }
    }
    $result{\'maxnvd\'} = $maxnvd;
    $result{\'nvds\'} = [@nvds];
    @nvds = ();
    return %result;
}
 
sub putNvd {
    my ($content,$nvd_file) = @_;
    if ( open(FILE, "> $nvd_file") ){
        foreach (@$content){
            if ($_ =~ m/[\w-]+Anchor_[\d]+">([\w-]+)<\/a>/){
                print FILE $1."\n";
            }
        }
        close FILE;
    }
}
 
sub getNewNvdRds {
    my ($maxNvd_lastday,$nvdsToday,$content) = @_;
    my @newNvds = ();
    foreach (@{$nvdsToday}){
        my $nvd = \'\';
        if( $_ gt $maxNvd_lastday){
            my $str = $_;
            chomp($str);
            foreach my $ln1 (@{$content}){
                if( $ln1 =~ m/^([\w-]+Anchor_[\d]+\">)$str<\/a>$/ ){
                    my $nvdID = $1;
                    foreach my $ln2 (@{$content}){
                        if( $ln2 =~ m/^$nvdID([\d.]+)<\/a>([\w]+)$/ ){
                            $nvd = \'<a href="\'.$nvd_url_pre.$str.\'">\'.$str.\'</a>  CVSS Severity:  \'.encode(\'UTF-8\',$1).\'  \'.encode(\'UTF-8\',$2).\'<br>\';
                        }
                    }
                }
            }
            push(@newNvds,$nvd);
        }
    }
    return @newNvds;
}
 
# get max value of last day
my %tmpHsh = ();
%tmpHsh = &getNvd($nvdfile_lastday);
my $maxNvd_lastday = $tmpHsh{\'maxnvd\'};
 
# get content of today
# nvd 记录的详细信息
my @content = &getContent(@query_keywords);
 
# put values of today
&putNvd([@content],$nvdfile_today);
 
# get max value of today
%tmpHsh = &getNvd($nvdfile_today);
my $maxNvd_today = $tmpHsh{\'maxnvd\'};
 
# get all values of today
my @nvdsToday = @{$tmpHsh{\'nvds\'}};
 
%tmpHsh = ();
 
# find new values
# 排版后新记录的详细信息
my @newNvdRds = &getNewNvdRds($maxNvd_lastday,[@nvdsToday],[@content]);
 
# send email
my $count = @newNvdRds;
if( $count ){
    $theSmtpBody .= &arr2str(@newNvdRds);
    $theSmtpBody .= \'<br><br>\'.$TO_DAY.\'<br><br>\';
 
    my $theSmtp = Net::SMTP->new($theSmtpServer,Timeout=>10);
    $theSmtp->auth($theSmtpUser,$theSmtpPasswd);
    $theSmtp->mail($theSmtpSend);
    $theSmtp->to(@theSmtpTo);
    $theSmtp->data();
    $theSmtp->datasend("To: @theSmtpTo\n");
    $theSmtp->datasend("Content-Type:text/html;charset=UTF-8\n");
    $theSmtp->datasend("Subject:=?UTF-8?B?".encode_base64($theSmtpSubject, \'\')."?=\n\n");
    $theSmtp->datasend("\n");
    $theSmtp->datasend($theSmtpBody);
    $theSmtp->dataend();
    $theSmtp->quit;
}

 


鲜花

握手

雷人

路过

鸡蛋
该文章已有0人参与评论

请发表评论

全部评论

专题导读
上一篇:
perl学习笔记发布时间:2022-07-22
下一篇:
序言:我为什么学Perl发布时间:2022-07-22
热门推荐
热门话题
阅读排行榜

扫描微信二维码

查看手机版网站

随时了解更新最新资讯

139-2527-9053

在线客服(服务时间 9:00~18:00)

在线QQ客服
地址:深圳市南山区西丽大学城创智工业园
电邮:jeky_zhao#qq.com
移动电话:139-2527-9053

Powered by 互联科技 X3.4© 2001-2213 极客世界.|Sitemap