批量文本读取URL获取正常访问且保留对应IP

#coding=utf-8

import sys

import requests

for i in range(3000,4999,1):

  url = 'http://192.168.88.139:8888/20150602'+str(i)+'.html'

  r = requests.get(url)

  if r.status_code == 200:

    print url

    print r.content

原文

http://zone.wooyun.org/content/20885

多线程+文本逐行读取+URL的IP转换+写入

# -*-coding:utf-8-*-

import os

import sys

import Queue

import getopt

import logging

import requests

import threading

import time

import socket

print "start:" + (time.strftime("%H:%M:%S"))

logging.basicConfig(

    level=logging.WARNING,

    format="[%(asctime)s] %(message)s"

)

class BatchThreads(threading.Thread):

    def __init__(self, queue):

        super(BatchThreads, self).__init__()

        self.queue = queue

    def run(self):

        while True:

            if self.queue.empty():

                break

            else:

                tempurl = self.queue.get()

                #print tempurl

                try:

                    url = 'http://'+tempurl

                    #print url

                    r = requests.get(url, timeout=5)

                    if r.status_code == 200 :

                        print url+' '+'access-comman:200'

                        #print tempurl

                        ip = socket.gethostbyname(tempurl)

                        #print ip

                        yes = open('yes.txt','a')

                        yes.write(url+'    ')

                        yes.write('    '+ip+'\n')

                        yes.close()

                except:

                    pass

                    print url+" error"

                    noaccess = open('noaccess.txt','a')

                    noaccess.write(url+'\n')

                    noaccess.close()

def batch_queue(_queue, _thread_number):

    with open('url-hz.txt') as f:

        urls = [line.strip() for line in f.readlines()]

    urls = set(filter(lambda url: url and not url.startswith("#"), urls))

    if urls:

        for url in urls:

            queue.put(url)

        if _thread_number > (queue.qsize() / 2):

            _thread_number = (queue.qsize())

        for _ in xrange(_thread_number):

            threads.append(BatchThreads(_queue))

        for t in threads:

            t.start()

        for t in threads:

            t.join()

threads = []

queue = Queue.Queue()

thread_number = 20

batch_queue(queue, thread_number)

print"end:" + (time.strftime("%H:%M:%S"))