用Bing API接口实现同IP域名查询(python)

在https://datamarket.azure.com/account/datasets这里注册帐号后申请一个免费的KEY,一个月可以查询5000次,其申请方法网上有很多教程了。

#!/usr/bin/python
 
import socket, sys, re, urllib2, StringIO, gzip, zlib
from bs4 import BeautifulSoup
import tldextract
 
if len(sys.argv) <> 2:
    print '\n[!] Two arguments required.'
    print 'Example: python neighbs.py www.website.com'
    print 'Example: python neighbs.py 1.2.3.4'
    sys.exit()
else:
    sharedHost = sys.argv[1]
    duplicateCheckList = []
 
def validateHostIP(target):
    isIP = re.match("^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$", target)
    isHostName = re.match("^(([a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9])$", target)
    if isIP:
        return 'IP'
    elif isHostName:
        return 'HOSTNAME'
    else:
        return False
 
def resolve(target):
    if hasattr(socket, 'setdefaulttimeout'):
        socket.setdefaulttimeout(3)
    try:
        peos = socket.gethostbyaddr(target)
        return peos[2][0]
    except:
        return False
 
def make_requests(sharedTarget):
    response = [None]
    responseText = None
 
    for requests in range (1, 101):
        if(request_www_bing_com(response, requests, sharedTarget)):
            responseText = read_response(response[0])
            soup = BeautifulSoup(responseText)
            for A in soup.find_all('a', href=True):
                domain = str('.'.join(list(tldextract.extract(A['href']))[:10]))
                if not domain.startswith('.') and not len(domain) < 4:
                    if domain not in duplicateCheckList:
                        if resolve(sharedHost) == resolve(domain):
                            duplicateCheckList.append(domain)
                            print '[+] '+domain
                domain = ''
            
            response[0].close()
 
def read_response(response):
    if response.info().get('Content-Encoding') == 'gzip':
        buf = StringIO.StringIO(response.read())
        return gzip.GzipFile(fileobj=buf).read()
 
    elif response.info().get('Content-Encoding') == 'deflate':
        decompress = zlib.decompressobj(-zlib.MAX_WBITS)
        inflated = decompress.decompress(response.read())
        inflated += decompress.flush()
        return inflated
 
    return response.read()
 
def request_www_bing_com(response, requests, sharedTarget):
    response[0] = None
    try:
        req = urllib2.Request("http://www.bing.com/search?q=ip%3A"+str(sharedTarget)+"&first="+str(requests))
 
        req.add_header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:32.0) Gecko/20100101 Firefox/32.0")
        req.add_header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
        req.add_header("Accept-Language", "en-US,en;q=0.5")
        req.add_header("Accept-Encoding", "gzip, deflate")
        req.add_header("Referer", "http://www.bing.com/")
        req.add_header("Cookie", "_EDGE_V=1; MUID=1A8733283AE36E853EB935873BA66FEF; SRCHD=AF=NOFORM; SRCHUID=V=2&GUID=DF63C8A66FB64714818DCF4DFC12DEE1; SRCHUSR=AUTOREDIR=0&GEOVAR=&DOB=20140908; MUIDB=1A8733283AE36E853EB935873BA66FEF; SRCHHPGUSR=CW=1280&CH=887; _RwBf=s=70&o=16; _HOP=; _SS=SID=BCC8BCA1D49C412281A16C56834A77B5&bIm=819187; SCRHDN=ASD=0&DURL=#; WLS=TS=63547515929")
        req.add_header("Connection", "keep-alive")
 
        response[0] = urllib2.urlopen(req)
 
    except urllib2.URLError, e:
        if not hasattr(e, "code"):
            return False
        response[0] = e
    except:
        return False
 
    return True
 
print '\n[*] Scanning for shared hosts. Please wait...'
print '------------------------------------------------'
if validateHostIP(sharedHost) == 'IP':
    make_requests(sharedHost)
elif validateHostIP(sharedHost) == 'HOSTNAME':
    make_requests(resolve(sharedHost))
else:
    print 'Something went wrong. Try again.'
print '------------------------------------------------'
print '[*] '+str(len(duplicateCheckList))+' unique domains found and verified to be on the same server.'

附另外一个查询脚本代码供学习、参考:此脚本使用Sqlite作数据库,申请的KEY可以在使用前保存在数据库中。使用方法看使用提示。

3

3、使用示例:

bing.py -add P1f3+ZAPBRv/wy7i3bxxxxxm2MXq3aA+iuu1UT6ypkY             //保存一个key到数据库中

bing.py -set 1                                                     //把ID为1的Key设为默认使用的KEY.

bing.py -url www.google.com                         //查询google.com同IP的其他域名

bing.py -url 192.168.1.1-192.168.1.254 -save       //查询指定IP段的域名,并保存到数据库中以便查询

bing.py -url -view                                               //浏览数据库中保存的数据

以下是Python代码

# -*- coding: utf-8 -*-
#https://datamarket.azure.com/account/datasets
#Code by Anle
import sys
import urllib  
import urllib2  
import base64
import sqlite3
import socket
import struct
import time
from datetime import datetime,timedelta
try:  
    import json  
except ImportError:  
    import simplejson as json
   
AccountKey=''  
top=100  
skip=0
formats='json'
useCounts = 0
useID = 0
ConfigFile='%s\\bing.db' % (sys.path[0])
 
def _usage():
    print 'Bing Domain dig Usage:'
    print '\t%s <options> [Host]' % sys.argv[0]
    print 'Options:\t'
    print '-url\t\tOnly view url.\t'
    print '-uri\t\tView url of full path.\t'
    print '-desc\t\tView description info.\t'
    print '-key\t\tView all account key.\t'
    print '-add <key>\tAdd a key to database.\t'
    print '-set <id> [num]\tSet a key to current use.\t'
    print '-del <id>\tDelete a key from database.\t'
    print '-view\t\tView the last saved record.\t'
    print '-save\t\tSave to databse.\t'
    print '-export <path>\tExport data to file.\n'
 
def CheckDatabaseExist():
    szSQL="SELECT name FROM sqlite_master WHERE type='table' order by name;"
    try:
        conn=sqlite3.connect(ConfigFile)
        cursor=conn.execute(szSQL)
    except sqlite3.Error,e:
        print '[-] ',sqlite3.Error,e
        return 0
    nTable=0
    for row in cursor:
        if row[0]=='AllKey' or row[0]=='Data':
            nTable += 1
    if nTable!=2:
        conn.execute("Create table AllKey(ID integer PRIMARY KEY autoincrement,bActive INT NOT NULL,KeyName TEXT UNIQUE NOT NULL,VisitMonth INT default 0,UseCount INT default 0);")
        conn.execute("Create table Data(ID integer PRIMARY KEY autoincrement,IP TEXT NOT NULL,URI TEXT NOT NULL,Title TEXT,Descript TEXT);")
    cursor.close()
    conn.close()
    return nTable==2
 
def GetAccountKey():
    conn = sqlite3.connect(ConfigFile)
    cf=conn.cursor()
    cursor = cf.execute("Select ID,KeyName,UseCount from AllKey where bActive=1")
    keys=cf.fetchone()
    if keys==None:
        keys=[0,'',0]
    cf.close()
    conn.close()
    return keys
 
def BuildHostRange(strHost):
    slash=[]                                      
    startIpStr=""
    endIpStr=""
    ranges=0
    submask=0
 
    realStartIP=0
    realEndIP=0
 
    if strHost.find('-')>0:
        slash = strHost.split('-')
    startIpStr=slash[0]
    endIpStr=slash[1]
    else:
    startIpStr=strHost
    try:
        startIpStr=socket.gethostbyname(startIpStr)
        if strHost.find('-')>0:
            realStartIP = socket.ntohl(struct.unpack('I',socket.inet_aton(startIpStr))[0])
            realEndIP = socket.ntohl(struct.unpack('I',socket.inet_aton(endIpStr))[0])
        else:
            realStartIP=realEndIP=socket.ntohl(struct.unpack('I',socket.inet_aton(startIpStr))[0])
    except:
        return [0,0]
 
    return [realStartIP,realEndIP]
 
def ViewResult(Keyword,curIP,nType,bSave):
    conn = sqlite3.connect(ConfigFile)
    #conn.text_factory=str
    ResData=BingSearch(Keyword)
    JsonData={}
    try:
        JsonData=json.loads(ResData)
    except:
        pass
    url=[]
    for key,value in JsonData.items():
        for key1,value1 in value.items():
            for lv in value1:
                try:
                    if nType==0:
                        uu=lv['Url']
                        i=uu.find('/',8)
                        url.append(uu[0:i])
                    elif nType==1:
                        print lv['Url']
                    elif nType==2:
                        print '%s -> %s' % (lv['Url'],lv['Title'])
                    if bSave:
                        szSQL="Insert into Data(IP,URI,Title,Descript) values ('%s','%s','%s','%s');" % (curIP,lv['Url'],lv['Title'],lv['Description'])
                        conn.execute(szSQL)
                        conn.commit()
                except:
                    pass
    if nType==0:
        for l in list(set(url)):
            print l
    print '\r\n'
    conn.close()
 
def ViewSaveData(nType):
    conn = sqlite3.connect(ConfigFile)
    conn.text_factory=str
    cf=conn.cursor()
    cf.execute("Select IP From DATA group by IP;")
    ipList=cf.fetchall()
 
    for IP in ipList:
        cf.execute("Select URI,Title,Descript from Data where IP='%s';" % (IP))
        dataList=cf.fetchall()
        i=0
        url=[]
        print '[%s]' % (IP)
        for row in dataList:
            i += 1
            if nType==0:
                uu=row[0]
                i=uu.find('/',8)
                url.append(uu[0:i])
            elif nType==1:
                print row[0]
            elif nType==2:
                print '%s -> %s' % (row[0],row[1])
        if i<=0:
            print '[-] No records.'
        elif nType==0:
            for l in list(set(url)):
                print l
    cf.close()
    conn.close()
 
def UpdateUseCount():
    global useCounts
    try:
        NowMonth=time.gmtime().tm_mon
        OldMonth=1
        conn = sqlite3.connect(ConfigFile)
        cf=conn.cursor()
        cursor = cf.execute("Select VisitMonth from AllKey where ID=%d" % useID)
        keys=cf.fetchone()
        if keys!=None:
            OldMonth=keys[0]
        cf.close()
        if NowMonth - OldMonth >= 1:
            useCounts = 1
        szSQL="Update AllKey set useCount=%d,VisitMonth=%d where ID=%d" % (useCounts,NowMonth,useID)
        cursor = conn.execute(szSQL)
        conn.commit()
    except sqlite3.Error,e:
        print '[-] ',e
    conn.close()
    
def BingSearch(query):
    global useCounts
    payload={}  
    payload['$top']=top  
    payload['$skip']=skip  
    payload['$format']=formats  
    payload['Query']="'"+query+"'"
    url='https://api.datamarket.azure.com/Bing/Search/Web?' + urllib.urlencode(payload)       
    sAuth='Basic '+base64.b64encode(':'+AccountKey)  
      
    headers = { }  
    headers['Authorization']= sAuth  
    try:  
        req = urllib2.Request(url,headers=headers)          
        response = urllib2.urlopen(req)
        the_page=response.read()
        useCounts += 1
        return the_page 
    except:
        print '[-] Connect failed.'
      
if __name__ == '__main__':
    argLen = len(sys.argv)
    if argLen < 2:
        _usage()
        sys.exit(0)
    arg=sys.argv
    if not CheckDatabaseExist():
        print '[-] Database is not exist, but would auto create.'
    
    nType=0
    strHost=""
    bSave=False
    bView=False
    bManualStr=False
    
    keyInfo=GetAccountKey()
    useID=keyInfo[0]
    AccountKey=keyInfo[1]
    useCounts=keyInfo[2]
    
    for x in range(1,argLen):
        if arg[x]=='-url' and argLen>2:
            nType=0
            if (x+1)<argLen:
                strHost=arg[x+1]
        if arg[x]=='-uri' and argLen>2:
            nType=1
            if (x+1)<argLen:
                strHost=arg[x+1]
        if arg[x]=='-desc' and argLen>2:
            nType=2
            if (x+1)<argLen:
                strHost=arg[x+1]
        if arg[x]=='-view':
            bView=True
        if arg[x]=='-key':
            conn = sqlite3.connect(ConfigFile)
            cf=conn.cursor()
            cf.execute("SELECT ID,bActive,KeyName,UseCount from AllKey;")
            keylist=[]
            keylist=cf.fetchall()
            i=0
            for row in keylist:
                xx=' '
                if row[1]==1:
                    xx='>'
                print '%s%-5d %s %5d' % (xx,row[0],row[2],row[3])
                i += 1
            if i<=0:
                print '[-] No records.'
            cf.close()
            conn.close()
            sys.exit(0)
        if arg[x]=='-add' and argLen>2:
            keystr=arg[x+1]
            if not keystr:
                print '[-] Please input a key value.\n'
                sys.exit(0)
            NowMonth=time.gmtime().tm_mon
            szSQL="Insert into AllKey(bActive,KeyName,VisitMonth,UseCount) values (%d,'%s',%d,%d);" % (0,keystr,NowMonth,1)
            conn = sqlite3.connect(ConfigFile)
            try:
                conn.execute(szSQL)
                conn.commit()
                conn.close()
                print '[+] Add a key records successfully.\r\n'
            except:
                print '[-] This record has exsits.'
            sys.exit(0)
        if arg[x]=='-set' and argLen>2:
            ID=int(arg[x+1])
            if argLen>3:
                useCounts=int(arg[x+2])
            if not ID:
                print '[-] ID value incorrect.'
                sys.exit()
            try:
                szSQL="Update AllKey set bActive=0;"
                conn = sqlite3.connect(ConfigFile)
                cursor = conn.execute(szSQL)
                conn.commit()
                szSQL="Update AllKey set bActive=1 where ID=%d" % (ID)
                cursor = conn.execute(szSQL)
                conn.commit()
                print '[+] Set success.'
            except:
                print '[-] Set failed.'
            if useCounts>0:
                UpdateUseCount()
            sys.exit(0)
        if arg[x]=='-del' and argLen>2:
            ID=int(arg[x+1])
            if not ID:
                print '[-] ID value incorrect.'
                sys.exit()
            try:
                szSQL = 'Delete from AllKey where ID=%d;' % (ID)
                conn = sqlite3.connect(ConfigFile)
                cursor = conn.execute(szSQL)
                conn.commit()
                print '[+] Delete success.'
            except:
                print '[-] Delete failed.'
            sys.exit(0)
        if arg[x]=='-save':
            bSave=True
        if arg[x]=='-api':
            bManualStr=True
        if arg[x]=='-export':
            pass
 
    if not strHost and bView==False:
        _usage()
        sys.exit()
 
    if useID==0:
        print '[-] Please set a active ID.'
        sys.exit()
 
    if bSave:
        conn = sqlite3.connect(ConfigFile)
        conn.execute("Delete from Data where ID>0;")
        conn.commit()
        conn.close()
 
    if bManualStr:
        ViewResult(strHost,strHost,nType,bSave)
        UpdateUseCount()
        sys.exit()
 
    if bView:
        ViewSaveData(nType)
        sys.exit()
        
    """uri, type, ID, Title, Description, DisplayUrl, Url"""
    IpRange=BuildHostRange(strHost)
    for index in range(IpRange[0],IpRange[1]+1):
        curIP=socket.inet_ntoa(struct.pack('I',socket.htonl(index)))
        SearchKeyWord='IP:' + curIP
        print '[%s]' % (curIP)
        ViewResult(SearchKeyWord,curIP,nType,bSave)
    UpdateUseCount()

文章转自91ri.org
安了的笔记–《用Bing API接口实现同IP域名查询(Python版) 》:http://blog.csdn.net/u013814153/article/details/26981523

发表评论

电子邮件地址不会被公开。 必填项已用*标注