#!/usr/bin/env python
#-*-coding:utf-8-*-
#
# $Id: neighbours,v 0.18 2009/03/21 15:57:29 st3f Exp $
#
# This program is used to find the different web sites that are hosted
# on the same machine using virtual hosting. It is meant to be easy and
# fast to use while being as thourough as possible. The main source of
# information is Microsoft Live Search with its ip: operator.
#
# The script assumes that the addresses it recieves are to web servers.
# If you feed it mail.foobar.com it will output it as a web server even
# though nothing was found during Live Search.
#
# Be warned that this script parses data directly from the GUI of Live
# Search. I'm not sure if this is allowed so suit yourself if you get
# banned. Google is kind of twitchy in this respect, don't know about
# Live Search though.
#
# Copyright (c) Stefan Pettersson 2008-2009, http://www.bigpointyteeth.se/
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
# - fix the doublettes
# - add https:// only to the ones that actually run it, skip the rest!
# - show that not all pages were found when -p is too low
#   possibility to start at a specific page?
#

import re, os, sys
import optparse
import socket
import httplib
import time


################################################
#    functions
################################################

def live_search_ip(addr):
    """Take an host address 'addr' as input, query DNS for IP if necessary
    and contact Microsoft Live Search with the ip: operator, parse the
    results and return a list of the virtual host names."""

    if ipregex.match(addr):
        ip = addr
        # if we get an ip, leave results empty
        live_results = []
    else:
        verbose(3, "Resolving IP for '%s'" % addr)
        # we take the first ip address
        try:
            ip = socket.gethostbyname_ex(addr)[2][0]
        except socket.gaierror:
            error("cannot resolve '%s' to an IP address" % addr)
            return (-1, -1)
        # if we get a host name, add it to list of vhosts
        live_results = ["+ %s" % addr]
          
    # the raw html data recieved from live searches
    data = ""

    verbose(2, "Connecting to Live Search to do '%s'" % addr)

    h = httplib.HTTPConnection("search.live.com")
    # we need english so that the parsing will work properly
    hdr = {"User-agent":useragent,"Accept-Language":"en-us,en;q=0.5"}

    regex = re.compile(r'<li><cite>.*?</cite></li>')
    live_raw_results = []

    # loop through the results pages
    next_exists = True
    page = 0
    while next_exists and page < options.maxpages:
        next_exists = False
        verbose(2, "Getting page %d on Live Search" % page)
        url = "/results.aspx?q=ip%3A" + ip + "&first=" + str(10 * page + 1)
        verbose(3, "Search URL: %s" % url)
        h.request("GET", url, headers=hdr)
        data = h.getresponse().read()
        live_raw_results += regex.findall(data)
        
        # increment and determine if there are more pages
        if data.find("\">Next</a></li></ul>") > 0:
            page += 1
            next_exists = True

        time.sleep(2)
    h.close()

    verbose(1, "Got %d raw results from a %d-page search for 'ip:%s' (%s)" % (len(live_raw_results), page+1, ip, addr))

    for vhost in live_raw_results:
        # strip the characters that come with the regex
        name = vhost[10:-12]
        verbose(3, "Found URL %s" % name)

        # TODO maybe we want to keep the https:// part

        if name.startswith("https://"):
            name = name[8:]

        if "/" in name:
            name = name[:name.find("/")]

        # avoid saving duplicates
        if "- " + name not in live_results:
            verbose(2, "Added %s" % name)
            live_results.append("- %s" % name)

    return (ip, live_results)


def fatal(msg):
    print>>sys.stderr,"%s: %s" % (me, msg)
    sys.exit(1)


def error(msg):
    print>>sys.stderr,"%s: %s" % (me, msg)


def verbose(lvl, msg):
    """Take verbosity level 'lvl' and a message 'msg' as input. If the
    verbosity level is higher that the desired, 'verbosity', print the
    message to stdout."""
    if lvl <= options.verbosity:
        print msg


def print_version(option, opt, value, parser):
    print "%s %s" % (me, version)
    sys.exit(0)


################################################
#                 globals
################################################

me = os.path.basename(sys.argv[0])

version = "$Revision: 0.18 $"

ipregex = re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$")

useragent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)"

# list of host addresses to analyse
addresses = []

# dictionary that will hold the results
results = {}


################################################
#         parse command line options
################################################

usage = "(1) %prog [opts] <addr file>\n" +\
        "       (2) %prog [opts] <host addr>[,host addr]"

desc = "%prog is used to find the virtual hosts present on a web server. " +\
       "This is done by querying Microsoft Live Search. The positional " +\
       "argument can be (1) a file containing host addresses or (2) a " +\
       "comma-separated list of host addresses. For each vhost found, its " +\
       "IP address, host name and, if known, protocol is output."

optparser = optparse.OptionParser(description=desc, usage=usage)

optparser.add_option("-d", action="store_true", dest="debug", default=False,\
             help="print debug (and verbose) output")
optparser.add_option("-o", type="string", dest="outfile", metavar="FILE",\
             help="write output to FILE, verbose and debug output not included")
optparser.add_option("-p", type="int", dest="maxpages", metavar="PAGES", default=10,\
             help="never go beyond PAGES number of results pages (default: %default)")
optparser.add_option("-v", action="count", dest="verbosity", default=0,\
             help="print verbose output (can be used several times)")
optparser.add_option("-V", action="callback", callback=print_version,\
             help="print version information")

(options, args) = optparser.parse_args()



################################################
#    prepare address list according to mode
################################################

# only one positional argument is accepted
if len(args) == 1:
    # we fist try to open a file with args[0] as file name
    try:
        # this file contains a list of host names and ip addresses
        verbose(2, "Trying to open '%s' as a file" % args[0])
        inputfile = open(args[0], "rU")
        for line in inputfile.readlines():
            # skip comments and blank lines
            line = line.strip()
            if line == "" or line.startswith("#"):
                # skip it
                continue
            else:
                # add it
                addresses.append(line)
        inputfile.close()
    except IOError:
        # okay, it's not a file name
        verbose(2, "Failed to open '%s' as a file" % args[0])
        verbose(2, "Trying to use %s as a comma-separated list of addresses" % args[0])
        # we assume this is a comma-separated list of host addresses
        # split up the comma-separated list of host addresses
        addresses = args[0].split(",")
        verbose(2, "Found the following addresses:")
        for address in addresses:
            verbose(2, "  %s" % address)

else:
    # bad (number of) arguments
    optparser.print_help()
    sys.exit(-1)

# open handle for output file
if options.outfile:
    # die if output file already exists
    if os.access(options.outfile, os.F_OK):
        fatal("file '%s' already exists" % options.outfile)
    try:
        outputfile = open(options.outfile, "wt")
        verbose(1, "Saving output to '%s'" % options.outfile)
    except IOError:
        fatal("cannot open file '%s'" % options.outfile)
else:
    outputfile = None


################################################
#    analyse the addresses
################################################

verbose(1, "Doing queries for %s address(es)" % len(addresses))
for address in addresses:
    verbose(2, "  %s" % address)

# analyse addresses
for address in addresses:
    ip, vhosts = live_search_ip(address)
    if ip == -1:
        continue
    results[ip] = vhosts

# print output from analysis
for key in results.keys():
    if len(results[key]) > 0:
        for value in results[key]:
            print "%s\t%s" % (key, value)
        # if we want to save output to a file
        if options.outfile:
            for value in results[key]:
                outputfile.write("%s\t%s\n" % (key, value))

if options.outfile:
    outputfile.close()

# eof