[Python] Extract IP:Port from a proxy list

[Gonzalez]

#!/usr/bin/python
#
# linux ONLY
#
# ProxyHarvest.py v1.1
#
# REQUIREMENTS:
#   - GeoIP Database + GeoIP Python API
#   - sudo apt-get install libgeoip1 && sudo apt-get install python-geoip (ubuntu/debian)
#
# Extract IP:Port from a proxylist site code from low1z lurking at darkc0de.com
# this code is protected under the gpl get your copy at <http://www.gnu.org/licenses/>
#
# update from 0.9 - 1.1 notes
# - fetch planetlab(codeen) proxylist & clean our list with it
# - validate external ip with whatsmyip.com
# - GeoIP
#
# - !! due to urllib1/2 limitations there is no way yet to except username/passwd input !!

import sys, os, urllib, urllib2, re, httplib, sets, socket
from time import time, localtime, strftime
from socket import gethostbyaddr

nogeoip = 0
try:
   import GeoIP
except:
   nogeoip = 1
   print "\nGeoIP Module/Database NOT found, try:"
   print "sudo apt-get install libgeoip1 && sudo apt-get install python-geoip"
   print "or visit www[.]maxmind[.]com for download"
   print "GeoIP is not required but highly recommended!\n"

output = 'proxylist.txt'
sleeptimer = 3
socket.setdefaulttimeout(2)
alivelist = []
myipadress = urllib.urlopen('http://www.whatismyip.com/automation/n09230945.asp').read()
anon_list = []
trans_list = []
planetlab = []

sites = ['http://www.darkc0de.com/cgi-bin/proxies.py',
    'http://www.1proxyfree.com/',
    'http://www.atomintersoft.com/products/alive-proxy/socks5-list/',
    'http://www.proxylist.net/',
    'http://www.proxylists.net/http_highanon.txt']

def StripTags(text):
   return re.sub(r'<[^>]*?>','', text)

def timer():
   now = strftime('%H:%M:%S-%d/%b/%Y', localtime())
   return now

def ipcheck(proxy):
   try:
      pxhandle = urllib2.ProxyHandler({"http": proxy})
      opener = urllib2.build_opener(pxhandle)
      urllib2.install_opener(opener)
      myip = urllib2.urlopen('http://www.whatismyip.com/automation/n09230945.asp').read()
      xs =  re.findall(('\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}'), StripTags(myip))
      if xs[0] == myipadress or myipadress == myip:
         trans_list.append(proxy)
         print proxy[:-1],"\t- ALIVE -", timer(), "- TRANSPARENT"
      elif xs == None:
         pass
      else:
         anon_list.append(proxy)
         print proxy[:-1],"\t- ALIVE -", timer(), "- EXT-iP :",xs[0]
   except KeyboardInterrupt:
      print "\n\nCTRL+C - check temporary proxylist file\n\n"
      sys.exit(0)
   except:
      pass

def proxyvalidator(proxylist):
        finalcount = 0
   for proxy in proxylist:
      proxy.replace('\n', '')
      try:
         proxies = {'http': "http://"+proxy[:-1]}
         opener = urllib.FancyURLopener(proxies)
         try:
            loopchk = opener.open("http://www.google.com").read()
         except:
            pass
      except(IOError,socket.timeout), detail:
         pass
      ipcheck(proxy)      
      alivelist.append(proxy)
      finalcount += 1
   return alivelist

def getsamairdotru():
   counter = 1
   pxycnt = 0
   maxpages = 10
   urls = []
   pfile = file(output, 'a')
   while counter <= maxpages:
      if counter < 10: # workaround for page-01 to page-09
         opener = urllib2.build_opener()
         opener.addheaders = [('User-agent', 'Mozilla/5.0')]
         url = opener.open('http://www.samair.ru/proxy/proxy-0'+repr(counter)+'.htm').read()
      else:
         opener = urllib2.build_opener()
         opener.addheaders = [('User-agent', 'Mozilla/5.0')]
         url = opener.open('http://www.samair.ru/proxy/proxy-'+repr(counter)+'.htm').read()
      strings = re.findall(('\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}:\d{1,5}'), StripTags(url))
      for string in strings:
         pfile.write(string+"\n")
         pxycnt = pxycnt+1
      counter = counter+1      
      opener.close()
   print pxycnt, "\t: Proxies received from : http://www.samair.ru/proxy/"
   pfile.close()

def getsinglesitelist(site):
        pxycnt = 0
        urls = []
        pfile = file(output, 'a')
        opener = urllib2.build_opener()
        opener.addheaders = [('User-agent', 'Mozilla/5.0')]
        url = opener.open(site).read()
        strings = re.findall(('\d{1,3}[.]\d{1,3}[.]\d{1,3}[.]\d{1,3}[:]\d{1,5}'), StripTags(url))
        for string in strings:
      pfile.write(string+"\n")
                pxycnt = pxycnt+1
   print pxycnt, "\t: Proxies recieved from :", site.split("//",3)[1]
        opener.close()
        pfile.close()

def getplanetlabs():
   opener = urllib2.build_opener()
        url = opener.open('http://fall.cs.princeton.edu/codeen/tabulator.cgi?table=table_all').read()
   strings = re.findall(('\d{1,3}[.]\d{1,3}[.]\d{1,3}[.]\d{1,3}'), StripTags(url))
   for string in strings:
      planetlab.append(string)
   print len(planetlab), "\t: PlanetLab Proxylist Loaded", "\n"

def cleanup():
   pfile = open(output, 'r').readlines()
   outfile = file(output, 'w')
   sorted = []
   finalcount = 0
   psremove = 0
   for proxy in pfile:
      if proxy.split(':',1)[0] not in planetlab:
         if proxy not in sorted:
            sorted.append(proxy)
            outfile.write(proxy)
            finalcount += 1
      if proxy.split(':',1)[0] in planetlab:
         psremove += 1
   print "\n", psremove, "\t: PlanetLab (CoDeen) Proxies removed!"
   print finalcount,"\t: unique Proxies found\n"
   print "+-[Starting Validation]-----------------------------------------------------+"
   outfile.close()

def fileConst():
   fileC = open(output, 'w')
   falive = []
   fileC.write('+ This List has been generated with proxyharvest_1.1.py // www.darkc0de.com\n')
   fileC.write('+ ANONYMOUS PROXIES\n\n')
   for anon in anon_list:
      fileC.write(anon)
      if anon in alivelist:
         alivelist.remove(anon)
        fileC.write('\n\n+ TRANSPARENT PROXIES\n\n')
        for trans in trans_list:
                fileC.write(trans)
      if trans in alivelist:
         alivelist.remove(trans)
   fileC.write('\n\n+ WORKING BUT UNCLEAR PROXIES\n\n')
   alivelist.sort()
   for alive in alivelist:
      fileC.write(alive)
   fileC.close()   

def helpme():
   print "| -s  / -sitecollect   :: gathers proxylists    |"
   print "| -m  / -multipage     :: get incremental pages |"
   print "| -a  / -all           :: do ALL!!!             |"
   print "| -vl / - validatelist :: check a file          |"
   print "+-----------------------------------------------+"
try:
   os.remove(output)
except:
   pass
print "+-----------------------------------------------+"
print "|              ProxyHarvest.py 1.1              |"
print "|            low1z 2009 // darkc0de             |"
print "+-----------------------------------------------+"
print "IP:", myipadress, "//", timer(), "\n"
getplanetlabs()

if len(sys.argv) <= 1:
        print "\n\t < use -help to get options >\n"
        sys.exit(1)

for arg in sys.argv[1:]:
   if arg.lower() == "-h" or arg.lower() == "-help":
           helpme()      
   if arg.lower() == "-s" or arg.lower() == "-sitecollect":
      for site in sites:
         try:
                 getsinglesitelist(site)
         except:
            print "Error   :", site
      cleanup()
           proxylist = open(output, 'r').readlines()
      proxyvalidator(proxylist)
   if arg.lower() == "-m" or arg.lower() == "-multipage":
      getsamairdotru()
      cleanup()
      print "may take some time to print out good proxies, be patient"
      try:
              proxylist = open(output, 'r').readlines()
         proxyvalidator(proxylist)
      except:
         pass
   if arg.lower() == "-a" or arg.lower() == "-all":
      try:
                   for site in sites:
                           getsinglesitelist(site)
         getsamairdotru()
         cleanup()
         proxylist = open(output, 'r').readlines()      
         proxyvalidator(proxylist)
      except:
         print "something went wront... using -a is seems a bit buggy"
   if arg.lower() == "-vl" or arg.lower() == "-validatelist":
                try:
         proxyfile = open(sys.argv[2], 'r').readlines()
         proxyvalidator(proxyfile)
                except(IndexError):
                        print "Error: check you proxy file ...\n"
         sys.exit(0)

print "\n+-[ANON LIST]-------------------------------------------------------------+\n"
for anon_proxy in anon_list:
   try:
           haddr = gethostbyaddr(anon_proxy.split(':',1)[0])
   except:
      haddr = '-'
   if nogeoip == 1:
      print anon_proxy.replace('\n',''),"\t| HostAdress:", haddr[0]
      pass
   elif nogeoip == 0:
      gi = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
      gx = gi.country_code_by_addr(anon_proxy.split(':',1)[0])
      print anon_proxy.replace('\n',''), "\t| Country:", gx,"\t| HostAdress:", haddr[0]
print "\n\t", len(anon_list), ": Total tested AnonProxies\n"
print "+-[TRANS LIST]--------------------------------------------------------------+\n"
for trans_proxy in trans_list:
        if nogeoip == 1:
                print trans_proxy.replace('\n','')
                pass
        elif nogeoip == 0:
      gi = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
      gx = gi.country_code_by_addr(trans_proxy.split(':',1)[0])
      print trans_proxy.replace('\n',''), "\t| Country:", gx
print "\n\t", len(trans_list), ": Total tested Transparent Proxies\n"
print "+-[OTHER SERVERS]-----------------------------------------------------------+\n"
if len(alivelist) > 16:
   print len(alivelist), "Alive but unverified Servers, check", output
else:
   for alive in alivelist:
      if alive not in trans_list:
         if alive not in anon_list:
             print alive.replace('\n','')
fileConst()