web.py

import re
import random
import time
import cookielib
import urllib2
import urllib
import mimetypes
import gzip
import StringIO
import urlparse
import collections
import json
import csv
import os
import multiprocessing
import httplib
import copy
import inspect
import Queue
import tempfile
import subprocess
import sys
import functools

import greenlet
import gevent
from gevent import monkey
from gevent import queue
from gevent import select
from gevent import pool
monkey.patch_all(thread=False)

from lxml import etree
import pybloom

from urllib import quote_plus

DBC_USERNAME = None
DBC_PASSWORD = None

EXCLUDED_LINK_EXTENSIONS = ('jpg', 'gif', 'jpeg','pdf', 'doc', 'docx', 'ppt', 'txt', 'png', 'zip', 'rar', 'mp3')

def unique_domains_filter(iterable):
	domains = set()
	for i in iterable:
		parsed = urlparse.urlparse(i.strip())
		if parsed.netloc not in domains:
			domains.add(parsed.netloc)
			yield i.strip()

class BloomFilter(object):
	def __init__(self, name=None):
		if name and not name.endswith('.bloom'):
			name += '.bloom'
		self.name = name
		self.add_counter = 0
		try:
			self.bloom = pybloom.ScalableBloomFilter.fromfile(open(self.name, 'rb'))
		except:
			self.bloom = pybloom.ScalableBloomFilter(initial_capacity=100, error_rate=0.001, mode=pybloom.ScalableBloomFilter.SMALL_SET_GROWTH)
		
	def save(self):
		if self.name:
			self.bloom.tofile(open(self.name+'.bloom', 'wb'))

	def __del__(self):
		self.save()

	def add(self, key):
		self.bloom.add(key)
		self.add_counter += 1
		if len(self) / self.add_counter > 10 and self.add_counter > 100:
			self.save()
			self.add_counter = 0

	def __contains__(self, key, autoadd=True):
		result = key in self.bloom
		if autoadd:
			self.add(key)
		return result

	@property
	def count(self):
		return len(self.bloom)

	def __len__(self):
		return len(self.bloom)

class RandomLines(object):
	def __init__(self, input_file, cache_index=True, repetitions=1):
		if isinstance(input_file, basestring):
			self.source_file = open(input_file,'rb')
			self.filename = input_file
		else:
			self.source_file = input_file
			self.filename = input_file.name
		self.index = []
		self.cache_index = cache_index

		if not os.path.isfile(self.filename+'.lineindex'):
			self.index_file()
		else:
			for line_counter, line in enumerate(open(self.filename+'.lineindex')):
				line = line.strip()
				if line_counter == 0:
					if int(line) != os.path.getsize(self.filename):
						self.index_file()
						break
				elif len(line):
					self.index.append(int(line))
		self.index *= repetitions
		self.start_index_len = len(self.index)

	def __iter__(self):
		return self

	def __len__(self):
		return len(self.index)

	def index_file(self):
		bytes_counter = 0
		for line in self.source_file:
			bytes_counter += len(line)
			if len(line.strip()):
				self.index.append(bytes_counter-len(line))
		if self.cache_index:
			open(self.filename+'.lineindex','w').write('\n'.join(str(i) for i in [os.path.getsize(self.filename)] + self.index))		

	def next(self):
		while len(self.index):
			offset = self.index.pop(random.randrange(0, len(self.index)))
			self.source_file.seek(offset, 0)
			return self.source_file.readline().strip()
		raise StopIteration

	def percentage(self):
		if len(self.index) == 0:
			return 100
		else:
			return 100 - int((float(len(self.index)) / self.start_index_len) * 100) #this is buggy

def spin(text_input, unique_choices=False):
	seen_fields = {}
	if text_input.count('{') - text_input.count('}') == 1:
		text_input += '}'
	for _ in range(text_input.count('{')):
		field = re.search('{([^{}]*)}', text_input).group(0)

		if unique_choices:
			if field not in seen_fields:
				seen_fields[field] = field.split('|')
			if len(seen_fields[field]):
				replacement = seen_fields[field].pop(random.randint(0,len(seen_fields[field])))
			else:
				replacement = ''
		else:
			replacement = random.choice(field[1:-1].split('|'))
		text_input = text_input.replace(field, replacement, 1)
	return text_input

class HTTPResponse(object):
	def __init__(self, response=None, url=None, fake=False, http=None):
		self._xpath = None
		self._json = None
		#self._encoded_data = None #might cache encoded data again in future, for now don't see the point
		if fake:
			self.original_domain = urlparse.urlparse(url).netloc.lower()
			self.original_url = url
			self.final_url = url
			self.final_domain = self.original_domain
			self._data = '<html><body><p>Hello!</p></body></html>'
		else:
			self.headers = response.info()
			compressed_data = response.read()
			if filter(lambda (k,v): k.lower() == 'content-encoding' and v.lower() == 'gzip', self.headers.items()):
				self.headers['Content-type'] = 'text/html; charset=utf-8'
				self._data = gzip.GzipFile(fileobj=StringIO.StringIO(compressed_data)).read()
			else:
				self._data = compressed_data
			
			self.original_domain = urlparse.urlparse(url).netloc.lower()
			self.original_url = url
			self.final_url = response.geturl()
			self.final_domain = urlparse.urlparse(self.final_url).netloc.lower()

		if http:
			self.http = http

	def encoded_data(self):
		return unicode(self._data,'ISO-8859-1').encode('ISO-8859-1')
		
	def __str__(self):
		return self._data
		
	def __len__(self):
		return len(str(self))

	def __contains__(self,x):
		return x.lower() in str(self).lower()
		
	def save(self, handle):
		if isinstance(handle, basestring):
			handle = open(handle, 'w')
		handle.write(str(self))
		handle.close()

	def json(self):
		if not self._json:
			self._json = json.loads(self._data)
		return self._json	
		
	def xpath(self,expression, xml=False):
		if self._xpath is None:
			if xml:
				self._xpath = etree.XML(self.encoded_data())
			else:
				self._xpath = etree.HTML(self.encoded_data())
			if self._xpath is None:
				return []

		if not isinstance(expression,basestring):
			expression = '||'.join(expression)
		if '||' in expression:
			results = []
			for part in expression.split('||'):
				results.append(self.xpath(part))
			return zip(*results)

		results = []
		original_expression = expression
		if expression.endswith('/string()'):
			expression = expression.split('/string()')[0]
		with gevent.Timeout(30, False):	
			xpath_result = self._xpath.xpath(expression)
		if isinstance(xpath_result, basestring) or not isinstance(xpath_result, collections.Iterable):
			return xpath_result
		for result in xpath_result:
			if expression.endswith('@href') or expression.endswith('@src') or expression.endswith('@action'):
				if not result.startswith('http'):
					result = urlparse.urljoin(self.final_url,result)
				result = result.split('#')[0]
			if original_expression.endswith('/string()'):
				result = result.xpath('string()')
			if isinstance(result,basestring) and len(result.strip()):
					results.append(result.strip())
			else:
				results.append(result)
		return list(results)
				
		
	def single_xpath(self,expression):
		results = self.xpath(expression)
		if isinstance(results,basestring) or not isinstance(results,collections.Iterable):
			return results
		if results:
			return results[0]
		else:
			return ''

	def links(self):
		return {link.split('#')[0] for link in self.xpath('//a/@href')}
			
	def internal_links(self):
		return {link for link in self.links() if urlparse.urlparse(link).netloc.lower() == self.final_domain if not link.split('.')[-1].lower() in EXCLUDED_LINK_EXTENSIONS}
		
	def external_links(self, exclude_subdomains=True):
		if exclude_subdomains:
			return {link for link in self.links() if max(self.final_domain.split('.'), key=len) not in urlparse.urlparse(link).netloc and link.lower().startswith('http') and link.lower().split('.')[-1] not in EXCLUDED_LINK_EXTENSIONS}
		else:
			return {link for link in self.links() if urlparse.urlparse(link).netloc != self.final_domain and link.lower().startswith('http') and link.lower().split('.')[-1] not in EXCLUDED_LINK_EXTENSIONS}
		
	def dofollow_links(self):
		return set(self.xpath('//a[@rel!="nofollow" or not(@rel)]/@href'))
	
	def nofollow_links(self):
		return set(self.xpath('//a[@rel="nofollow"]/@href'))
		
	def external_images(self):
		return set([image for image in self.xpath('//img/@src') if urlparse.urlparse(image).netloc != self._domain])

	def csv(self):
		return csv.reader(self.encoded_data())

	def regex(self,expression):
		if not isinstance(expression,basestring):
			expression = '||'.join(expression)
		if '||' in expression:
			results = []
			for part in expression.split('||'):
				results.append(self.regex(part))
			return zip(*results)
		return re.compile(expression,re.S|re.I).findall(self.encoded_data())

	def url_regex(self,expression):
		if not isinstance(expression,basestring):
			expression = '||'.join(expression)
		if '||' in expression:
			results = []
			for part in expression.split('||'):
				results.append(self.xpath(part))
			return zip(*results)
		return re.compile(expression).findall(self.final_url)
		
	def __repr__(self):
		return '<HTTPResponse for %s>' % self.final_url
		
	def link_with_url(self, link, domain=False):
		if not isinstance(link, basestring):
			for l in links:
				result = self.link_with_url(l, domain=domain)
				if result is not False:
					return result
		if domain:
			link = urlparse.urlparse(link).netloc
		for l, l_obj in self.xpath('//a/@href||//a[@href]'):
			if domain:
				if urlparse.urlparse(l).netloc == link:
					return l_obj
			else:
				if link in (l,l+'/',l.rstrip('/')):
					return l_obj
		return False

	def link_with_anchor(self, anchor):
		if not isinstance(anchor, basestring):
			for a in anchor:
				result = self.link_with_anchor(a, domain=domain)
				if result is not False:
					return result
		results = self.xpath('//a[text()="%s"]' % anchor)
		if len(results):
			return results[0]
		return False

	def image_captcha(self,xpath):
		try:
			from captcha import DBC_USERNAME, DBC_PASSWORD
		except:
			pass
		image_source = self.single_xpath(xpath)
		if image_source:
			image = grab(image_source, http_obj=self.http)
			import deathbycaptcha
			result = deathbycaptcha.HttpClient(DBC_USERNAME, DBC_PASSWORD).decode(StringIO.StringIO(str(image)))
			if result:
				return result['text']

	def recaptcha(self):
		iframe_source = self.single_xpath('//iframe[contains(@src,"recaptcha")]/@src')
		if iframe_source:
			iframe = grab(iframe_source,http_obj=self.http,ref=self.final_url)
			return (iframe.single_xpath('//input[@id="recaptcha_challenge_field"]/@value'),iframe.image_captcha('//center/img/@src'))

	def solvemedia(self):
		iframe_source = self.single_xpath('//iframe[contains(@src, "api.solvemedia.com")]/@src')
		if iframe_source:
			iframe = grab(iframe_source,http_obj=self.http,ref=self.final_url)
			response = iframe.image_captcha('//img[@id="adcopy-puzzle-image"]/@src')

			post = iframe.hidden_fields()
			post['adcopy_response'] = response

			submit_iframe = grab('http://api.solvemedia.com/papi/verify.noscript', http_obj=self.http, ref=iframe_source, post=post)

			if submit_iframe:
				if len(submit_iframe.regex('c=(.+?)"')):
					return (response, submit_iframe.regex('c=(.+?)"')[0])
				else:
					return ('', '')
			else:
				return ('', '')

	def hidden_fields(self):
		fields = {}
		for name, value in self.xpath('//input[@type="hidden"]/@name||//input[@type="hidden"]/@value'):
			fields[name] = value
		return fields

	def view(self):
		p = tempfile.mktemp() + '.html'
		self.save(p)
		if sys.platform == 'darwin':      subprocess.call(('open', p))
		elif sys.platform == 'nt':     os.startfile(p) #duno lol
		elif sys.platform.startswith('linux'):  subprocess.call(('xdg-open', p))

class ProxyManager(object):
	def __init__(self, proxy=True, min_delay=20, max_delay=None):
		if isinstance(proxy,list):
			proxies = proxy
		elif proxy == True:
			try:
				proxies = open('proxies.txt').read().strip().split('\n')
			except:
				proxies = [None]
		elif isinstance(proxy, basestring):
			if proxy.startswith('http'):
				proxies = [p.strip() for p in str(grab(proxy)).strip().split('\n') if len(p.strip())]
			elif os.path.isfile(proxy):
				proxies = [p.strip() for p in open(proxy) if len(p.strip())]
			elif ':' in proxy:
				proxies = proxy.strip().split('\n')
			new_proxies = []
			for proxy in proxies:
				if proxy.count(':') == 3:
					ip, port, username, password = proxy.split(':')
					proxy = username+':'+password+'@'+ip+':'+port
				new_proxies.append(proxy)
			proxies = new_proxies
		elif isinstance(proxy, ProxyManager):
			proxies = proxy.records.keys()
		else:
			proxies = [None]
			
		self.records = dict(zip(proxies,[0 for p in proxies]))
		self.min_delay = min_delay
		self.max_delay = max_delay or min_delay
		
	def get(self):
		while True:
			proxies = [proxy for proxy, proxy_time in self.records.items() if proxy_time + random.randint(self.min_delay, self.max_delay) < time.time()]
			if not proxies:
				gevent.sleep(time.time() - min(self.records.values()))
			else:
				proxy = random.sample(proxies, 1)[0]
				self.records[proxy] = int(time.time())
				return proxy

	def __len__(self):
		return len(self.records)

	def split(self, number):
		chunk_size = len(self) / number
		managers = []
		for i in range(number):
			if len(self) % chunk_size >= number - i:
				managers.append(ProxyManager(self.records.keys()[chunk_size*i:chunk_size*(i+1)+1], min_delay=self.min_delay, max_delay=self.max_delay))
			else:
				managers.append(ProxyManager(self.records.keys()[chunk_size*i:chunk_size*(i+1)], min_delay=self.min_delay, max_delay=self.max_delay))
		return managers

class RedisProxyManager(ProxyManager):
	def __init__(self, name, proxy=True, min_delay=20, max_delay=None, host='localhost', port=6379):
		ProxyManager.__init__(self, proxy=proxy)
		import redis
		self.r = redis.Redis(host=host, port=port)
		self.name = name
		for record in self.records:
			if self.r.zrank(self.name, record) == None:
				self.r.zadd(self.name, record, 0)
		self.cache = []
		self.caching = False
		self.fill_cache()

	def fill_cache(self):
		while True:
			if len(self.cache) != 0:
				return
			if self.caching:
				gevent.sleep(1)
			else:
				self.caching = True
				while True:
					proxies = self.r.zrangebyscore(self.name, 0, int(time.time()), start=1, num=max(10, len(self.records) / 5))
					if proxies != 0 and len(proxies) != 0:
						print 'filling the proxy cache', len(proxies)
						for proxy in proxies:
							self.r.zadd(self.name, proxy, int(time.time()) + 300) #Yours for 5 minutes
						self.cache = proxies
						self.caching = False
						return
					else:
						gevent.sleep(1)

	def get(self):
			if not len(self.cache):
				self.fill_cache()
			proxy = self.cache.pop()
			self.last_proxy = proxy
			self.r.zadd(self.name, proxy, int(time.time()) + random.randint(self.min_delay, self.max_delay)) #about to be used, so set a normal time on it
			return proxy

	def available(self):
		return len(self.r.zrangebyscore(self.name, 0, int(time.time()))) + len(self.cache)

	def __len__(self):
		return self.r.zcard(self.name)

		
class HeadRequest(urllib2.Request):
	def get_method(self):
		return 'HEAD'

def useragent():
	agents = ('Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6','Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)','Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)','Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)','Mozilla/5.0 (X11; Arch Linux i686; rv:2.0) Gecko/20110321 Firefox/4.0','Mozilla/5.0 (Windows; U; Windows NT 6.1; ru; rv:1.9.2.3) Gecko/20100401 Firefox/4.0 (.NET CLR 3.5.30729)','Mozilla/5.0 (Windows NT 6.1; rv:2.0) Gecko/20110319 Firefox/4.0','Mozilla/5.0 (Windows NT 6.1; rv:1.9) Gecko/20100101 Firefox/4.0','Opera/9.20 (Windows NT 6.0; U; en)','Opera/9.00 (Windows NT 5.1; U; en)','Opera/9.64(Windows NT 5.1; U; en) Presto/2.1.1')
	return random.choice(agents)
	
def encode_multipart_formdata(fields, files):
	'''
	fields is a sequence of (name, value) elements for regular form fields.
	files is a sequence of (name, filename, value) elements for data to be uploaded as files
	Return (content_type, body) ready for httplib.HTTP instance
	'''
	BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
	CRLF = '\r\n'
	L = []
	for (key, value) in fields:
		L.append('--' + BOUNDARY)
		L.append('Content-Disposition: form-data; name="%s"' % key)
		L.append('')
		L.append(value)
	for (key, filename, value) in files:
		L.append('--' + BOUNDARY)
		L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
		L.append('Content-Type: %s' % get_content_type(filename))
		L.append('')
		L.append(value)
	L.append('--' + BOUNDARY + '--')
	L.append('')
	body = CRLF.join(L)
	content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
	return content_type, body

def get_content_type(filename):
	return mimetypes.guess_type(filename)[0] or 'application/octet-stream'

class DisabledHTTPRedirectHandler(urllib2.HTTPRedirectHandler):
	def redirect_request(self, req, fp, code, msg, headers, newurl):
		req.get_full_url()
		raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp)

class http(object):
	def __init__(self, proxy=None, cookie_filename=None, cookies=True, redirects=True):
		self.handlers = set()
		try:
			useragents = [ua.strip() for ua in open('useragents.txt') if len(ua.strip())]
			self.useragent = random.choice(useragents).strip()
		except:
			self.useragent = useragent()
			
		self.opener = urllib2.OpenerDirector()
		
		if cookies:
			self.cookie_jar = cookielib.LWPCookieJar()
			if cookie_filename:
				self.cookie_jar = cookielib.MozillaCookieJar(cookie_filename)
				self.cookie_jar.load()
			cookie_support = urllib2.HTTPCookieProcessor(self.cookie_jar)
		else:
			cookie_support = None
			
		self.proxy = False
		proxy_auth = None
		
		if proxy:
			if isinstance(proxy, ProxyManager):
				self.proxy = proxy.get()
			else:
				self.proxy = ProxyManager(proxy).get()
			#print 'proxy in http = ', self.proxy

		if self.proxy:
			self.proxy = self.proxy.strip()
			proxy_support = urllib2.ProxyHandler({'http' : self.proxy,'https':self.proxy})
			if '@' in self.proxy:
				proxy_auth = urllib2.HTTPBasicAuthHandler()
			else:
				proxy_auth = None
		else:
			proxy_support = None

		if not redirects:
			self.build_opener(DisabledHTTPRedirectHandler())

		self.build_opener(proxy_support,cookie_support,proxy_auth)
			
	def build_opener(self,*handlers):
		self.handlers |= set([handler for handler in handlers if handler is not None])
		self.opener = urllib2.build_opener(*self.handlers)

	def urlopen(self, url, post=None, ref=None, files=None, username=None, password=None, compress=True, head=False, timeout=30):
		assert url.lower().startswith('http')
		if isinstance(post, basestring):
			post = dict([part.split('=') for part in post.strip().split('&')])
		if post:
			for k, v in post.items():
				post[k] = spin(unicode(v).encode('utf-8'))
		if username and password:
			password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
			password_manager.add_password(None, url, username, password)
			password_auth = urllib2.HTTPBasicAuthHandler(password_manager)
			self.build_opener(password_auth)
		urllib2.install_opener(self.opener)
		if compress:
			headers = {'User-Agent' : self.useragent, 'Accept-encoding' : 'gzip'}
		else:
			headers = {'User-Agent' : self.useragent}
		if ref:
			 headers['Referer'] = ref
		if files:
			content_type,post = encode_multipart_formdata(post.items(), files)
			headers['content-type'] = content_type
			headers['content-length'] = str(len(post))
		elif post:
			post = urllib.urlencode(post)
		if head:
			req = HeadRequest(url, post, headers)
		else:
			req = urllib2.Request(url, post, headers)
		with gevent.Timeout(timeout):
			response = urllib2.urlopen(req)
			return HTTPResponse(response, url, http=self)
		
def grab(url, proxy=None, post=None, ref=None, compress=True, retries=1, http_obj=None, cookies=False, redirects=True, timeout=30):
	data = None
	if retries < 1:
		retries = 1
	for i in range(retries):
		if not http_obj:
			http_obj = http(proxy, cookies=cookies, redirects=redirects)
		try:
			data = http_obj.urlopen(url=url, post=post, ref=ref, compress=compress, timeout=timeout)
			break
		except urllib2.HTTPError, e:
			if str(e.code).startswith('3') and not redirects:
				data = HTTPResponse(url=url, fake=True)
				break
		except:
			pass
	if data:
		return data
	return False

class RedisQueue(object):
	def __init__(self, name, host='localhost', port=6379):
		import redis
		self.r = redis.Redis(host=host, port=port)
		self.name = name

	def put(self, item):
		self.r.sadd(self.name, item)

	def get(self, timeout=60):
		timeout_counter = 0
		while True:
			result = self.r.spop(self.name)
			if result == None:
				if timeout == timeout_counter:
					return None
				gevent.sleep(1)
				timeout_counter += 1
			else:
				return result

	def get_nowait(self):
		return self.get()

	def __len__(self):
		return self.r.scard(self.name)

	def empty(self):
		return len(self) == 0

def WebQueue(iterator=None):
	queue = Queue.Queue()
	if iterator:
		[queue.put(item) for item in iterator]
	return queue

def generic_iterator(iterator):
	if isinstance(iterator, basestring):
		if '\n' in iterator:
			for i in iterator.split('\n'):
				if len(i.strip()):
					yield i.strip()
		else:
			yield iterator.strip()
	else:
		for i in iterator:
			yield i

class DomainQueue(object):
	def __init__(self, urls):
		self.domains = collections.defaultdict(list)
		for url in urls:
			if isinstance(url, basestring):
				url = urlparse.urlparse(url)
			self.domains[url.netloc].append(url.geturl())
		self.counter = {domain:0 for domain in self.domains.keys()}

	def empty(self):
		return len(self.domains) == 0

	def get_nowait(self):
		domain = min(self.counter, key=self.counter.get)
		url = self.domains[domain].pop()
		if len(self.domains[domain]) == 0:
			del(self.domains[domain])
			del(self.counter[domain])
		else:
			self.counter[domain] += 1
		return url

	def get(self):
		return self.get_nowait()

	def put(self, url):
		if isinstance(url, basestring):
			url = urlparse.urlparse(url)
		self.domains[url.netloc].append(url.geturl())
		if url.netloc not in self.counter:
			self.counter[url.netloc] = 0

	def __len__(self):
		return sum((len(d) for d in self.domains.values()))

def multi_grab(urls, pool_size=100, timeout=30, max_pages=-1, queuify=True, proxy=None):
	if queuify:
		in_q = WebQueue(generic_iterator(urls))
	else:
		in_q = urls
	for result_counter, result in enumerate(pooler(grab, in_q, pool_size=pool_size, timeout=timeout, proxy=proxy)):
		yield result
		if result_counter == max_pages and max_pages > 0:
			break

def domain_crawl(urls, pool_size=100, timeout=30, max_pages=-1, link_filter=None, max_domain_pages=-1):
	urls = {url for url in generic_iterator(urls)}
	domains = {urlparse.urlparse(url).netloc for url in urls}
	domain_counter = collections.Counter()
	seen_urls = set(urls)
	while True:
		if not len(urls):
			break
		urls_queue = Queue.Queue()
		[urls_queue.put(url) for url in urls]
		urls = set()
		for page_counter, page in enumerate(multi_grab(urls_queue, pool_size, timeout, queuify=False)):
			if page.final_domain in domains:
				domain_counter[page.final_domain] += 1
				if not (domain_counter[page.final_domain] > max_domain_pages and max_domain_pages > 0):
					try:
						new_urls = {url for url in page.internal_links() if url not in seen_urls}
						if callable(link_filter):
							new_urls = {url for url in new_urls if link_filter(url)}
						urls |= new_urls
						seen_urls |= new_urls
					except:
						pass
				if max_pages > 0 and page_counter > max_pages:
					break
				yield page


def redirecturl(url, proxy=None):
	return http(proxy).urlopen(url, head=True).geturl()

def cloud_pooler(func, in_q, chunk_size=1000, _env='python-web', _type='c2', _max_runtime=60, get_results=True, **kwargs):
	import cloud
	if chunk_size > 1:
		if isinstance(in_q, collections.Iterable):
			in_q = WebQueue(in_q)
		chunks = []
		chunk = []
		while not in_q.empty():
			chunk.append(in_q.get())
			if len(chunk) == chunk_size:
				chunks.append(chunk)
				chunk = []
		if len(chunk):
			chunks.append(chunk)
	else:
		chunks = in_q

	partial_func = functools.partial(func, **kwargs)
	jids = cloud.map(partial_func, chunks, _env=_env, _type=_type, _max_runtime=_max_runtime)

	if get_results:
		print jids
		for result in cloud.iresult(jids, ignore_errors=True):
			if result:
				yield result
	else:
		for jid in jids:
			yield jid

def pooler(func, in_q, pool_size=100, proxy=False, max_results=0, **kwargs):
	if isinstance(in_q, collections.Iterable):
		in_q = WebQueue(in_q)
	out_q = multiprocessing.Queue()
	if proxy and not isinstance(proxy, ProxyManager):
		proxy = ProxyManager(proxy)

	p = pool.Pool(pool_size)
	greenlets = set()
	if proxy:
		kwargs['proxy'] = proxy
	result_counter = 0
	while True:
		#print len(greenlets), 'greenlets'
		finished_greenlets = {g for g in greenlets if g.value != None}
		greenlets -= finished_greenlets
		for g in finished_greenlets:
			if g.value != False:
				yield g.value
				result_counter += 1
		if max_results > 0 and result_counter >= max_results:
			break
		if len(greenlets) > pool_size:
			print 'uhoh, greenlets are getting stuck', len(greenlets)
		if len(greenlets) < pool_size:		
			try:
				i = in_q.get_nowait()
			except:
				break
			if not isinstance(i, dict):
				i = {inspect.getargspec(func).args[0]: i}
			kwargs = dict(kwargs.items() + i.items())
			greenlets.add(p.spawn(func, **kwargs))
		else:
			time.sleep(1)


	p.join()
	for g in greenlets:
		if g.value:
			yield g.value