From 886ff07734187d7411bb20ad43ddecabd2a714b9 Mon Sep 17 00:00:00 2001 From: Kazuki Suzuki Przyborowski Date: Tue, 22 Oct 2024 12:35:35 -0500 Subject: [PATCH] Update pywwwgetmini.py --- pywwwgetmini.py | 2370 +---------------------------------------------- 1 file changed, 4 insertions(+), 2366 deletions(-) diff --git a/pywwwgetmini.py b/pywwwgetmini.py index adf912b..e3966ba 100644 --- a/pywwwgetmini.py +++ b/pywwwgetmini.py @@ -430,7 +430,7 @@ def download_file_from_http_file(url, headers=None, usehttp=__use_http_lib__): return httpfile -def download_file_from_http_string(url, headers=geturls_ua_pywwwget_python_alt, usehttp=__use_http_lib__): +def download_file_from_http_string(url, headers=geturls_headers_pywwwget_python_alt, usehttp=__use_http_lib__): httpfile = download_file_from_http_file(url, headers, usehttp) return httpfile.read() @@ -667,7 +667,8 @@ def upload_file_to_pysftp_string(sftpstring, url): def upload_file_to_pysftp_string(url): return False -def download_file_from_internet_file(url, headers=geturls_ua_pywwwget_python_alt, usehttp=__use_http_lib__): + +def download_file_from_internet_file(url, headers=geturls_headers_pywwwget_python_alt, usehttp=__use_http_lib__): urlparts = urlparse(url) if(urlparts.scheme == "http" or urlparts.scheme == "https"): return download_file_from_http_file(url, headers, usehttp) @@ -683,7 +684,7 @@ def download_file_from_internet_file(url, headers=geturls_ua_pywwwget_python_alt return False -def download_file_from_internet_string(url, headers=geturls_ua_pywwwget_python_alt, usehttp=__use_http_lib__): +def download_file_from_internet_string(url, headers=geturls_headers_pywwwget_python_alt, usehttp=__use_http_lib__): urlparts = urlparse(url) if(urlparts.scheme == "http" or urlparts.scheme == "https"): return download_file_from_http_string(url, headers, usehttp) @@ -729,2366 +730,3 @@ def upload_file_to_internet_string(ifp, url): else: return False return False - - return path + "/" + execfile - - -def listize(varlist): - il = 0 - ix = len(varlist) - ilx = 1 - newlistreg = {} - newlistrev = {} - newlistfull = {} - while(il < ix): - newlistreg.update({ilx: varlist[il]}) - newlistrev.update({varlist[il]: ilx}) - ilx = ilx + 1 - il = il + 1 - newlistfull = {1: newlistreg, 2: newlistrev, - 'reg': newlistreg, 'rev': newlistrev} - return newlistfull - - -def twolistize(varlist): - il = 0 - ix = len(varlist) - ilx = 1 - newlistnamereg = {} - newlistnamerev = {} - newlistdescreg = {} - newlistdescrev = {} - newlistfull = {} - while(il < ix): - newlistnamereg.update({ilx: varlist[il][0].strip()}) - newlistnamerev.update({varlist[il][0].strip(): ilx}) - newlistdescreg.update({ilx: varlist[il][1].strip()}) - newlistdescrev.update({varlist[il][1].strip(): ilx}) - ilx = ilx + 1 - il = il + 1 - newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, - 'reg': newlistnamereg, 'rev': newlistnamerev} - newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, - 'reg': newlistdescreg, 'rev': newlistdescrev} - newlistfull = {1: newlistnametmp, 2: newlistdesctmp, - 'name': newlistnametmp, 'desc': newlistdesctmp} - return newlistfull - - -def arglistize(proexec, *varlist): - il = 0 - ix = len(varlist) - ilx = 1 - newarglist = [proexec] - while(il < ix): - if varlist[il][0] is not None: - newarglist.append(varlist[il][0]) - if varlist[il][1] is not None: - newarglist.append(varlist[il][1]) - il = il + 1 - return newarglist - - -def fix_header_names(header_dict): - if(sys.version[0] == "2"): - header_dict = {k.title(): v for k, v in header_dict.iteritems()} - if(sys.version[0] >= "3"): - header_dict = {k.title(): v for k, v in header_dict.items()} - return header_dict - -# hms_string by ArcGIS Python Recipes -# https://arcpy.wordpress.com/2012/04/20/146/ - - -def hms_string(sec_elapsed): - h = int(sec_elapsed / (60 * 60)) - m = int((sec_elapsed % (60 * 60)) / 60) - s = sec_elapsed % 60.0 - return "{}:{:>02}:{:>05.2f}".format(h, m, s) - -# get_readable_size by Lipis -# http://stackoverflow.com/posts/14998888/revisions - - -def get_readable_size(bytes, precision=1, unit="IEC"): - unit = unit.upper() - if(unit != "IEC" and unit != "SI"): - unit = "IEC" - if(unit == "IEC"): - units = [" B", " KiB", " MiB", " GiB", " TiB", " PiB", " EiB", " ZiB"] - unitswos = ["B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB"] - unitsize = 1024.0 - if(unit == "SI"): - units = [" B", " kB", " MB", " GB", " TB", " PB", " EB", " ZB"] - unitswos = ["B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB"] - unitsize = 1000.0 - return_val = {} - orgbytes = bytes - for unit in units: - if abs(bytes) < unitsize: - strformat = "%3."+str(precision)+"f%s" - pre_return_val = (strformat % (bytes, unit)) - pre_return_val = re.sub( - r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val) - pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val) - alt_return_val = pre_return_val.split() - return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, - 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]} - return return_val - bytes /= unitsize - strformat = "%."+str(precision)+"f%s" - pre_return_val = (strformat % (bytes, "YiB")) - pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val) - pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val) - alt_return_val = pre_return_val.split() - return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, - 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]} - return return_val - - -def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"): - unit = unit.upper() - usehashtypes = usehashtypes.lower() - getfilesize = os.path.getsize(infile) - return_val = get_readable_size(getfilesize, precision, unit) - if(usehashes): - hashtypelist = usehashtypes.split(",") - openfile = open(infile, "rb") - filecontents = openfile.read() - openfile.close() - listnumcount = 0 - listnumend = len(hashtypelist) - while(listnumcount < listnumend): - hashtypelistlow = hashtypelist[listnumcount].strip() - hashtypelistup = hashtypelistlow.upper() - filehash = hashlib.new(hashtypelistup) - filehash.update(filecontents) - filegethash = filehash.hexdigest() - return_val.update({hashtypelistup: filegethash}) - listnumcount += 1 - return return_val - - -def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"): - unit = unit.upper() - usehashtypes = usehashtypes.lower() - getfilesize = len(instring) - return_val = get_readable_size(getfilesize, precision, unit) - if(usehashes): - hashtypelist = usehashtypes.split(",") - listnumcount = 0 - listnumend = len(hashtypelist) - while(listnumcount < listnumend): - hashtypelistlow = hashtypelist[listnumcount].strip() - hashtypelistup = hashtypelistlow.upper() - filehash = hashlib.new(hashtypelistup) - if(sys.version[0] == "2"): - filehash.update(instring) - if(sys.version[0] >= "3"): - filehash.update(instring.encode('utf-8')) - filegethash = filehash.hexdigest() - return_val.update({hashtypelistup: filegethash}) - listnumcount += 1 - return return_val - - -def http_status_to_reason(code): - reasons = { - 100: 'Continue', - 101: 'Switching Protocols', - 102: 'Processing', - 200: 'OK', - 201: 'Created', - 202: 'Accepted', - 203: 'Non-Authoritative Information', - 204: 'No Content', - 205: 'Reset Content', - 206: 'Partial Content', - 207: 'Multi-Status', - 208: 'Already Reported', - 226: 'IM Used', - 300: 'Multiple Choices', - 301: 'Moved Permanently', - 302: 'Found', - 303: 'See Other', - 304: 'Not Modified', - 305: 'Use Proxy', - 307: 'Temporary Redirect', - 308: 'Permanent Redirect', - 400: 'Bad Request', - 401: 'Unauthorized', - 402: 'Payment Required', - 403: 'Forbidden', - 404: 'Not Found', - 405: 'Method Not Allowed', - 406: 'Not Acceptable', - 407: 'Proxy Authentication Required', - 408: 'Request Timeout', - 409: 'Conflict', - 410: 'Gone', - 411: 'Length Required', - 412: 'Precondition Failed', - 413: 'Payload Too Large', - 414: 'URI Too Long', - 415: 'Unsupported Media Type', - 416: 'Range Not Satisfiable', - 417: 'Expectation Failed', - 421: 'Misdirected Request', - 422: 'Unprocessable Entity', - 423: 'Locked', - 424: 'Failed Dependency', - 426: 'Upgrade Required', - 428: 'Precondition Required', - 429: 'Too Many Requests', - 431: 'Request Header Fields Too Large', - 451: 'Unavailable For Legal Reasons', - 500: 'Internal Server Error', - 501: 'Not Implemented', - 502: 'Bad Gateway', - 503: 'Service Unavailable', - 504: 'Gateway Timeout', - 505: 'HTTP Version Not Supported', - 506: 'Variant Also Negotiates', - 507: 'Insufficient Storage', - 508: 'Loop Detected', - 510: 'Not Extended', - 511: 'Network Authentication Required' - } - return reasons.get(code, 'Unknown Status Code') - - -def ftp_status_to_reason(code): - reasons = { - 110: 'Restart marker reply', - 120: 'Service ready in nnn minutes', - 125: 'Data connection already open; transfer starting', - 150: 'File status okay; about to open data connection', - 200: 'Command okay', - 202: 'Command not implemented, superfluous at this site', - 211: 'System status, or system help reply', - 212: 'Directory status', - 213: 'File status', - 214: 'Help message', - 215: 'NAME system type', - 220: 'Service ready for new user', - 221: 'Service closing control connection', - 225: 'Data connection open; no transfer in progress', - 226: 'Closing data connection', - 227: 'Entering Passive Mode', - 230: 'User logged in, proceed', - 250: 'Requested file action okay, completed', - 257: '"PATHNAME" created', - 331: 'User name okay, need password', - 332: 'Need account for login', - 350: 'Requested file action pending further information', - 421: 'Service not available, closing control connection', - 425: 'Can\'t open data connection', - 426: 'Connection closed; transfer aborted', - 450: 'Requested file action not taken', - 451: 'Requested action aborted. Local error in processing', - 452: 'Requested action not taken. Insufficient storage space in system', - 500: 'Syntax error, command unrecognized', - 501: 'Syntax error in parameters or arguments', - 502: 'Command not implemented', - 503: 'Bad sequence of commands', - 504: 'Command not implemented for that parameter', - 530: 'Not logged in', - 532: 'Need account for storing files', - 550: 'Requested action not taken. File unavailable', - 551: 'Requested action aborted. Page type unknown', - 552: 'Requested file action aborted. Exceeded storage allocation', - 553: 'Requested action not taken. File name not allowed' - } - return reasons.get(code, 'Unknown Status Code') - - -def sftp_status_to_reason(code): - reasons = { - 0: 'SSH_FX_OK', - 1: 'SSH_FX_EOF', - 2: 'SSH_FX_NO_SUCH_FILE', - 3: 'SSH_FX_PERMISSION_DENIED', - 4: 'SSH_FX_FAILURE', - 5: 'SSH_FX_BAD_MESSAGE', - 6: 'SSH_FX_NO_CONNECTION', - 7: 'SSH_FX_CONNECTION_LOST', - 8: 'SSH_FX_OP_UNSUPPORTED' - } - return reasons.get(code, 'Unknown Status Code') - - -def make_http_headers_from_dict_to_list(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}): - if isinstance(headers, dict): - returnval = [] - if(sys.version[0] == "2"): - for headkey, headvalue in headers.iteritems(): - returnval.append((headkey, headvalue)) - if(sys.version[0] >= "3"): - for headkey, headvalue in headers.items(): - returnval.append((headkey, headvalue)) - elif isinstance(headers, list): - returnval = headers - else: - returnval = False - return returnval - - -def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}): - if isinstance(headers, dict): - returnval = [] - if(sys.version[0] == "2"): - for headkey, headvalue in headers.iteritems(): - returnval.append(headkey+": "+headvalue) - if(sys.version[0] >= "3"): - for headkey, headvalue in headers.items(): - returnval.append(headkey+": "+headvalue) - elif isinstance(headers, list): - returnval = headers - else: - returnval = False - return returnval - - -def make_http_headers_from_pycurl_to_dict(headers): - header_dict = {} - headers = headers.strip().split('\r\n') - for header in headers: - parts = header.split(': ', 1) - if(len(parts) == 2): - key, value = parts - header_dict[key.title()] = value - return header_dict - - -def make_http_headers_from_list_to_dict(headers=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", compression_supported), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]): - if isinstance(headers, list): - returnval = {} - mli = 0 - mlil = len(headers) - while(mli < mlil): - returnval.update({headers[mli][0]: headers[mli][1]}) - mli = mli + 1 - elif isinstance(headers, dict): - returnval = headers - else: - returnval = False - return returnval - - -def get_httplib_support(checkvalue=None): - global haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp - returnval = [] - returnval.append("ftp") - returnval.append("httplib") - if(havehttplib2): - returnval.append("httplib2") - returnval.append("urllib") - if(haveurllib3): - returnval.append("urllib3") - returnval.append("request3") - returnval.append("request") - if(haverequests): - returnval.append("requests") - if(haveaiohttp): - returnval.append("aiohttp") - if(havehttpx): - returnval.append("httpx") - returnval.append("httpx2") - if(havemechanize): - returnval.append("mechanize") - if(havepycurl): - returnval.append("pycurl") - if(hasattr(pycurl, "CURL_HTTP_VERSION_2_0")): - returnval.append("pycurl2") - if(hasattr(pycurl, "CURL_HTTP_VERSION_3_0")): - returnval.append("pycurl3") - if(haveparamiko): - returnval.append("sftp") - if(havepysftp): - returnval.append("pysftp") - if(not checkvalue is None): - if(checkvalue == "urllib1" or checkvalue == "urllib2"): - checkvalue = "urllib" - if(checkvalue == "httplib1"): - checkvalue = "httplib" - if(checkvalue in returnval): - returnval = True - else: - returnval = False - return returnval - - -def check_httplib_support(checkvalue="urllib"): - if(checkvalue == "urllib1" or checkvalue == "urllib2"): - checkvalue = "urllib" - if(checkvalue == "httplib1"): - checkvalue = "httplib" - returnval = get_httplib_support(checkvalue) - return returnval - - -def get_httplib_support_list(): - returnval = get_httplib_support(None) - return returnval - - -def download_from_url(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1, timeout=10): - global geturls_download_sleep, havezstd, havebrotli, haveaiohttp, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp - if(sleep < 0): - sleep = geturls_download_sleep - if(timeout <= 0): - timeout = 10 - if(httplibuse == "urllib1" or httplibuse == "urllib2" or httplibuse == "request"): - httplibuse = "urllib" - if(httplibuse == "httplib1"): - httplibuse = "httplib" - if(not haverequests and httplibuse == "requests"): - httplibuse = "urllib" - if(not haveaiohttp and httplibuse == "aiohttp"): - httplibuse = "urllib" - if(not havehttpx and httplibuse == "httpx"): - httplibuse = "urllib" - if(not havehttpx and httplibuse == "httpx2"): - httplibuse = "urllib" - if(not havehttpcore and httplibuse == "httpcore"): - httplibuse = "urllib" - if(not havehttpcore and httplibuse == "httpcore2"): - httplibuse = "urllib" - if(not havemechanize and httplibuse == "mechanize"): - httplibuse = "urllib" - if(not havepycurl and httplibuse == "pycurl"): - httplibuse = "urllib" - if(not havepycurl and httplibuse == "pycurl2"): - httplibuse = "urllib" - if(havepycurl and httplibuse == "pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")): - httplibuse = "pycurl" - if(not havepycurl and httplibuse == "pycurl3"): - httplibuse = "urllib" - if(havepycurl and httplibuse == "pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")): - httplibuse = "pycurl2" - if(havepycurl and httplibuse == "pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")): - httplibuse = "pycurl" - if(not havehttplib2 and httplibuse == "httplib2"): - httplibuse = "httplib" - if(not haveparamiko and httplibuse == "sftp"): - httplibuse = "ftp" - if(not havepysftp and httplibuse == "pysftp"): - httplibuse = "ftp" - urlparts = urlparse.urlparse(httpurl) - if(isinstance(httpheaders, list)): - httpheaders = make_http_headers_from_list_to_dict(httpheaders) - httpheaders = fix_header_names(httpheaders) - if(httpuseragent is not None): - if('User-Agent' in httpheaders): - httpheaders['User-Agent'] = httpuseragent - else: - httpuseragent.update({'User-Agent': httpuseragent}) - if(httpreferer is not None): - if('Referer' in httpheaders): - httpheaders['Referer'] = httpreferer - else: - httpuseragent.update({'Referer': httpreferer}) - if(urlparts.username is not None or urlparts.password is not None): - if(sys.version[0] == "2"): - inurlencode = b64encode( - str(urlparts.username+":"+urlparts.password)) - if(sys.version[0] >= "3"): - inurlencode = b64encode( - str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8") - httpheaders.update({'Authorization': "Basic "+inurlencode}) - geturls_opener = build_opener(HTTPCookieProcessor(httpcookie)) - if(httplibuse == "urllib" or httplibuse == "mechanize"): - if(isinstance(httpheaders, dict)): - httpheaders = make_http_headers_from_dict_to_list(httpheaders) - if(httplibuse == "pycurl" or httplibuse == "pycurl2" or httplibuse == "pycurl3"): - if(isinstance(httpheaders, dict)): - httpheaders = make_http_headers_from_dict_to_pycurl(httpheaders) - geturls_opener.addheaders = httpheaders - time.sleep(sleep) - if(postdata is not None and not isinstance(postdata, dict)): - postdata = urlencode(postdata) - if(httplibuse == "urllib" or httplibuse == "request"): - geturls_request = Request(httpurl) - try: - if(httpmethod == "GET"): - geturls_text = geturls_opener.open(geturls_request) - elif(httpmethod == "POST"): - geturls_text = geturls_opener.open( - geturls_request, data=postdata) - else: - geturls_text = geturls_opener.open(geturls_request) - except HTTPError as geturls_text_error: - geturls_text = geturls_text_error - log.info("Error With URL "+httpurl) - except URLError: - log.info("Error With URL "+httpurl) - return False - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - httpcodeout = geturls_text.getcode() - try: - httpcodereason = geturls_text.reason - except AttributeError: - httpcodereason = http_status_to_reason(geturls_text.getcode()) - try: - httpversionout = geturls_text.version - except AttributeError: - httpversionout = "1.1" - httpmethodout = geturls_request.get_method() - httpurlout = geturls_text.geturl() - httpheaderout = geturls_text.info() - httpheadersentout = httpheaders - elif(httplibuse == "httplib"): - if(urlparts[0] == "http"): - httpconn = HTTPConnection(urlparts[1], timeout=timeout) - elif(urlparts[0] == "https"): - httpconn = HTTPSConnection(urlparts[1], timeout=timeout) - else: - return False - if(postdata is not None and not isinstance(postdata, dict)): - postdata = urlencode(postdata) - try: - if(httpmethod == "GET"): - httpconn.request("GET", urlparts[2], headers=httpheaders) - elif(httpmethod == "POST"): - httpconn.request( - "GET", urlparts[2], body=postdata, headers=httpheaders) - else: - httpconn.request("GET", urlparts[2], headers=httpheaders) - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - except socket.gaierror: - log.info("Error With URL "+httpurl) - return False - except BlockingIOError: - log.info("Error With URL "+httpurl) - return False - geturls_text = httpconn.getresponse() - httpcodeout = geturls_text.status - httpcodereason = geturls_text.reason - if(geturls_text.version == "10"): - httpversionout = "1.0" - else: - httpversionout = "1.1" - httpmethodout = geturls_text._method - httpurlout = httpurl - httpheaderout = geturls_text.getheaders() - httpheadersentout = httpheaders - elif(httplibuse == "httplib2"): - if(urlparts[0] == "http"): - httpconn = HTTPConnectionWithTimeout(urlparts[1], timeout=timeout) - elif(urlparts[0] == "https"): - httpconn = HTTPSConnectionWithTimeout(urlparts[1], timeout=timeout) - else: - return False - if(postdata is not None and not isinstance(postdata, dict)): - postdata = urlencode(postdata) - try: - if(httpmethod == "GET"): - httpconn.request("GET", urlparts[2], headers=httpheaders) - elif(httpmethod == "POST"): - httpconn.request( - "GET", urlparts[2], body=postdata, headers=httpheaders) - else: - httpconn.request("GET", urlparts[2], headers=httpheaders) - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - except socket.gaierror: - log.info("Error With URL "+httpurl) - return False - except BlockingIOError: - log.info("Error With URL "+httpurl) - return False - geturls_text = httpconn.getresponse() - httpcodeout = geturls_text.status - httpcodereason = geturls_text.reason - if(geturls_text.version == "10"): - httpversionout = "1.0" - else: - httpversionout = "1.1" - httpmethodout = httpmethod - httpurlout = httpurl - httpheaderout = geturls_text.getheaders() - httpheadersentout = httpheaders - elif(httplibuse == "urllib3" or httplibuse == "request3"): - timeout = urllib3.util.Timeout(connect=timeout, read=timeout) - urllib_pool = urllib3.PoolManager(headers=httpheaders, timeout=timeout) - try: - if(httpmethod == "GET"): - geturls_text = urllib_pool.request( - "GET", httpurl, headers=httpheaders, preload_content=False) - elif(httpmethod == "POST"): - geturls_text = urllib_pool.request( - "POST", httpurl, body=postdata, headers=httpheaders, preload_content=False) - else: - geturls_text = urllib_pool.request( - "GET", httpurl, headers=httpheaders, preload_content=False) - except urllib3.exceptions.ConnectTimeoutError: - log.info("Error With URL "+httpurl) - return False - except urllib3.exceptions.ConnectError: - log.info("Error With URL "+httpurl) - return False - except urllib3.exceptions.MaxRetryError: - log.info("Error With URL "+httpurl) - return False - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - except ValueError: - log.info("Error With URL "+httpurl) - return False - httpcodeout = geturls_text.status - httpcodereason = geturls_text.reason - if(geturls_text.version == "10"): - httpversionout = "1.0" - else: - httpversionout = "1.1" - httpmethodout = httpmethod - httpurlout = geturls_text.geturl() - httpheaderout = geturls_text.info() - httpheadersentout = httpheaders - elif(httplibuse == "requests"): - try: - reqsession = requests.Session() - if(httpmethod == "GET"): - geturls_text = reqsession.get( - httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie) - elif(httpmethod == "POST"): - geturls_text = reqsession.post( - httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie) - else: - geturls_text = reqsession.get( - httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie) - except requests.exceptions.ConnectTimeout: - log.info("Error With URL "+httpurl) - return False - except requests.exceptions.ConnectError: - log.info("Error With URL "+httpurl) - return False - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - httpcodeout = geturls_text.status_code - httpcodereason = geturls_text.reason - if(geturls_text.raw.version == "10"): - httpversionout = "1.0" - else: - httpversionout = "1.1" - httpmethodout = httpmethod - httpurlout = geturls_text.url - httpheaderout = geturls_text.headers - httpheadersentout = geturls_text.request.headers - elif(httplibuse == "aiohttp"): - try: - reqsession = aiohttp.ClientSession(cookie_jar=httpcookie, headers=httpheaders, - timeout=timeout, read_timeout=timeout, conn_timeout=timeout, read_bufsize=buffersize) - if(httpmethod == "GET"): - geturls_text = reqsession.get(httpurl) - elif(httpmethod == "POST"): - geturls_text = reqsession.post(httpurl, data=postdata) - else: - geturls_text = reqsession.get(httpurl) - except aiohttp.exceptions.ConnectTimeout: - log.info("Error With URL "+httpurl) - return False - except aiohttp.exceptions.ConnectError: - log.info("Error With URL "+httpurl) - return False - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - httpcodeout = geturls_text.status - httpcodereason = geturls_text.reason - httpversionout = geturls_text.version - httpmethodout = geturls_text.method - httpurlout = geturls_text.url - httpheaderout = geturls_text.headers - httpheadersentout = geturls_text.request_info.headers - elif(httplibuse == "httpx"): - try: - if(httpmethod == "GET"): - httpx_pool = httpx.Client( - http1=True, http2=False, trust_env=True) - geturls_text = httpx_pool.get( - httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie) - elif(httpmethod == "POST"): - httpx_pool = httpx.Client( - http1=True, http2=False, trust_env=True) - geturls_text = httpx_pool.post( - httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie) - else: - httpx_pool = httpx.Client( - http1=True, http2=False, trust_env=True) - geturls_text = httpx_pool.get( - httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie) - except httpx.ConnectTimeout: - log.info("Error With URL "+httpurl) - return False - except httpx.ConnectError: - log.info("Error With URL "+httpurl) - return False - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - httpcodeout = geturls_text.status_code - try: - httpcodereason = geturls_text.reason_phrase - except: - httpcodereason = http_status_to_reason(geturls_text.status_code) - httpversionout = geturls_text.http_version - httpmethodout = httpmethod - httpurlout = str(geturls_text.url) - httpheaderout = geturls_text.headers - httpheadersentout = geturls_text.request.headers - elif(httplibuse == "httpx2"): - try: - if(httpmethod == "GET"): - httpx_pool = httpx.Client( - http1=True, http2=True, trust_env=True) - geturls_text = httpx_pool.get( - httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie) - elif(httpmethod == "POST"): - httpx_pool = httpx.Client( - http1=True, http2=True, trust_env=True) - geturls_text = httpx_pool.post( - httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie) - else: - httpx_pool = httpx.Client( - http1=True, http2=True, trust_env=True) - geturls_text = httpx_pool.get( - httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie) - except httpx.ConnectTimeout: - log.info("Error With URL "+httpurl) - return False - except httpx.ConnectError: - log.info("Error With URL "+httpurl) - return False - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - httpcodeout = geturls_text.status_code - try: - httpcodereason = geturls_text.reason_phrase - except: - httpcodereason = http_status_to_reason(geturls_text.status_code) - httpversionout = geturls_text.http_version - httpmethodout = httpmethod - httpurlout = str(geturls_text.url) - httpheaderout = geturls_text.headers - httpheadersentout = geturls_text.request.headers - elif(httplibuse == "httpcore"): - try: - if(httpmethod == "GET"): - httpx_pool = httpcore.ConnectionPool(http1=True, http2=False) - geturls_text = httpx_pool.request( - "GET", httpurl, headers=httpheaders) - elif(httpmethod == "POST"): - httpx_pool = httpcore.ConnectionPool(http1=True, http2=False) - geturls_text = httpx_pool.request( - "GET", httpurl, data=postdata, headers=httpheaders) - else: - httpx_pool = httpcore.ConnectionPool(http1=True, http2=False) - geturls_text = httpx_pool.request( - "GET", httpurl, headers=httpheaders) - except httpcore.ConnectTimeout: - log.info("Error With URL "+httpurl) - return False - except httpcore.ConnectError: - log.info("Error With URL "+httpurl) - return False - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - httpcodeout = geturls_text.status - httpcodereason = http_status_to_reason(geturls_text.status) - httpversionout = "1.1" - httpmethodout = httpmethod - httpurlout = str(httpurl) - httpheaderout = geturls_text.headers - httpheadersentout = httpheaders - elif(httplibuse == "httpcore2"): - try: - if(httpmethod == "GET"): - httpx_pool = httpcore.ConnectionPool(http1=True, http2=True) - geturls_text = httpx_pool.request( - "GET", httpurl, headers=httpheaders) - elif(httpmethod == "POST"): - httpx_pool = httpcore.ConnectionPool(http1=True, http2=True) - geturls_text = httpx_pool.request( - "GET", httpurl, data=postdata, headers=httpheaders) - else: - httpx_pool = httpcore.ConnectionPool(http1=True, http2=True) - geturls_text = httpx_pool.request( - "GET", httpurl, headers=httpheaders) - except httpcore.ConnectTimeout: - log.info("Error With URL "+httpurl) - return False - except httpcore.ConnectError: - log.info("Error With URL "+httpurl) - return False - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - httpcodeout = geturls_text.status - httpcodereason = http_status_to_reason(geturls_text.status) - httpversionout = "1.1" - httpmethodout = httpmethod - httpurlout = str(httpurl) - httpheaderout = geturls_text.headers - httpheadersentout = httpheaders - elif(httplibuse == "mechanize"): - geturls_opener = mechanize.Browser() - if(isinstance(httpheaders, dict)): - httpheaders = make_http_headers_from_dict_to_list(httpheaders) - time.sleep(sleep) - geturls_opener.addheaders = httpheaders - geturls_opener.set_cookiejar(httpcookie) - geturls_opener.set_handle_robots(False) - if(postdata is not None and not isinstance(postdata, dict)): - postdata = urlencode(postdata) - try: - if(httpmethod == "GET"): - geturls_text = geturls_opener.open(httpurl) - elif(httpmethod == "POST"): - geturls_text = geturls_opener.open(httpurl, data=postdata) - else: - geturls_text = geturls_opener.open(httpurl) - except mechanize.HTTPError as geturls_text_error: - geturls_text = geturls_text_error - log.info("Error With URL "+httpurl) - except URLError: - log.info("Error With URL "+httpurl) - return False - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - httpcodeout = geturls_text.code - httpcodereason = geturls_text.msg - httpversionout = "1.1" - httpmethodout = httpmethod - httpurlout = geturls_text.geturl() - httpheaderout = geturls_text.info() - reqhead = geturls_opener.request - httpheadersentout = reqhead.header_items() - elif(httplibuse == "pycurl"): - retrieved_body = BytesIO() - retrieved_headers = BytesIO() - try: - if(httpmethod == "GET"): - geturls_text = pycurl.Curl() - geturls_text.setopt(geturls_text.URL, httpurl) - geturls_text.setopt(geturls_text.HTTP_VERSION, - geturls_text.CURL_HTTP_VERSION_1_1) - geturls_text.setopt( - geturls_text.WRITEFUNCTION, retrieved_body.write) - geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders) - geturls_text.setopt( - geturls_text.HEADERFUNCTION, retrieved_headers.write) - geturls_text.setopt(geturls_text.FOLLOWLOCATION, True) - geturls_text.setopt(geturls_text.TIMEOUT, timeout) - geturls_text.perform() - elif(httpmethod == "POST"): - geturls_text = pycurl.Curl() - geturls_text.setopt(geturls_text.URL, httpurl) - geturls_text.setopt(geturls_text.HTTP_VERSION, - geturls_text.CURL_HTTP_VERSION_1_1) - geturls_text.setopt( - geturls_text.WRITEFUNCTION, retrieved_body.write) - geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders) - geturls_text.setopt( - geturls_text.HEADERFUNCTION, retrieved_headers.write) - geturls_text.setopt(geturls_text.FOLLOWLOCATION, True) - geturls_text.setopt(geturls_text.TIMEOUT, timeout) - geturls_text.setopt(geturls_text.POST, True) - geturls_text.setopt(geturls_text.POSTFIELDS, postdata) - geturls_text.perform() - else: - geturls_text = pycurl.Curl() - geturls_text.setopt(geturls_text.URL, httpurl) - geturls_text.setopt(geturls_text.HTTP_VERSION, - geturls_text.CURL_HTTP_VERSION_1_1) - geturls_text.setopt( - geturls_text.WRITEFUNCTION, retrieved_body.write) - geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders) - geturls_text.setopt( - geturls_text.HEADERFUNCTION, retrieved_headers.write) - geturls_text.setopt(geturls_text.FOLLOWLOCATION, True) - geturls_text.setopt(geturls_text.TIMEOUT, timeout) - geturls_text.perform() - retrieved_headers.seek(0) - if(sys.version[0] == "2"): - pycurlhead = retrieved_headers.read() - if(sys.version[0] >= "3"): - pycurlhead = retrieved_headers.read().decode('UTF-8') - pyhttpverinfo = re.findall( - r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip().rstrip('\r\n'))[0] - pycurlheadersout = make_http_headers_from_pycurl_to_dict( - pycurlhead) - retrieved_body.seek(0) - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - except socket.gaierror: - log.info("Error With URL "+httpurl) - return False - except ValueError: - log.info("Error With URL "+httpurl) - return False - httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE) - httpcodereason = http_status_to_reason( - geturls_text.getinfo(geturls_text.HTTP_CODE)) - httpversionout = pyhttpverinfo[0] - httpmethodout = httpmethod - httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL) - httpheaderout = pycurlheadersout - httpheadersentout = httpheaders - elif(httplibuse == "pycurl2"): - retrieved_body = BytesIO() - retrieved_headers = BytesIO() - try: - if(httpmethod == "GET"): - geturls_text = pycurl.Curl() - geturls_text.setopt(geturls_text.HTTP_VERSION, - geturls_text.CURL_HTTP_VERSION_2_0) - geturls_text.setopt(geturls_text.URL, httpurl) - geturls_text.setopt( - geturls_text.WRITEFUNCTION, retrieved_body.write) - geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders) - geturls_text.setopt( - geturls_text.HEADERFUNCTION, retrieved_headers.write) - geturls_text.setopt(geturls_text.FOLLOWLOCATION, True) - geturls_text.setopt(geturls_text.TIMEOUT, timeout) - geturls_text.perform() - elif(httpmethod == "POST"): - geturls_text = pycurl.Curl() - geturls_text.setopt(geturls_text.HTTP_VERSION, - geturls_text.CURL_HTTP_VERSION_2_0) - geturls_text.setopt(geturls_text.URL, httpurl) - geturls_text.setopt( - geturls_text.WRITEFUNCTION, retrieved_body.write) - geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders) - geturls_text.setopt( - geturls_text.HEADERFUNCTION, retrieved_headers.write) - geturls_text.setopt(geturls_text.FOLLOWLOCATION, True) - geturls_text.setopt(geturls_text.TIMEOUT, timeout) - geturls_text.setopt(geturls_text.POST, True) - geturls_text.setopt(geturls_text.POSTFIELDS, postdata) - geturls_text.perform() - else: - geturls_text = pycurl.Curl() - geturls_text.setopt(geturls_text.HTTP_VERSION, - geturls_text.CURL_HTTP_VERSION_2_0) - geturls_text.setopt(geturls_text.URL, httpurl) - geturls_text.setopt( - geturls_text.WRITEFUNCTION, retrieved_body.write) - geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders) - geturls_text.setopt( - geturls_text.HEADERFUNCTION, retrieved_headers.write) - geturls_text.setopt(geturls_text.FOLLOWLOCATION, True) - geturls_text.setopt(geturls_text.TIMEOUT, timeout) - geturls_text.perform() - retrieved_headers.seek(0) - if(sys.version[0] == "2"): - pycurlhead = retrieved_headers.read() - if(sys.version[0] >= "3"): - pycurlhead = retrieved_headers.read().decode('UTF-8') - pyhttpverinfo = re.findall( - r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip())[0] - pycurlheadersout = make_http_headers_from_pycurl_to_dict( - pycurlhead) - retrieved_body.seek(0) - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - except socket.gaierror: - log.info("Error With URL "+httpurl) - return False - except ValueError: - log.info("Error With URL "+httpurl) - return False - httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE) - httpcodereason = http_status_to_reason( - geturls_text.getinfo(geturls_text.HTTP_CODE)) - httpversionout = pyhttpverinfo[0] - httpmethodout = httpmethod - httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL) - httpheaderout = pycurlheadersout - httpheadersentout = httpheaders - elif(httplibuse == "pycurl3"): - retrieved_body = BytesIO() - retrieved_headers = BytesIO() - try: - if(httpmethod == "GET"): - geturls_text = pycurl.Curl() - geturls_text.setopt(geturls_text.HTTP_VERSION, - geturls_text.CURL_HTTP_VERSION_3_0) - geturls_text.setopt(geturls_text.URL, httpurl) - geturls_text.setopt( - geturls_text.WRITEFUNCTION, retrieved_body.write) - geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders) - geturls_text.setopt( - geturls_text.HEADERFUNCTION, retrieved_headers.write) - geturls_text.setopt(geturls_text.FOLLOWLOCATION, True) - geturls_text.setopt(geturls_text.TIMEOUT, timeout) - geturls_text.perform() - elif(httpmethod == "POST"): - geturls_text = pycurl.Curl() - geturls_text.setopt(geturls_text.HTTP_VERSION, - geturls_text.CURL_HTTP_VERSION_3_0) - geturls_text.setopt(geturls_text.URL, httpurl) - geturls_text.setopt( - geturls_text.WRITEFUNCTION, retrieved_body.write) - geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders) - geturls_text.setopt( - geturls_text.HEADERFUNCTION, retrieved_headers.write) - geturls_text.setopt(geturls_text.FOLLOWLOCATION, True) - geturls_text.setopt(geturls_text.TIMEOUT, timeout) - geturls_text.setopt(geturls_text.POST, True) - geturls_text.setopt(geturls_text.POSTFIELDS, postdata) - geturls_text.perform() - else: - geturls_text = pycurl.Curl() - geturls_text.setopt(geturls_text.HTTP_VERSION, - geturls_text.CURL_HTTP_VERSION_3_0) - geturls_text.setopt(geturls_text.URL, httpurl) - geturls_text.setopt( - geturls_text.WRITEFUNCTION, retrieved_body.write) - geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders) - geturls_text.setopt( - geturls_text.HEADERFUNCTION, retrieved_headers.write) - geturls_text.setopt(geturls_text.FOLLOWLOCATION, True) - geturls_text.setopt(geturls_text.TIMEOUT, timeout) - geturls_text.perform() - retrieved_headers.seek(0) - if(sys.version[0] == "2"): - pycurlhead = retrieved_headers.read() - if(sys.version[0] >= "3"): - pycurlhead = retrieved_headers.read().decode('UTF-8') - pyhttpverinfo = re.findall( - r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip().rstrip('\r\n'))[0] - pycurlheadersout = make_http_headers_from_pycurl_to_dict( - pycurlhead) - retrieved_body.seek(0) - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - except socket.gaierror: - log.info("Error With URL "+httpurl) - return False - except ValueError: - log.info("Error With URL "+httpurl) - return False - httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE) - httpcodereason = http_status_to_reason( - geturls_text.getinfo(geturls_text.HTTP_CODE)) - httpversionout = pyhttpverinfo[0] - httpmethodout = httpmethod - httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL) - httpheaderout = pycurlheadersout - httpheadersentout = httpheaders - elif(httplibuse == "ftp"): - geturls_text = download_file_from_ftp_file(httpurl) - if(not geturls_text): - return False - downloadsize = None - if(downloadsize is not None): - downloadsize = int(downloadsize) - if downloadsize is None: - downloadsize = 0 - fulldatasize = 0 - prevdownsize = 0 - log.info("Downloading URL "+httpurl) - with BytesIO() as strbuf: - while True: - databytes = geturls_text.read(buffersize) - if not databytes: - break - datasize = len(databytes) - fulldatasize = datasize + fulldatasize - percentage = "" - if(downloadsize > 0): - percentage = str("{0:.2f}".format( - float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%" - downloaddiff = fulldatasize - prevdownsize - log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")[ - 'ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']) - prevdownsize = fulldatasize - strbuf.write(databytes) - strbuf.seek(0) - returnval_content = strbuf.read() - returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size( - fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None} - geturls_text.close() - elif(httplibuse == "sftp"): - geturls_text = download_file_from_sftp_file(httpurl) - if(not geturls_text): - return False - downloadsize = None - if(downloadsize is not None): - downloadsize = int(downloadsize) - if downloadsize is None: - downloadsize = 0 - fulldatasize = 0 - prevdownsize = 0 - log.info("Downloading URL "+httpurl) - with BytesIO() as strbuf: - while True: - databytes = geturls_text.read(buffersize) - if not databytes: - break - datasize = len(databytes) - fulldatasize = datasize + fulldatasize - percentage = "" - if(downloadsize > 0): - percentage = str("{0:.2f}".format( - float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%" - downloaddiff = fulldatasize - prevdownsize - log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")[ - 'ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']) - prevdownsize = fulldatasize - strbuf.write(databytes) - strbuf.seek(0) - returnval_content = strbuf.read() - returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size( - fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None} - geturls_text.close() - return returnval - elif(httplibuse == "pysftp"): - geturls_text = download_file_from_pysftp_file(httpurl) - if(not geturls_text): - return False - downloadsize = None - if(downloadsize is not None): - downloadsize = int(downloadsize) - if downloadsize is None: - downloadsize = 0 - fulldatasize = 0 - prevdownsize = 0 - log.info("Downloading URL "+httpurl) - with BytesIO() as strbuf: - while True: - databytes = geturls_text.read(buffersize) - if not databytes: - break - datasize = len(databytes) - fulldatasize = datasize + fulldatasize - percentage = "" - if(downloadsize > 0): - percentage = str("{0:.2f}".format( - float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%" - downloaddiff = fulldatasize - prevdownsize - log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")[ - 'ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']) - prevdownsize = fulldatasize - strbuf.write(databytes) - strbuf.seek(0) - returnval_content = strbuf.read() - returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size( - fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None} - geturls_text.close() - return returnval - else: - returnval = False - if(isinstance(httpheaderout, list) and (httplibuse != "pycurl" and httplibuse != "pycurl2" and httplibuse != "pycurl3")): - httpheaderout = dict( - make_http_headers_from_list_to_dict(httpheaderout)) - if(isinstance(httpheaderout, list) and (httplibuse == "pycurl" or httplibuse == "pycurl2" or httplibuse == "pycurl3")): - httpheaderout = dict(make_http_headers_from_pycurl_to_dict( - "\r\n".join(httpheaderout))) - if(sys.version[0] == "2"): - try: - prehttpheaderout = httpheaderout - httpheaderkeys = httpheaderout.keys() - imax = len(httpheaderkeys) - ic = 0 - httpheaderout = {} - while(ic < imax): - httpheaderout.update( - {httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]}) - ic += 1 - except AttributeError: - pass - httpheaderout = fix_header_names(httpheaderout) - if(isinstance(httpheadersentout, list) and (httplibuse != "pycurl" and httplibuse != "pycurl2" and httplibuse != "pycurl3")): - httpheadersentout = dict( - make_http_headers_from_list_to_dict(httpheadersentout)) - if(isinstance(httpheadersentout, list) and (httplibuse == "pycurl" or httplibuse == "pycurl2" or httplibuse == "pycurl3")): - httpheadersentout = dict(make_http_headers_from_pycurl_to_dict( - "\r\n".join(httpheadersentout))) - httpheadersentout = fix_header_names(httpheadersentout) - log.info("Downloading URL "+httpurl) - if(httplibuse == "urllib" or httplibuse == "request" or httplibuse == "request3" or httplibuse == "aiohttp" or httplibuse == "httplib" or httplibuse == "httplib2" or httplibuse == "urllib3" or httplibuse == "mechanize"): - downloadsize = httpheaderout.get('Content-Length') - if(downloadsize is not None): - downloadsize = int(downloadsize) - if downloadsize is None: - downloadsize = 0 - fulldatasize = 0 - prevdownsize = 0 - log.info("Downloading URL "+httpurl) - with BytesIO() as strbuf: - while True: - databytes = geturls_text.read(buffersize) - if not databytes: - break - datasize = len(databytes) - fulldatasize = datasize + fulldatasize - percentage = "" - if(downloadsize > 0): - percentage = str("{0:.2f}".format( - float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%" - downloaddiff = fulldatasize - prevdownsize - log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")[ - 'ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']) - prevdownsize = fulldatasize - strbuf.write(databytes) - strbuf.seek(0) - returnval_content = strbuf.read() - geturls_text.close() - if(httpheaderout.get("Content-Encoding") == "gzip"): - try: - returnval_content = zlib.decompress( - returnval_content, 16+zlib.MAX_WBITS) - except zlib.error: - pass - elif(httpheaderout.get("Content-Encoding") == "deflate"): - try: - returnval_content = zlib.decompress(returnval_content) - except zlib.error: - pass - elif(httpheaderout.get("Content-Encoding") == "br" and havebrotli): - try: - returnval_content = brotli.decompress(returnval_content) - except brotli.error: - pass - elif(httpheaderout.get("Content-Encoding") == "zstd" and havezstd): - try: - returnval_content = zstandard.decompress(returnval_content) - except zstandard.error: - pass - elif((httpheaderout.get("Content-Encoding") == "lzma" or httpheaderout.get("Content-Encoding") == "xz") and havelzma): - try: - returnval_content = lzma.decompress(returnval_content) - except zstandard.error: - pass - elif(httpheaderout.get("Content-Encoding") == "bzip2"): - try: - returnval_content = bz2.decompress(returnval_content) - except zstandard.error: - pass - elif(httplibuse == "httpx" or httplibuse == "httpx2" or httplibuse == "httpcore" or httplibuse == "httpcore2"): - downloadsize = httpheaderout.get('Content-Length') - if(downloadsize is not None): - downloadsize = int(downloadsize) - if downloadsize is None: - downloadsize = 0 - fulldatasize = 0 - prevdownsize = 0 - log.info("Downloading URL "+httpurl) - with BytesIO() as strbuf: - while True: - databytes = geturls_text.read() - if not databytes: - break - datasize = len(databytes) - fulldatasize = datasize + fulldatasize - percentage = "" - if(downloadsize > 0): - percentage = str("{0:.2f}".format( - float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%" - downloaddiff = fulldatasize - prevdownsize - log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")[ - 'ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']) - prevdownsize = fulldatasize - strbuf.write(databytes) - break - strbuf.seek(0) - returnval_content = strbuf.read() - geturls_text.close() - if(httpheaderout.get("Content-Encoding") == "gzip"): - try: - returnval_content = zlib.decompress( - returnval_content, 16+zlib.MAX_WBITS) - except zlib.error: - pass - elif(httpheaderout.get("Content-Encoding") == "deflate"): - try: - returnval_content = zlib.decompress(returnval_content) - except zlib.error: - pass - elif(httpheaderout.get("Content-Encoding") == "br" and havebrotli): - try: - returnval_content = brotli.decompress(returnval_content) - except brotli.error: - pass - elif(httpheaderout.get("Content-Encoding") == "zstd" and havezstd): - try: - returnval_content = zstandard.decompress(returnval_content) - except zstandard.error: - pass - elif((httpheaderout.get("Content-Encoding") == "lzma" or httpheaderout.get("Content-Encoding") == "xz") and havelzma): - try: - returnval_content = lzma.decompress(returnval_content) - except zstandard.error: - pass - elif(httpheaderout.get("Content-Encoding") == "bzip2"): - try: - returnval_content = bz2.decompress(returnval_content) - except zstandard.error: - pass - elif(httplibuse == "requests"): - log.info("Downloading URL "+httpurl) - downloadsize = httpheaderout.get('Content-Length') - if(downloadsize is not None): - downloadsize = int(downloadsize) - if downloadsize is None: - downloadsize = 0 - fulldatasize = 0 - prevdownsize = 0 - log.info("Downloading URL "+httpurl) - with BytesIO() as strbuf: - while True: - databytes = geturls_text.raw.read(buffersize) - if not databytes: - break - datasize = len(databytes) - fulldatasize = datasize + fulldatasize - percentage = "" - if(downloadsize > 0): - percentage = str("{0:.2f}".format( - float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%" - downloaddiff = fulldatasize - prevdownsize - log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")[ - 'ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']) - prevdownsize = fulldatasize - strbuf.write(databytes) - strbuf.seek(0) - returnval_content = strbuf.read() - geturls_text.close() - if(httpheaderout.get("Content-Encoding") == "gzip"): - try: - returnval_content = zlib.decompress( - returnval_content, 16+zlib.MAX_WBITS) - except zlib.error: - pass - elif(httpheaderout.get("Content-Encoding") == "deflate"): - try: - returnval_content = zlib.decompress(returnval_content) - except zlib.error: - pass - elif(httpheaderout.get("Content-Encoding") == "br" and havebrotli): - try: - returnval_content = brotli.decompress(returnval_content) - except brotli.error: - pass - elif(httpheaderout.get("Content-Encoding") == "zstd" and havezstd): - try: - returnval_content = zstandard.decompress(returnval_content) - except zstandard.error: - pass - elif((httpheaderout.get("Content-Encoding") == "lzma" or httpheaderout.get("Content-Encoding") == "xz") and havelzma): - try: - returnval_content = lzma.decompress(returnval_content) - except zstandard.error: - pass - elif(httpheaderout.get("Content-Encoding") == "bzip2"): - try: - returnval_content = bz2.decompress(returnval_content) - except zstandard.error: - pass - elif(httplibuse == "pycurl" or httplibuse == "pycurl2" or httplibuse == "pycurl3"): - log.info("Downloading URL "+httpurl) - downloadsize = httpheaderout.get('Content-Length') - if(downloadsize is not None): - downloadsize = int(downloadsize) - if downloadsize is None: - downloadsize = 0 - fulldatasize = 0 - prevdownsize = 0 - log.info("Downloading URL "+httpurl) - with BytesIO() as strbuf: - while True: - databytes = retrieved_body.read(buffersize) - if not databytes: - break - datasize = len(databytes) - fulldatasize = datasize + fulldatasize - percentage = "" - if(downloadsize > 0): - percentage = str("{0:.2f}".format( - float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%" - downloaddiff = fulldatasize - prevdownsize - log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")[ - 'ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']) - prevdownsize = fulldatasize - strbuf.write(databytes) - strbuf.seek(0) - returnval_content = strbuf.read() - geturls_text.close() - if(httpheaderout.get("Content-Encoding") == "gzip"): - try: - returnval_content = zlib.decompress( - returnval_content, 16+zlib.MAX_WBITS) - except zlib.error: - pass - elif(httpheaderout.get("Content-Encoding") == "deflate"): - try: - returnval_content = zlib.decompress(returnval_content) - except zlib.error: - pass - elif(httpheaderout.get("Content-Encoding") == "br" and havebrotli): - try: - returnval_content = brotli.decompress(returnval_content) - except brotli.error: - pass - elif(httpheaderout.get("Content-Encoding") == "zstd" and havezstd): - try: - returnval_content = zstandard.decompress(returnval_content) - except zstandard.error: - pass - elif((httpheaderout.get("Content-Encoding") == "lzma" or httpheaderout.get("Content-Encoding") == "xz") and havelzma): - try: - returnval_content = lzma.decompress(returnval_content) - except zstandard.error: - pass - elif(httpheaderout.get("Content-Encoding") == "bzip2"): - try: - returnval_content = bz2.decompress(returnval_content) - except zstandard.error: - pass - elif(httplibuse == "ftp" or httplibuse == "sftp" or httplibuse == "pysftp"): - pass - else: - returnval = False - returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size( - fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason, 'HTTPLib': httplibuse} - return returnval - - -def download_from_url_from_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1, timeout=10): - if(isinstance(httpurl, list)): - pass - elif(isinstance(httpurl, tuple)): - pass - elif(isinstance(httpurl, dict)): - httpurl = httpurl.values() - else: - httpurl = [httpurl] - listsize = len(httpurl) - listcount = 0 - returnval = [] - while(listcount < listsize): - ouputval = download_from_url(httpurl[listcount], httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, httplibuse, buffersize, sleep, timeout) - returnval.append(ouputval) - listcount += 1 - return returnval - - -def download_from_url_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix, haveaiohttp, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp - exec_time_start = time.time() - myhash = hashlib.new("sha1") - if(sys.version[0] == "2"): - myhash.update(httpurl) - myhash.update(str(buffersize)) - myhash.update(str(exec_time_start)) - if(sys.version[0] >= "3"): - myhash.update(httpurl.encode('utf-8')) - myhash.update(str(buffersize).encode('utf-8')) - myhash.update(str(exec_time_start).encode('utf-8')) - newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest()) - if(sleep < 0): - sleep = geturls_download_sleep - if(timeout <= 0): - timeout = 10 - if(httplibuse == "urllib1" or httplibuse == "urllib2" or httplibuse == "request"): - httplibuse = "urllib" - if(httplibuse == "httplib1"): - httplibuse = "httplib" - if(not haverequests and httplibuse == "requests"): - httplibuse = "urllib" - if(not haveaiohttp and httplibuse == "aiohttp"): - httplibuse = "urllib" - if(not havehttpx and httplibuse == "httpx"): - httplibuse = "urllib" - if(not havehttpx and httplibuse == "httpx2"): - httplibuse = "urllib" - if(not havehttpcore and httplibuse == "httpcore"): - httplibuse = "urllib" - if(not havehttpcore and httplibuse == "httpcore2"): - httplibuse = "urllib" - if(not havemechanize and httplibuse == "mechanize"): - httplibuse = "urllib" - if(not havepycurl and httplibuse == "pycurl"): - httplibuse = "urllib" - if(not havepycurl and httplibuse == "pycurl2"): - httplibuse = "urllib" - if(havepycurl and httplibuse == "pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")): - httplibuse = "pycurl" - if(not havepycurl and httplibuse == "pycurl3"): - httplibuse = "urllib" - if(havepycurl and httplibuse == "pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")): - httplibuse = "pycurl2" - if(havepycurl and httplibuse == "pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")): - httplibuse = "pycurl" - if(not havehttplib2 and httplibuse == "httplib2"): - httplibuse = "httplib" - if(not haveparamiko and httplibuse == "sftp"): - httplibuse = "ftp" - if(not haveparamiko and httplibuse == "pysftp"): - httplibuse = "ftp" - pretmpfilename = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, httplibuse, buffersize, sleep, timeout) - if(not pretmpfilename): - return False - with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f: - tmpfilename = f.name - try: - os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), - time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()))) - except AttributeError: - try: - os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), - time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()))) - except ValueError: - pass - except ValueError: - pass - returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get( - 'Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': pretmpfilename.get('HTTPLib')} - f.write(pretmpfilename.get('Content')) - f.close() - exec_time_end = time.time() - log.info("It took "+hms_string(exec_time_start - - exec_time_end)+" to download file.") - returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size( - os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)}) - return returnval - - -def download_from_url_file_with_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - if(isinstance(httpurl, list)): - pass - elif(isinstance(httpurl, tuple)): - pass - elif(isinstance(httpurl, dict)): - httpurl = httpurl.values() - else: - httpurl = [httpurl] - listsize = len(httpurl) - listcount = 0 - returnval = [] - while(listcount < listsize): - ouputval = download_from_url_file(httpurl[listcount], httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize, sleep, timeout) - returnval.append(ouputval) - listcount += 1 - return returnval - - -def download_from_url_to_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - global geturls_download_sleep, havezstd, havebrotli, haveaiohttp, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp - if(sleep < 0): - sleep = geturls_download_sleep - if(timeout <= 0): - timeout = 10 - if(httplibuse == "urllib1" or httplibuse == "urllib2" or httplibuse == "request"): - httplibuse = "urllib" - if(httplibuse == "httplib1"): - httplibuse = "httplib" - if(not haverequests and httplibuse == "requests"): - httplibuse = "urllib" - if(not haveaiohttp and httplibuse == "aiohttp"): - httplibuse = "urllib" - if(not havehttpx and httplibuse == "httpx"): - httplibuse = "urllib" - if(not havehttpx and httplibuse == "httpx2"): - httplibuse = "urllib" - if(not havehttpcore and httplibuse == "httpcore"): - httplibuse = "urllib" - if(not havehttpcore and httplibuse == "httpcore2"): - httplibuse = "urllib" - if(not havemechanize and httplibuse == "mechanize"): - httplibuse = "urllib" - if(not havepycurl and httplibuse == "pycurl"): - httplibuse = "urllib" - if(not havepycurl and httplibuse == "pycurl2"): - httplibuse = "urllib" - if(havepycurl and httplibuse == "pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")): - httplibuse = "pycurl" - if(not havepycurl and httplibuse == "pycurl3"): - httplibuse = "urllib" - if(havepycurl and httplibuse == "pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")): - httplibuse = "pycurl2" - if(havepycurl and httplibuse == "pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")): - httplibuse = "pycurl" - if(not havehttplib2 and httplibuse == "httplib2"): - httplibuse = "httplib" - if(not haveparamiko and httplibuse == "sftp"): - httplibuse = "ftp" - if(not havepysftp and httplibuse == "pysftp"): - httplibuse = "ftp" - if(not outfile == "-"): - outpath = outpath.rstrip(os.path.sep) - filepath = os.path.realpath(outpath+os.path.sep+outfile) - if(not os.path.exists(outpath)): - os.makedirs(outpath) - if(os.path.exists(outpath) and os.path.isfile(outpath)): - return False - if(os.path.exists(filepath) and os.path.isdir(filepath)): - return False - pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize[0], sleep, timeout) - if(not pretmpfilename): - return False - tmpfilename = pretmpfilename.get('Filename') - downloadsize = int(os.path.getsize(tmpfilename)) - fulldatasize = 0 - log.info("Moving file "+tmpfilename+" to "+filepath) - exec_time_start = time.time() - shutil.move(tmpfilename, filepath) - try: - os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), - time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()))) - except AttributeError: - try: - os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), - time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()))) - except ValueError: - pass - except ValueError: - pass - exec_time_end = time.time() - log.info("It took "+hms_string(exec_time_start - - exec_time_end)+" to move file.") - if(os.path.exists(tmpfilename)): - os.remove(tmpfilename) - returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string( - exec_time_start - exec_time_end), 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'Method': httpmethod, 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': pretmpfilename.get('HTTPLib')} - if(outfile == "-"): - pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize[0], sleep, timeout) - tmpfilename = pretmpfilename.get('Filename') - downloadsize = int(os.path.getsize(tmpfilename)) - fulldatasize = 0 - prevdownsize = 0 - exec_time_start = time.time() - with open(tmpfilename, 'rb') as ft: - f = BytesIO() - while True: - databytes = ft.read(buffersize[1]) - if not databytes: - break - datasize = len(databytes) - fulldatasize = datasize + fulldatasize - percentage = "" - if(downloadsize > 0): - percentage = str("{0:.2f}".format( - float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%" - downloaddiff = fulldatasize - prevdownsize - log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")[ - 'ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']) - prevdownsize = fulldatasize - f.write(databytes) - f.seek(0) - fdata = f.getvalue() - f.close() - ft.close() - os.remove(tmpfilename) - exec_time_end = time.time() - log.info("It took "+hms_string(exec_time_start - - exec_time_end)+" to copy file.") - returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string( - exec_time_start - exec_time_end), 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'Method': httpmethod, 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': httplibuse} - return returnval - - -def download_from_url_to_file_with_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - if(isinstance(httpurl, list)): - pass - elif(isinstance(httpurl, tuple)): - pass - elif(isinstance(httpurl, dict)): - httpurl = httpurl.values() - else: - httpurl = [httpurl] - listsize = len(httpurl) - listcount = 0 - returnval = [] - while(listcount < listsize): - ouputval = download_from_url_to_file(httpurl[listcount], httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, httplibuse, outfile, outpath, ranges, buffersize, sleep, timeout) - returnval.append(ouputval) - listcount += 1 - return returnval - - -def download_from_url_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "urllib", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "urllib", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "request3", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "httplib", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "httplib2", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "urllib3", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "requests", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "aiohttp", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "httpx", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "httpx2", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "httpcore", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "httpcore2", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "mechanize", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "pycurl", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "pycurl2", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "pycurl3", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "ftp", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "sftp", buffersize, sleep, timeout) - return returnval - - -def download_from_url_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "pysftp", buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "urllib", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "urllib", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "request3", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "httplib", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "httplib2", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "urllib3", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "requests", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "aiohttp", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "httpx", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "httpx2", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "httpcore", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "httpcore2", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "mechanize", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "pycurl", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "pycurl2", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "pycurl3", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "ftp", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "sftp", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10): - returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, - httpcookie, httpmethod, postdata, "pysftp", ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "urllib", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "request", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "urllib", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "httplib", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "httplib2", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "urllib3", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "requests", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "aiohttp", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "httpx", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "httpx2", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "httpcore", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "httpcore2", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "mechanize", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "pycurl", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "pycurl2", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "pycurl3", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "ftp", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "sftp", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_from_url_to_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10): - returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, - httpmethod, postdata, "pysftp", outfile, outpath, ranges, buffersize, sleep, timeout) - return returnval - - -def download_file_from_ftp_file(url): - urlparts = urlparse.urlparse(url) - file_name = os.path.basename(urlparts.path) - file_dir = os.path.dirname(urlparts.path) - if(urlparts.username is not None): - ftp_username = urlparts.username - else: - ftp_username = "anonymous" - if(urlparts.password is not None): - ftp_password = urlparts.password - elif(urlparts.password is None and urlparts.username == "anonymous"): - ftp_password = "anonymous" - else: - ftp_password = "" - if(urlparts.scheme == "ftp"): - ftp = FTP() - elif(urlparts.scheme == "ftps"): - ftp = FTP_TLS() - else: - return False - if(urlparts.scheme == "http" or urlparts.scheme == "https"): - return False - ftp_port = urlparts.port - if(urlparts.port is None): - ftp_port = 21 - try: - ftp.connect(urlparts.hostname, ftp_port) - except socket.gaierror: - log.info("Error With URL "+httpurl) - return False - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - ftp.login(urlparts.username, urlparts.password) - if(urlparts.scheme == "ftps"): - ftp.prot_p() - ftpfile = BytesIO() - ftp.retrbinary("RETR "+urlparts.path, ftpfile.write) - #ftp.storbinary("STOR "+urlparts.path, ftpfile.write); - ftp.close() - ftpfile.seek(0, 0) - return ftpfile - - -def download_file_from_ftp_string(url): - ftpfile = download_file_from_ftp_file(url) - return ftpfile.read() - - -def upload_file_to_ftp_file(ftpfile, url): - urlparts = urlparse.urlparse(url) - file_name = os.path.basename(urlparts.path) - file_dir = os.path.dirname(urlparts.path) - if(urlparts.username is not None): - ftp_username = urlparts.username - else: - ftp_username = "anonymous" - if(urlparts.password is not None): - ftp_password = urlparts.password - elif(urlparts.password is None and urlparts.username == "anonymous"): - ftp_password = "anonymous" - else: - ftp_password = "" - if(urlparts.scheme == "ftp"): - ftp = FTP() - elif(urlparts.scheme == "ftps"): - ftp = FTP_TLS() - else: - return False - if(urlparts.scheme == "http" or urlparts.scheme == "https"): - return False - ftp_port = urlparts.port - if(urlparts.port is None): - ftp_port = 21 - try: - ftp.connect(urlparts.hostname, ftp_port) - except socket.gaierror: - log.info("Error With URL "+httpurl) - return False - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - ftp.login(urlparts.username, urlparts.password) - if(urlparts.scheme == "ftps"): - ftp.prot_p() - ftp.storbinary("STOR "+urlparts.path, ftpfile) - ftp.close() - ftpfile.seek(0, 0) - return ftpfile - - -def upload_file_to_ftp_string(ftpstring, url): - ftpfileo = BytesIO(ftpstring) - ftpfile = upload_file_to_ftp_file(ftpfileo, url) - ftpfileo.close() - return ftpfile - - -if(haveparamiko): - def download_file_from_sftp_file(url): - urlparts = urlparse.urlparse(url) - file_name = os.path.basename(urlparts.path) - file_dir = os.path.dirname(urlparts.path) - if(urlparts.scheme == "http" or urlparts.scheme == "https"): - return False - sftp_port = urlparts.port - if(urlparts.port is None): - sftp_port = 22 - else: - sftp_port = urlparts.port - if(urlparts.username is not None): - sftp_username = urlparts.username - else: - sftp_username = "anonymous" - if(urlparts.password is not None): - sftp_password = urlparts.password - elif(urlparts.password is None and urlparts.username == "anonymous"): - sftp_password = "anonymous" - else: - sftp_password = "" - if(urlparts.scheme != "sftp"): - return False - ssh = paramiko.SSHClient() - ssh.load_system_host_keys() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - try: - ssh.connect(urlparts.hostname, port=sftp_port, - username=urlparts.username, password=urlparts.password) - except paramiko.ssh_exception.SSHException: - return False - except socket.gaierror: - log.info("Error With URL "+httpurl) - return False - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - sftp = ssh.open_sftp() - sftpfile = BytesIO() - sftp.getfo(urlparts.path, sftpfile) - sftp.close() - ssh.close() - sftpfile.seek(0, 0) - return sftpfile -else: - def download_file_from_sftp_file(url): - return False - -if(haveparamiko): - def download_file_from_sftp_string(url): - sftpfile = download_file_from_sftp_file(url) - return sftpfile.read() -else: - def download_file_from_ftp_string(url): - return False - -if(haveparamiko): - def upload_file_to_sftp_file(sftpfile, url): - urlparts = urlparse.urlparse(url) - file_name = os.path.basename(urlparts.path) - file_dir = os.path.dirname(urlparts.path) - sftp_port = urlparts.port - if(urlparts.scheme == "http" or urlparts.scheme == "https"): - return False - if(urlparts.port is None): - sftp_port = 22 - else: - sftp_port = urlparts.port - if(urlparts.username is not None): - sftp_username = urlparts.username - else: - sftp_username = "anonymous" - if(urlparts.password is not None): - sftp_password = urlparts.password - elif(urlparts.password is None and urlparts.username == "anonymous"): - sftp_password = "anonymous" - else: - sftp_password = "" - if(urlparts.scheme != "sftp"): - return False - ssh = paramiko.SSHClient() - ssh.load_system_host_keys() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - try: - ssh.connect(urlparts.hostname, port=sftp_port, - username=urlparts.username, password=urlparts.password) - except paramiko.ssh_exception.SSHException: - return False - except socket.gaierror: - log.info("Error With URL "+httpurl) - return False - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - sftp = ssh.open_sftp() - sftp.putfo(sftpfile, urlparts.path) - sftp.close() - ssh.close() - sftpfile.seek(0, 0) - return sftpfile -else: - def upload_file_to_sftp_file(sftpfile, url): - return False - -if(haveparamiko): - def upload_file_to_sftp_string(sftpstring, url): - sftpfileo = BytesIO(sftpstring) - sftpfile = upload_file_to_sftp_files(ftpfileo, url) - sftpfileo.close() - return sftpfile -else: - def upload_file_to_sftp_string(url): - return False - - -if(havepysftp): - def download_file_from_pysftp_file(url): - urlparts = urlparse.urlparse(url) - file_name = os.path.basename(urlparts.path) - file_dir = os.path.dirname(urlparts.path) - if(urlparts.scheme == "http" or urlparts.scheme == "https"): - return False - sftp_port = urlparts.port - if(urlparts.port is None): - sftp_port = 22 - else: - sftp_port = urlparts.port - if(urlparts.username is not None): - sftp_username = urlparts.username - else: - sftp_username = "anonymous" - if(urlparts.password is not None): - sftp_password = urlparts.password - elif(urlparts.password is None and urlparts.username == "anonymous"): - sftp_password = "anonymous" - else: - sftp_password = "" - if(urlparts.scheme != "sftp"): - return False - try: - pysftp.Connection(urlparts.hostname, port=sftp_port, - username=urlparts.username, password=urlparts.password) - except paramiko.ssh_exception.SSHException: - return False - except socket.gaierror: - log.info("Error With URL "+httpurl) - return False - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - sftp = ssh.open_sftp() - sftpfile = BytesIO() - sftp.getfo(urlparts.path, sftpfile) - sftp.close() - ssh.close() - sftpfile.seek(0, 0) - return sftpfile -else: - def download_file_from_pysftp_file(url): - return False - -if(havepysftp): - def download_file_from_pysftp_string(url): - sftpfile = download_file_from_pysftp_file(url) - return sftpfile.read() -else: - def download_file_from_ftp_string(url): - return False - -if(havepysftp): - def upload_file_to_pysftp_file(sftpfile, url): - urlparts = urlparse.urlparse(url) - file_name = os.path.basename(urlparts.path) - file_dir = os.path.dirname(urlparts.path) - sftp_port = urlparts.port - if(urlparts.scheme == "http" or urlparts.scheme == "https"): - return False - if(urlparts.port is None): - sftp_port = 22 - else: - sftp_port = urlparts.port - if(urlparts.username is not None): - sftp_username = urlparts.username - else: - sftp_username = "anonymous" - if(urlparts.password is not None): - sftp_password = urlparts.password - elif(urlparts.password is None and urlparts.username == "anonymous"): - sftp_password = "anonymous" - else: - sftp_password = "" - if(urlparts.scheme != "sftp"): - return False - try: - pysftp.Connection(urlparts.hostname, port=sftp_port, - username=urlparts.username, password=urlparts.password) - except paramiko.ssh_exception.SSHException: - return False - except socket.gaierror: - log.info("Error With URL "+httpurl) - return False - except socket.timeout: - log.info("Error With URL "+httpurl) - return False - sftp = ssh.open_sftp() - sftp.putfo(sftpfile, urlparts.path) - sftp.close() - ssh.close() - sftpfile.seek(0, 0) - return sftpfile -else: - def upload_file_to_pysftp_file(sftpfile, url): - return False - -if(havepysftp): - def upload_file_to_pysftp_string(sftpstring, url): - sftpfileo = BytesIO(sftpstring) - sftpfile = upload_file_to_pysftp_files(ftpfileo, url) - sftpfileo.close() - return sftpfile -else: - def upload_file_to_pysftp_string(url): - return False