title added

fortune_http_unseen.py

@@ -12,17 +12,34 @@
 LOG_FILE = LOCAL_DIR / 'http_unseen.txt'
 DISALLOW_RE = re.compile(
     r'^User-agent:\s+\*$[\n\r]+^Disallow:\s+/$', re.IGNORECASE | re.MULTILINE)
+TITLE_RE = re.compile(r'<title[^>]*>([^<]+)', re.IGNORECASE)
+H1_RE = re.compile(r'<h1[^>]*>([^<]+)', re.IGNORECASE)
 
 
 def check_host(ip, lock):
     with HTTPConnection(ip, 80, timeout=1.5) as c:
         response = c.get('/robots.txt')
-        if response.ok:
-            if DISALLOW_RE.findall(response.body):
-                with lock:
-                    print(ip)
-                    with LOG_FILE.open('a') as f:
-                        f.write('%s\n' % ip)
+
+        if not response.ok:
+            return
+
+        if not DISALLOW_RE.findall(response.body):
+            return
+
+    # coz connection closes often after 1st request
+    with HTTPConnection(ip, 80) as c:
+        page = c.get('/').body
+
+        t_match = TITLE_RE.findall(page)
+        h_match = H1_RE.findall(page)
+
+        title = t_match[0] if t_match else h_match[0] if h_match else '-'
+        title = title.strip().replace('\n', ' ').replace('\r', '')
+
+        with lock:
+            print(ip, title)
+            with LOG_FILE.open('a') as f:
+                f.write('%s %s\n' % (ip, title))
 
 
 def main(c=10_000_000, w=16):

lib/net.py

@@ -212,8 +212,14 @@ def get(self, url):
 
         try:
             connection.sendall(str(request).encode())
-            # TODO: get overall response
-            data = connection.recv(1024).decode(errors='ignore')
+            maxlen = 1024 * 1024 * 128
+            data_bytes = b''
+            while True:
+                d = connection.recv(1024)
+                if not d or len(data_bytes) > maxlen:
+                    break
+                data_bytes += d
+            data = data_bytes.decode(errors='ignore')
             if data.startswith('HTTP/'):
                 return Response(data)
         except OSError: