Initial Commit

data.csv

@@ -0,0 +1,67 @@
+"Name","Cont","Votes"
+"Tillis Thom","12384994","100"
+"Gardner Cory","7703171","100"
+"Ernst Joni","5395587","100"
+"Cotton Tom","4749047","86"
+"Shaheen Jeanne","4019564","75"
+"Cassidy Bill","3591044","88"
+"Peters Gary","2855420","63"
+"McConnell Mitch","2802630","88"
+"Perdue David","1898215","88"
+"Cochran Thad","1362756","100"
+"Markey Ed","1162284","38"
+"Sullivan Dan","961484","86"
+"Roberts Pat","919261","100"
+"Collins Susan M","622250","63"
+"Rounds Mike","615911","83"
+"Daines Steven","613018","100"
+"Alexander Lamar","563234","88"
+"Schatz Brian","540949","38"
+"Lankford James","465436","88"
+"Sasse Ben","397981","63"
+"Graham Lindsey","292418","100"
+"Capito Shelley Moore","237947","100"
+"Warner Mark","204740","63"
+"Franken Al","176644","50"
+"Merkley Jeff","151772","38"
+"Booker Cory","147727","50"
+"Cornyn John","83884","100"
+"Scott Tim","9476","75"
+"Inhofe James M","7213","100"
+"Reed Jack","5566","25"
+"Toomey Pat","4554724","75"
+"Burr Richard","2636703","75"
+"Portman Rob","1517051","100"
+"Johnson Ron","1316985","100"
+"McCain John","1255184","88"
+"Blunt Roy","665634","100"
+"Shelby Richard C","480929","83"
+"Isakson Johnny","261822","100"
+"Murkowski Lisa","108018","88"
+"Bennet Michael F","44416","63"
+"Murray Patty","24055","38"
+"Kaine Tim","16029664","75"
+"Brown Sherrod","11753260","50"
+"Baldwin Tammy","10725266","63"
+"Tester Jon","6046591","50"
+"Nelson Bill","4274077","63"
+"Heller Dean","4177848","100"
+"Heinrich Martin","3105320","38"
+"King Angus","3087734","63"
+"Heitkamp Heidi","2659603","100"
+"Flake Jeff","2449224","63"
+"Warren Elizabeth","1445154","25"
+"Fischer Deb","889978","100"
+"McCaskill Claire","748798","75"
+"Murphy Christopher S","366322","50"
+"Menendez Robert","345348","50"
+"Hirono Mazie K","252342","38"
+"Hatch Orrin G","158462","100"
+"Stabenow Debbie","116263","63"
+"Cruz Ted","66308","100"
+"Feinstein Dianne","22908","63"
+"Gillibrand Kirsten","17934","25"
+"Cardin Ben","10689","50"
+"Klobuchar Amy","9440","63"
+"Barrasso John A","4534","100"
+"Corker Bob","2974","100"

main.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+import requests
+from bs4 import BeautifulSoup
+import csv
+from time import sleep
+URL = "https://www.opensecrets.org/outsidespending/nonprof_cands.php"
+open_secret_urls = ["https://www.opensecrets.org/outsidespending/nonprof_cands.php?cycle=2014","https://www.opensecrets.org/outsidespending/nonprof_cands.php","https://www.opensecrets.org/outsidespending/nonprof_cands.php?cycle=2012"]
+CoC_url = "https://www.uschamber.com/how-they-voted/2016"
+# Url is https://www.opensecrets.org/outsidespending/nonprof_cands.php
+DEBUG = True
+
+def clean_up(line):
+	l = str(line).strip()
+	l = l.replace(' ','_')
+	l = l.replace(',','')
+	l = l.replace('(D)','')
+	l = l.replace('(R)','')
+	l = l.replace('(I)','')
+	return l
+
+def numericify(data):
+	d = str(clean_up(data))
+	d = d.replace('_','')
+	d = d.replace('$','')
+	# print(d)
+	return d
+
+def unpack(stri):
+	s = stri.replace('_',' ')
+	s = s.strip()
+	return s
+
+def lookup(name,rows):
+	first = name.split(" ")[1]
+	last = name.split(" ")[0]
+	if first == "Steven":
+		first = "Steve" #The only special case that doesn't work with the web scraper. 
+
+	index = -1
+	if DEBUG:
+		print ("Looking up the CoC Index for " + first + " " + last + ".")
+	for row in rows:
+		if "Senate" not in str(row.contents[1].text):
+			continue
+		match = str(row.contents[1].text).split('Senate')[0].strip()
+
+		if (first in match and last in match):
+			index_str = str(row.contents[5].text).strip()
+			index_str = index_str[:len(index_str)-1]
+			index = int(index_str)
+
+	return index
+
+
+
+
+def openSecrets(url):
+
+	CoC_r = requests.get(CoC_url)
+	CoC_text_html = CoC_r.text
+	#print(text_html)
+	CoC_soup = BeautifulSoup(CoC_text_html,'html.parser')
+	CoC_rows = CoC_soup.find_all('tr', class_="views-row")
+
+	r = requests.get(url)
+	text_html = r.text
+	#print(text_html)
+	soup = BeautifulSoup(text_html,'html.parser')
+	rows = soup.find_all('tr')
+	counter = 0
+	names = []
+	contributions = []
+
+	votes = []
+	for row in rows:
+		if counter == 0:
+			counter += 1
+			continue
+		region = str(row.contents[2].text).strip()
+		name = clean_up(str(row.contents[0].text))
+		cont = numericify(row.contents[4].text)
+		if region[2] == 'S' and row.contents[14].text == "Winner":
+			vote = lookup(unpack(name),CoC_rows)
+			if not (vote == -1): #THE API IS NOT PERFECT DUE TO TIME LIMITATIONS 
+				if DEBUG:	
+					print("[+] " + unpack(name) + " has recived $" + cont + " dark money contributions and has a CoC index of " + str(vote));
+				contributions.append(cont)
+				votes.append(vote)
+				names.append(name)
+			else:
+				print("[-] " + unpack(name) + " Was not FOUND!")
+
+		counter += 1
+	print("Checking to make sure the amount of data collected is equal....")
+	print(len(names) == len(contributions))
+	print(len(votes) == len(names))
+	return (names,contributions,votes)
+
+def write(names,contributions,votes):
+	with open('data.csv', 'w') as myfile:
+	    wr = csv.writer(myfile, quoting=csv.QUOTE_ALL)
+	    wr.writerow(["Name","Cont","Votes"])
+	    i = 0;
+	    for name in names:
+	    	wr.writerow([unpack(name),contributions[i],votes[i]])
+	    	i += 1
+
+def main():
+	total_names = []
+	total_cont = []
+	total_votes = []
+	for url in open_secret_urls:
+		names, cont,votes = openSecrets(url)
+		total_names += names
+		total_cont += cont
+		total_votes += votes
+		sleep(1)
+	write(total_names,total_cont,total_votes)
+	print(len(total_names))
+
+main()

main.r

@@ -0,0 +1,5 @@
+rm(list=ls())
+setwd("/Users/alex/Desktop/GoPo/")
+data <- read.csv('data.csv')
+summary(data)
+plot(data$Cont, data$Votes, main="Dark Money contributions for a candidate v. Candidate’s pro business votes (%) as reported by the Chamber of Commerce",xlab="Dark Money contributions in $")