Merge pull request #11 from ninabina921/master

Add Wordcloud and Table for Demo Output UI

.gitignore

@@ -0,0 +1,11 @@
+__pycache__/app.cpython-38.pyc
+app.py
+predict_tweets/__pycache__/predictedData.cpython-38.pyc
+twint_integrate/__pycache__/twint_search.cpython-38.pyc
+twint_wordcloud/__pycache__/wordcloud_html.cpython-38.pyc
+twint_integrate/__pycache__/__init__.cpython-38.pyc
+predict_tweets/__pycache__/model.cpython-38.pyc
+__pycache__/app.cpython-38.pyc
+userTweets.csv
+.gitignore
+predictedTweets.csv

Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.8-slim-buster
+# Set the working directory to /app
+WORKDIR /app
+
+# copy the requirements file used for dependencies
+COPY requirements.txt .
+
+# Install any needed packages specified in requirements.txt
+RUN pip install --trusted-host pypi.python.org -r requirements.txt
+
+# Copy the rest of the working directory contents into the container at /app
+COPY . .
+
+# Run app.py when the container launches
+ENTRYPOINT ["python", "app.py"]
+# WORKDIR /python-docker
+
+# COPY requirements.txt requirements.txt
+# RUN pip3 install -r requirements.txt
+# COPY . .
+
+# CMD [ "python3", "-m" , "flask", "run", "--host=0.0.0.0"]

app.py

@@ -1,12 +1,16 @@
 from flask import Flask, render_template, request, make_response
 from twint_integrate.twint_search import search
-from model import twint_parse
+from twint_wordcloud.wordcloud_html import word_cloud
+from predict_tweets.model import twint_parse
+from predict_tweets.predictedData import predictedTweets
+
 
 app = Flask(__name__, template_folder="templates", static_folder='static')
 
 title = "detecting COVID-19 misinformation in text-based social media posts."
 footer = "This Single Page Application is powered by Flask and JQuery"
 
+
 @app.route("/", methods=["GET", "POST"])
 def index():
     if request.method == "POST":
@@ -32,8 +36,10 @@ def demo_input():
 # Demo Output Route
 @app.route("/demo-output")
 def demo_output():
-    json_data = twint_parse("userTweets.csv")
-    return render_template("demo-output.html", title=title, footer=footer)
+    wc, row_count = word_cloud('userTweets.csv')
+    twint_parse('userTweets.csv')
+    predict, df = predictedTweets("predictedTweets.csv")
+    return render_template("demo-output.html", title=title, footer=footer, wc=wc, tweets= row_count, misinfo= predict["misinfo"], handle = predict["handle"], tables=[df.to_html(index=False,classes=["data", "mystyle"])], titles=df.columns.values )
 
 if __name__ == '__main__':
     app.run(debug=True)

model.py → predict_tweets/model.py

@@ -89,17 +89,19 @@ def log_reg(features, series):  # Runs our classification model
 # TWINT DATA
 def twint_parse(csv):
     print("in twint_parse")
-    twint_df = pd.read_csv(csv)
+    twint_df = pd.read_csv(csv) # encoding='cp1252' for Windows
     twint_tweets = twint_df[['tweet']]  # Subsets the dataframe to only have the "tweet column"
 
     padded, attention_mask = pre_proc(twint_tweets['tweet'], tokenizer)  # Pre-processes the twint_dataframe
     features = encode(model, attention_mask, padded)  # Create features out of encode()
 
     twint_labels = lr_clf.predict(features)  # Saves predicted labels as an array
     twint_df['label'] = twint_labels  # Adds a new column to twint_df for predicted labels
-    twint_json = pd.DataFrame.to_json(twint_df)  # Converts dataframe into JSON
+
+    twint_df.to_csv("predictedTweets.csv", index=False)  # Converts dataframe into CSV
+    # twint_json = pd.Dataframe.to_json(twint_df)  # Converts dataframe into CSV
 
-    return twint_json
+    # return twint_json
 
 
 

predict_tweets/predictedData.py

@@ -0,0 +1,21 @@
+# importing pandas as pd
+import pandas as pd
+
+def predictedTweets(csv):
+    with open(csv, encoding="utf8", errors='ignore') as f:
+            df = pd.read_csv(f)
+    # Get Values for Prediction Results
+    print("Shape", df.shape)
+    cols = ["username", "tweet", "label"]
+    df_small = df[cols]
+    df_small = df_small[df_small["label"] == 1]
+
+    # Misinformation Dict for Data
+    prediction_data = {}
+
+    prediction_data['misinfo'] = df_small.shape[0]
+    prediction_data['handle'] = df["username"][0]
+
+#     print(df_small)
+
+    return prediction_data, df_small

requirements.txt

@@ -0,0 +1,11 @@
+Flask==2.0.2
+matplotlib==3.5.1
+numpy==1.22.2
+pandas==1.4.1
+Pillow==9.0.1
+scikit_learn==1.0.2
+torch==1.10.2
+transformers==4.17.0
+wordcloud==1.8.1
+git+https://github.com/twintproject/twint.git@origin/master#egg=twint
+

static/css/index.css

@@ -221,4 +221,38 @@ div.demo-div-right p.demo-text {
   .col-6 {
     width: inherit;
   }
+}
+
+/* table {
+  background-color: #2d90a6;
+}
+
+th {
+  text-align: center;
+} */
+
+.mystyle {
+  font-size: 11pt; 
+  font-family: 'Red Hat Text', sans-serif;
+  border-collapse: collapse; 
+  border: 1px solid #2d90a6;
+
+}
+
+.mystyle td, th {
+  padding: 5px;
+  text-align: center;
+}
+
+.mystyle tr {
+  background: #2d90a6;
+}
+
+.mystyle tr:hover {
+  background: #2c545e;
+  cursor: pointer;
+}
+
+div.col-6.demo img {
+  padding: 4em 1em;
 }

templates/demo-output.html

@@ -12,8 +12,11 @@
     <link href="https://fonts.googleapis.com/css2?family=Red+Hat+Text:wght@400;500&display=swap" rel="stylesheet">
 
 
-    <!-- Bootstrap core CSS -->
-    <link rel= "stylesheet" href= "/static/css/bootstrap.min.css"/>
+      <!-- Bootstrap core CSS -->
+      <link rel= "stylesheet" href= "/static/css/bootstrap.min.css"/>
+      <!-- Custom styles for this template -->
+      <link rel= "stylesheet" href= "/static/css/index.css"/>
+      <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
 
     <style>
       .bd-placeholder-img {
@@ -35,7 +38,7 @@
     <!-- Custom styles for this template -->
     <link rel= "stylesheet" href= "/static/css/index.css"/>
   </head>
-  <body class="d-flex h-100 text-center text-white">
+  <body class="d-flex h-100 text-center text-white bg-dark">
 
 <div class="container d-flex p-3 mx-auto flex-column">
   <header class="mb-auto">
@@ -48,14 +51,20 @@ <h3 class="float-md-start">{{title}}</h3>
     <div class="row">
       <div class="col-6 overview">
         <div class="section intro">
-        <div class="row about-text">Prediction Output</div>
-        <div class="input-group mb-3 handle-input">
-          <span class="input-group-text" id="basic-addon1">@</span>
-          <input type="text" class="form-control" placeholder="Username" aria-label="Username" aria-describedby="basic-addon1">
-        </div>
+          <p class="row about-text">Prediction Output</p>
+          <p class="about-text">Out of <b>{{tweets}}</b> tweets for <b>@{{handle}}</b>, our model detected <b>{{misinfo}}</b> tweet(s) that may contain misinformation. See labeled tweets below.</p>
+        <div class="row about-text">
+        <table class="mystyle">
+          {% for table in tables %}
+            {{titles[loop.index]}}
+            {{ table|safe }}
+          {% endfor %}
+        </table>
+      </div>
       </div>
       </div>
       <div class="col-6 demo">
+        <image class= "wc" src="{{wc}}" ></image>
       </div>
     </div>
 

twint_wordcloud/wordcloud_html.py

@@ -0,0 +1,25 @@
+from sqlite3 import Row
+from wordcloud import WordCloud
+import pandas as pd
+from PIL import Image
+import matplotlib.pyplot as plt
+import io
+import urllib
+import base64
+import re
+
+def word_cloud(csv):
+    with open(csv, encoding="utf8", errors='ignore') as f:
+        df = pd.read_csv(f)
+    if df.shape[0] == 0:
+        return '../static/img/404.png'
+    else:
+        df['content'] = df['tweet'].apply(lambda x: re.split('https:\/\/.*', str(x))[0])
+        row_count = df.shape[0]
+        txt = ' '.join(df['content'])
+        wc = WordCloud(width = 400, height = 200, random_state=1, background_color='black', colormap='Pastel1', collocations=False).generate(txt)
+        buffer = io.BytesIO()
+        wc.to_image().save(buffer, 'png')
+        b64 = base64.b64encode(buffer.getvalue())
+        image_64 = 'data:image/png;base64,' + urllib.parse.quote(b64)
+        return image_64, row_count