Skip to content

Commit

Permalink
feat(ui): reorganize e-mail lists and data training
Browse files Browse the repository at this point in the history
  • Loading branch information
@ozangulle
ozangulle committed Jan 7, 2025
1 parent 3c2ecdc commit 5eb7385
Show file tree
Hide file tree
Showing 4 changed files with 172 additions and 61 deletions.
3 changes: 2 additions & 1 deletion resources/templates/base.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
<script src="https://cdn.jsdelivr.net/npm/vega-lite@5" type="text/javascript"></script>
<script src="https://cdn.jsdelivr.net/npm/vega-embed@6" type="text/javascript"></script>
<script src="https://cdn.tailwindcss.com" type="text/javascript"></script>
<script src="https://cdn.jsdelivr.net/npm/flowbite@2.5.2/dist/flowbite.min.js"></script>
<link href="/css/custom.css" rel="stylesheet" type="text/css">
<link href="https://cdn.jsdelivr.net/npm/flowbite@2.5.2/dist/flowbite.min.css" rel="stylesheet" />
<meta charset="UTF-8">
</head>
<meta content="width=device-width, initial-scale=1" name="viewport">
Expand All @@ -15,7 +17,6 @@ <h1 class="uppercase text-lg my-8">plauna</h1>
<a class="nav-button" href="/admin">Admin</a>
<a class="nav-button" href="/statistics">Statistics</a>
<a class="nav-button" href="/emails">Emails</a>
<a class="nav-button" href="/training">Training Data</a>
<a class="nav-button" href="/watchers">Watchers</a>
</nav>
<main class="">
Expand Down
142 changes: 132 additions & 10 deletions resources/templates/emails.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,31 +4,145 @@


<h3 class="uppercase text-lg my-8">{{header}}</h3>
<div>
<p class="">Total e-mail count: {{page.total}}</p>
<div class="my-4">

<div id="accordion-collapse" data-accordion="collapse">
<h2 id="accordion-collapse-heading-1">
<button type="button" class="flex items-center justify-between w-full p-3 font-medium rtl:text-right text-gray-500 hover:text-white border border-b-0 border-gray-200 rounded-t-xl hover:bg-gray-700 gap-3" data-accordion-target="#accordion-collapse-body-1" aria-expanded="false" aria-controls="accordion-collapse-body-1">
<span>Enrichment Options</span>
<svg data-accordion-icon class="w-3 h-3 rotate-180 shrink-0" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 10 6">
<path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 5 5 1 1 5"/>
</svg>
</button>
</h2>
<div id="accordion-collapse-body-1" class="hidden" aria-labelledby="accordion-collapse-heading-1">
<button id="filterDropdown" data-dropdown-toggle="filter" class="text-white bg-gray-700 hover:bg-gray-800 font-medium rounded-lg text-sm px-5 py-2.5 text-center inline-flex items-center" type="button">Advanced Filter: {{page.filter}}
<svg class="w-2.5 h-2.5 ms-3" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 10 6">
<path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="m1 1 4 4 4-4"/>
</svg>
</button>

<div id="filter" class="z-10 hidden bg-white divide-y divide-gray-100 rounded-lg shadow w-44 dark:bg-gray-700">
<ul class="py-2 text-sm text-gray-700 dark:text-gray-200" aria-labelledby="filterDropdown">
<li>
<a href="/emails?page={{page.page}}&page-size={{page.size}}&filter=all" class="block px-4 py-2 hover:bg-gray-100 dark:hover:bg-gray-600 dark:hover:text-white">All</a>
</li>
<li>
<a href="/emails?page={{page.page}}&page-size={{page.size}}&filter=enriched-only" class="block px-4 py-2 hover:bg-gray-100 dark:hover:bg-gray-600 dark:hover:text-white">Enriched E-mails Only</a>
</li>
<li>
<a href="/emails?page={{page.page}}&page-size={{page.size}}&filter=without-category" class="block px-4 py-2 hover:bg-gray-100 dark:hover:bg-gray-600 dark:hover:text-white">E-mails without Category Only</a>
</li>
</ul>
</div>
</div>

<h2 id="accordion-collapse-heading-2">
<button type="button" class="flex items-center justify-between w-full p-3 font-medium rtl:text-right text-gray-500 hover:text-white border border-b-0 border-gray-200 rounded-t-xl hover:bg-gray-700 gap-3" data-accordion-target="#accordion-collapse-body-2" aria-expanded="false" aria-controls="accordion-collapse-body-1">
<span>Data Training</span>
<svg data-accordion-icon class="w-3 h-3 rotate-180 shrink-0" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 10 6">
<path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M9 5 5 1 1 5"/>
</svg>
</button>
</h2>
<div id="accordion-collapse-body-2" class="hidden" aria-labelledby="accordion-collapse-heading-2">
<div class="inline-flex rounded-md shadow-sm" role="group">
<form action="/training/new" method="post">
<input hidden="true" name="redirect-url" type="text" value="/emails">
<input class="px-4 py-2 ring-2 text-white bg-gray-800 hover:bg-gray-900 focus:outline-none focus:ring-4 focus:ring-gray-300 font-medium rounded-s-lg text-sm" type="submit" value="Generate new data">
</form>
<form action="/training" method="post">
<input hidden="true" name="redirect-url" type="text" value="/emails">
<input class="px-4 py-2 ring-2 text-white bg-gray-800 hover:bg-gray-900 focus:outline-none focus:ring-4 focus:ring-gray-300 font-medium rounded-e-lg text-sm" type="submit" value="Train with Existing Data">
</form>
</div>
</div>
</div>


<div class="flex flex-col items-center">
<span class="text-sm text-gray-700">
Showing page <span class="font-semibold text-gray-900">{{page.page}}</span> of <span class="font-semibold text-gray-900">{{page.total-pages}}</span>
</span>
<div class="inline-flex mt-2 my-2 xs:mt-0">

{% if page.page > 1 %}
<a class="rounded-none px-4 py-2 ring-2 ring-purple-900 hover:bg-purple-900 hover:text-white" href="/emails?page={{page.page|add:-1}}&page-size={{page.size}}">Prev</a>
<a class="flex items-center justify-center px-3 h-8 text-sm font-medium text-white bg-gray-700 rounded-s hover:bg-gray-900"
href="/emails?page={{page.page|add:-1}}&page-size={{page.size}}&filter={{page.filter}}">
<svg class="w-3.5 h-3.5 me-2 rtl:rotate-180" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 14 10">
<path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 5H1m0 0 4 4M1 5l4-4"/>
</svg>
Prev
</a>
{% else %}
<a class="disabled:text-indigo-100 rounded-lg px-4 py-2 disabled:bg-sky-500" href="#">Prev</a>
<a class="flex items-center justify-center px-3 h-8 text-sm font-medium text-white bg-gray-400 rounded-s hover:bg-gray-400" href="#">
<svg class="w-3.5 h-3.5 me-2 rtl:rotate-180" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 14 10">
<path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 5H1m0 0 4 4M1 5l4-4"/>
</svg>
Prev
</a>
{% endif %}
<span class="mx-2">{{page.page}}</span>
{% if page.page <= page.last-page %}
<a class="rounded-none px-4 py-2 ring-2 ring-purple-900 hover:bg-purple-900 hover:text-white" href="/emails?page={{page.page|add:1}}&page-size={{page.size}}">Next</a>
<a class="flex items-center justify-center px-3 h-8 text-sm font-medium text-white bg-gray-700 border-0 border-s rounded-e hover:bg-gray-900"
href="/emails?page={{page.page|add:1}}&page-size={{page.size}}&filter={{page.filter}}">
Next
<svg class="w-3.5 h-3.5 ms-2 rtl:rotate-180" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 14 10">
<path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M1 5h12m0 0L9 1m4 4L9 9"/>
</svg>
</a>
{% else %}
<a class="disabled:text-indigo-100 rounded-lg px-4 py-2 disabled:bg-sky-500" href="#">Next</a>
<a class="flex items-center justify-center px-3 h-8 text-sm font-medium text-white bg-gray-400 border-0 border-s rounded-e hover:bg-gray-400" href="#">
Next
<svg class="w-3.5 h-3.5 ms-2 rtl:rotate-180" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 14 10">
<path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M1 5h12m0 0L9 1m4 4L9 9"/>
</svg>
</a>
{% endif %}
</div>


<div>
<button id="pageSizeDropdown" data-dropdown-toggle="dropdown" class="text-white bg-gray-700 hover:bg-gray-800 font-medium rounded-lg text-sm px-5 py-2.5 text-center inline-flex items-center" type="button">Page Size: {{page.size}}
<svg class="w-2.5 h-2.5 ms-3" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 10 6">
<path stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="m1 1 4 4 4-4"/>
</svg>
</button>

<div id="dropdown" class="z-10 hidden bg-white divide-y divide-gray-100 rounded-lg shadow w-44 dark:bg-gray-700">
<ul class="py-2 text-sm text-gray-700 dark:text-gray-200" aria-labelledby="pageSizeDropdown">
<li>
<a href="/emails?page={{page.page}}&page-size=10&filter={{page.filter}}" class="block px-4 py-2 hover:bg-gray-100 dark:hover:bg-gray-600 dark:hover:text-white">10</a>
</li>
<li>
<a href="/emails?page={{page.page}}&page-size=20&filter={{page.filter}}" class="block px-4 py-2 hover:bg-gray-100 dark:hover:bg-gray-600 dark:hover:text-white">20</a>
</li>
<li>
<a href="/emails?page={{page.page}}&page-size=30&filter={{page.filter}}" class="block px-4 py-2 hover:bg-gray-100 dark:hover:bg-gray-600 dark:hover:text-white">30</a>
</li>
</ul>
</div>
</div>

</div>

<form action="/metadata" id="emails" method="post"></form>
<input form="emails" hidden="true" name="redirect-url" type="text" value="/emails?page={{page.page}}&page-size={{page.size}}&filter={{page.filter}}">
<div>
{% for email in emails %}
<input form="emails" hidden="true" name="message-id" type="text" value="{{email.header.message-id}}">
<input form="emails" hidden="true" name="language-confidence" type="text" value="1">
<input form="emails" hidden="true" name="category-confidence" type="text" value="1">
{% endfor %}

<table class="table-auto">
<thead><tr class="border-b p-4 pl-8 pt-0 pb-3 text-slate-700 text-left">
<th><strong>Date</strong></th>
<th><strong>Subject</strong></th>
<th><strong>From</strong></th>
<th><strong>To</strong></th>
<th><strong>Language</strong></th>
<th><strong>Lang. Confidence</strong></th>
<th><strong>Category</strong></th>
<th><strong>Cat. Confidence</strong></th>
</tr>
</thead>
<tbody>
Expand All @@ -38,10 +152,18 @@ <h3 class="uppercase text-lg my-8">{{header}}</h3>
<td><a class="block mr-4" href="/emails/{{email.header.message-id|urlescape}}">{{email.header.subject}}</a></td>
<td><a class="block mr-4" href="/emails/{{email.header.message-id|urlescape}}">{{email.participants|concat-senders}}</a></td>
<td><a class="block mr-4" href="/emails/{{email.header.message-id|urlescape}}">{{email.participants|concat-receivers}}</a></td>
<td><a class="block mr-4" href="/emails/{{email.header.message-id|urlescape}}">{{email.metadata.language}}</a></td>
<td><a class="block mr-4" href="/emails/{{email.header.message-id|urlescape}}">{{email.metadata.category}}</a></td>
<td><input class="mr-4" form="emails" name="language" type="text" value="{{email.metadata.language}}"></td>
<td><a class="block mr-4" href="/emails/{{email.header.message-id|urlescape}}">{{email.metadata.language-confidence|double-format-nillable:10}}</a></td>
<td><select class="block mr-4 bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg block w-full p-2.5 w-full" form="emails" name="category">
{% for category in categories %}
<option value="{{category.id}}" {% if email.metadata.category = category.name %} selected {% endif %}>{{category.name}}</option>
{% endfor %}
</select></td>
<td><a class="block mr-4" href="/emails/{{email.header.message-id|urlescape}}">{{email.metadata.category-confidence|double-format-nillable:10}}</a></td>
</tr>
{% endfor %}
<tr class="border-b p-4 pl-8 pt-0 pb-3 text-slate-700 text-left"><td>
<button class="px-4 py-2 ring-2 text-white bg-gray-800 hover:bg-gray-900 focus:outline-none focus:ring-4 focus:ring-gray-300 font-medium rounded-lg text-sm" form="emails">Batch Update</button></td></tr>
</tbody>
</table>
</div>
Expand Down
12 changes: 10 additions & 2 deletions src/plauna/markup.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,18 @@

(add-filter! :concat-bcc (partial concat-contacts :bcc))

(defn list-emails [emails page-info]
(add-filter! :double-format-nillable (fn [n & [decimal-places]]
(if (nil? n)
0
(let [n (double n)]
(format (str "%." (if decimal-places decimal-places "1") "f")
n)))
))

(defn list-emails [emails page-info categories]
(let [last-page {:last-page (quot (:total page-info) (:size page-info))}
emails-with-java-date (map #(update-in % [:header :date] timestamp->date) emails)]
(render-file "emails.html" {:emails emails-with-java-date :page (conj page-info last-page) :header "Emails"})))
(render-file "emails.html" {:emails emails-with-java-date :page (conj page-info last-page) :header "Emails" :categories categories})))

(defn list-email-contents [email-data categories]
(render-file "email.html" {:email (update-in email-data [:header :date] timestamp->date) :categories categories}))
Expand Down
76 changes: 28 additions & 48 deletions src/plauna/server.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,6 @@

(def html-headers {"Content-Type" "text/html; charset=UTF-8"})

(defn get-param-or [request param or]
(let [param-in-request (param request)]
(if (nil? param-in-request)
or
(Integer/parseInt param-in-request))))

(defn interleave-all [& seqs]
(reduce (fn [acc index] (into acc (map #(get % index) seqs)))
[]
Expand Down Expand Up @@ -179,11 +173,24 @@
:group-by [:interval]
:where where-clause})))

(defn paginated-strictly-enriched-emails [page]
(db/fetch-data {:entity :enriched-email :strict true :page page} {:where [:and [:<> :category nil] [:<> :language nil]] :order-by [[:category-modified :desc]]}))

(defn enriched-email-by-message-id [id] (first (db/fetch-data {:entity :enriched-email :strict false} {:where [:= :message-id id]})))

;; TODO change name template
(def emails-template {:page-size {:default 20 :type-fn Integer/parseInt}
:page {:default 1 :type-fn Integer/parseInt}
:filter {:default "all" :type-fn (fn [x] x)}})

(defn template->request-parameters [template]
(fn [rp] (reduce (fn [acc [k v]] (if (contains? rp k)
(conj acc {k ((:type-fn v) (get rp k))})
(conj acc {k (:default v)})))
{} template)))

(defn filter->sql-clause [filter]
(cond
(= filter "enriched-only") {:where [:and [:<> :metadata.category nil] [:<> :metadata.language nil]] :order-by [[:date :desc]]}
(= filter "without-category") {:where [:= :metadata.category nil] :order-by [[:date :desc]]}
:else {:order-by [[:date :desc]]}))

(comp/defroutes routes

Expand Down Expand Up @@ -298,17 +305,7 @@
{:years years
:selected-interval selected-interval
:selected-year (get params :year)}))))
(comp/GET "/training" {request-params :params}
(let [page (get-param-or request-params :page 1)
page-size (get-param-or request-params :page-size 10)
categories (db/get-categories)
result (paginated-strictly-enriched-emails {:size page-size :page page})
languages (db/get-languages)
size (:size result)
page (:page result)
total (:total result)]
(success-html-with-body (markup/email-training-page (:data result) categories languages {:size size :page page :total total}))))


(comp/POST "/metadata" request
(save-metadata-form (:params request))
(redirect-request request))
Expand All @@ -317,37 +314,20 @@
(write-emails-to-training-files-and-train)
(redirect-request request))

(comp/GET "/training/new" {params :params}
(let [page (get-param-or params :page 1)
page-size (get-param-or params :page-size 20)
language (get params :language)
sql-clause (if (nil? language)
{:where [:and [:<> :language nil] [:= :category nil]]}
{:where [:and [:= :language language] [:= :category nil]]})
untrained-emails (db/fetch-data {:entity :enriched-email :page (page/page-request page page-size) :strict false} sql-clause)
(comp/POST "/training/new" request
(let [n (get (:route-params request) :new 20)]
(categorize-uncategorized-n-emails n)
(redirect-request request)))

(comp/GET "/emails" {params :params}
(let [parse-fn (template->request-parameters emails-template)
{page-size :page-size page :page filter :filter} (parse-fn params)
categories (conj (db/get-categories) {:id -1 :name "n/a"})
page-info {:size (:size untrained-emails) :page (:page untrained-emails) :total (:total untrained-emails)}
languages (db/get-languages)]
sql-clause (filter->sql-clause filter)
result (db/fetch-data {:entity :enriched-email :strict false :page (page/page-request page page-size)} sql-clause)]
{:status 200
:header html-headers
:body (markup/email-new-training-page untrained-emails categories languages page-info)}))

(comp/POST "/training/new" {route-params :route-params}
(let [n (get route-params :new 20)]
(write-emails-to-training-files-and-train)
(categorize-uncategorized-n-emails n)
{:status 301
:header {"Location" "/training"}}))

(comp/GET "/emails" [page page-size]
(if (or (nil? page) (nil? page-size))
(redirect "/emails?page=1&page-size=20")
(let [parsed-page (as-int page)
parsed-page-size (as-int page-size)
result (db/fetch-data {:entity :enriched-email :strict false :page (page/page-request parsed-page parsed-page-size)} {:order-by [[:date :desc]]})]
{:status 200
:header html-headers
:body (markup/list-emails (:data result) {:size parsed-page-size :page (:page result) :total (:total result)})})))
:body (markup/list-emails (:data result) {:filter filter :total-pages (inc (int (clojure.math/ceil (quot (:total result) page-size)))) :size page-size :page (:page result) :total (:total result)} categories)}))

(comp/GET "/emails/:id" [id]
(let [decoded-id (url-decode id)
Expand Down

0 comments on commit 5eb7385

Please sign in to comment.