Skip to content

Commit 31f0e81

Browse files
icereedicereed
authored
Merge pull request #11 from icereed/redesign-tags-titles
Redesign
2 parents 4776486 + b47f062 commit 31f0e81

12 files changed

+755
-497
lines changed

‎main.go

+91-75
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,28 @@ type GetDocumentsApiResponse struct {
5151
} `json:"results"`
5252
}
5353

54+
// Document is a stripped down version of the document object from paperless-ngx.
55+
// Response payload for /documents endpoint and part of request payload for /generate-suggestions endpoint
5456
type Document struct {
55-
ID int `json:"id"`
56-
Title string `json:"title"`
57-
Content string `json:"content"`
58-
Tags []string `json:"tags"`
59-
SuggestedTitle string `json:"suggested_title,omitempty"`
60-
SuggestedTags []string `json:"suggested_tags,omitempty"`
57+
ID int `json:"id"`
58+
Title string `json:"title"`
59+
Content string `json:"content"`
60+
Tags []string `json:"tags"`
61+
}
62+
63+
// GenerateSuggestionsRequest is the request payload for generating suggestions for /generate-suggestions endpoint
64+
type GenerateSuggestionsRequest struct {
65+
Documents []Document `json:"documents"`
66+
GenerateTitles bool `json:"generate_titles,omitempty"`
67+
GenerateTags bool `json:"generate_tags,omitempty"`
68+
}
69+
70+
// DocumentSuggestion is the response payload for /generate-suggestions endpoint and the request payload for /update-documents endpoint (as an array)
71+
type DocumentSuggestion struct {
72+
ID int `json:"id"`
73+
OriginalDocument Document `json:"original_document"`
74+
SuggestedTitle string `json:"suggested_title,omitempty"`
75+
SuggestedTags []string `json:"suggested_tags,omitempty"`
6176
}
6277

6378
var (
@@ -207,14 +222,14 @@ func documentsHandler(c *gin.Context) {
207222
func generateSuggestionsHandler(c *gin.Context) {
208223
ctx := c.Request.Context()
209224

210-
var documents []Document
211-
if err := c.ShouldBindJSON(&documents); err != nil {
225+
var suggestionRequest GenerateSuggestionsRequest
226+
if err := c.ShouldBindJSON(&suggestionRequest); err != nil {
212227
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Invalid request payload: %v", err)})
213228
log.Printf("Invalid request payload: %v", err)
214229
return
215230
}
216231

217-
results, err := processDocuments(ctx, documents)
232+
results, err := generateDocumentSuggestions(ctx, suggestionRequest)
218233
if err != nil {
219234
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error processing documents: %v", err)})
220235
log.Printf("Error processing documents: %v", err)
@@ -227,7 +242,7 @@ func generateSuggestionsHandler(c *gin.Context) {
227242
// updateDocumentsHandler updates documents with new titles
228243
func updateDocumentsHandler(c *gin.Context) {
229244
ctx := c.Request.Context()
230-
var documents []Document
245+
var documents []DocumentSuggestion
231246
if err := c.ShouldBindJSON(&documents); err != nil {
232247
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Invalid request payload: %v", err)})
233248
log.Printf("Invalid request payload: %v", err)
@@ -244,50 +259,6 @@ func updateDocumentsHandler(c *gin.Context) {
244259
c.Status(http.StatusOK)
245260
}
246261

247-
func getIDMappingForTags(ctx context.Context, baseURL, apiToken string, tagsToFilter []string) (map[string]int, error) {
248-
url := fmt.Sprintf("%s/api/tags/", baseURL)
249-
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
250-
if err != nil {
251-
return nil, err
252-
}
253-
req.Header.Set("Authorization", fmt.Sprintf("Token %s", apiToken))
254-
255-
client := &http.Client{}
256-
resp, err := client.Do(req)
257-
if err != nil {
258-
return nil, err
259-
}
260-
defer resp.Body.Close()
261-
262-
if resp.StatusCode != http.StatusOK {
263-
bodyBytes, _ := io.ReadAll(resp.Body)
264-
return nil, fmt.Errorf("Error fetching tags: %d, %s", resp.StatusCode, string(bodyBytes))
265-
}
266-
267-
var tagsResponse struct {
268-
Results []struct {
269-
ID int `json:"id"`
270-
Name string `json:"name"`
271-
} `json:"results"`
272-
}
273-
274-
err = json.NewDecoder(resp.Body).Decode(&tagsResponse)
275-
if err != nil {
276-
return nil, err
277-
}
278-
279-
tagIDMapping := make(map[string]int)
280-
for _, tag := range tagsResponse.Results {
281-
for _, filterTag := range tagsToFilter {
282-
if tag.Name == filterTag {
283-
tagIDMapping[tag.Name] = tag.ID
284-
}
285-
}
286-
}
287-
288-
return tagIDMapping, nil
289-
}
290-
291262
func getDocumentsByTags(ctx context.Context, baseURL, apiToken string, tags []string) ([]Document, error) {
292263
tagQueries := make([]string, len(tags))
293264
for i, tag := range tags {
@@ -348,7 +319,7 @@ func getDocumentsByTags(ctx context.Context, baseURL, apiToken string, tags []st
348319
return documents, nil
349320
}
350321

351-
func processDocuments(ctx context.Context, documents []Document) ([]Document, error) {
322+
func generateDocumentSuggestions(ctx context.Context, suggestionRequest GenerateSuggestionsRequest) ([]DocumentSuggestion, error) {
352323
llm, err := createLLM()
353324
if err != nil {
354325
return nil, fmt.Errorf("failed to create LLM client: %v", err)
@@ -369,6 +340,9 @@ func processDocuments(ctx context.Context, documents []Document) ([]Document, er
369340
availableTagNames = append(availableTagNames, tagName)
370341
}
371342

343+
documents := suggestionRequest.Documents
344+
documentSuggestions := []DocumentSuggestion{}
345+
372346
var wg sync.WaitGroup
373347
var mu sync.Mutex
374348
errors := make([]error, 0)
@@ -385,27 +359,50 @@ func processDocuments(ctx context.Context, documents []Document) ([]Document, er
385359
content = content[:5000]
386360
}
387361

388-
suggestedTitle, err := getSuggestedTitle(ctx, llm, content)
389-
if err != nil {
390-
mu.Lock()
391-
errors = append(errors, fmt.Errorf("Document %d: %v", documentID, err))
392-
mu.Unlock()
393-
log.Printf("Error processing document %d: %v", documentID, err)
394-
return
362+
var suggestedTitle string
363+
var suggestedTags []string
364+
365+
if suggestionRequest.GenerateTitles {
366+
suggestedTitle, err = getSuggestedTitle(ctx, llm, content)
367+
if err != nil {
368+
mu.Lock()
369+
errors = append(errors, fmt.Errorf("Document %d: %v", documentID, err))
370+
mu.Unlock()
371+
log.Printf("Error processing document %d: %v", documentID, err)
372+
return
373+
}
395374
}
396375

397-
suggestedTags, err := getSuggestedTags(ctx, llm, content, suggestedTitle, availableTagNames)
398-
if err != nil {
399-
mu.Lock()
400-
errors = append(errors, fmt.Errorf("Document %d: %v", documentID, err))
401-
mu.Unlock()
402-
log.Printf("Error generating tags for document %d: %v", documentID, err)
403-
return
376+
if suggestionRequest.GenerateTags {
377+
suggestedTags, err = getSuggestedTags(ctx, llm, content, suggestedTitle, availableTagNames)
378+
if err != nil {
379+
mu.Lock()
380+
errors = append(errors, fmt.Errorf("Document %d: %v", documentID, err))
381+
mu.Unlock()
382+
log.Printf("Error generating tags for document %d: %v", documentID, err)
383+
return
384+
}
404385
}
405386

406387
mu.Lock()
407-
doc.SuggestedTitle = suggestedTitle
408-
doc.SuggestedTags = suggestedTags
388+
suggestion := DocumentSuggestion{
389+
ID: documentID,
390+
OriginalDocument: *doc,
391+
}
392+
// Titles
393+
if suggestionRequest.GenerateTitles {
394+
suggestion.SuggestedTitle = suggestedTitle
395+
} else {
396+
suggestion.SuggestedTitle = doc.Title
397+
}
398+
399+
// Tags
400+
if suggestionRequest.GenerateTags {
401+
suggestion.SuggestedTags = suggestedTags
402+
} else {
403+
suggestion.SuggestedTags = removeTagFromList(doc.Tags, tagToFilter)
404+
}
405+
documentSuggestions = append(documentSuggestions, suggestion)
409406
mu.Unlock()
410407
log.Printf("Document %d processed successfully.", documentID)
411408
}(&documents[i])
@@ -417,7 +414,17 @@ func processDocuments(ctx context.Context, documents []Document) ([]Document, er
417414
return nil, errors[0]
418415
}
419416

420-
return documents, nil
417+
return documentSuggestions, nil
418+
}
419+
420+
func removeTagFromList(tags []string, tagToRemove string) []string {
421+
filteredTags := []string{}
422+
for _, tag := range tags {
423+
if tag != tagToRemove {
424+
filteredTags = append(filteredTags, tag)
425+
}
426+
}
427+
return filteredTags
421428
}
422429

423430
func getSuggestedTags(ctx context.Context, llm llms.Model, content string, suggestedTitle string, availableTags []string) ([]string, error) {
@@ -507,7 +514,7 @@ Content:
507514
return strings.TrimSpace(strings.Trim(completion.Choices[0].Content, "\"")), nil
508515
}
509516

510-
func updateDocuments(ctx context.Context, baseURL, apiToken string, documents []Document) error {
517+
func updateDocuments(ctx context.Context, baseURL, apiToken string, documents []DocumentSuggestion) error {
511518
client := &http.Client{}
512519

513520
// Fetch all available tags
@@ -524,8 +531,13 @@ func updateDocuments(ctx context.Context, baseURL, apiToken string, documents []
524531

525532
newTags := []int{}
526533

534+
tags := document.SuggestedTags
535+
if len(tags) == 0 {
536+
tags = document.OriginalDocument.Tags
537+
}
538+
527539
// Map suggested tag names to IDs
528-
for _, tagName := range document.SuggestedTags {
540+
for _, tagName := range tags {
529541
if tagID, exists := availableTags[tagName]; exists {
530542
// Skip the tag that we are filtering
531543
if tagName == tagToFilter {
@@ -543,7 +555,11 @@ func updateDocuments(ctx context.Context, baseURL, apiToken string, documents []
543555
if len(suggestedTitle) > 128 {
544556
suggestedTitle = suggestedTitle[:128]
545557
}
546-
updatedFields["title"] = suggestedTitle
558+
if suggestedTitle != "" {
559+
updatedFields["title"] = suggestedTitle
560+
} else {
561+
log.Printf("No valid title found for document %d, skipping.", documentID)
562+
}
547563

548564
// Send the update request
549565
url := fmt.Sprintf("%s/api/documents/%d/", baseURL, documentID)

‎web-app/index.html

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
<!doctype html>
2-
<html lang="en">
2+
<html lang="en" class="dark">
33
<head>
44
<meta charset="UTF-8" />
55
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
66
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
77
<title>Paperless GPT</title>
88
</head>
9-
<body>
9+
<body class="bg-white dark:bg-gray-900 text-gray-800 dark:text-gray-200">
1010
<div id="root"></div>
1111
<script type="module" src="/src/main.tsx"></script>
1212
</body>
13-
</html>
13+
</html>

‎web-app/src/App.tsx

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import React from 'react';
2-
import DocumentProcessor from './components/DocumentProcessor';
2+
import DocumentProcessor from './DocumentProcessor';
33
import './index.css';
44

55
const App: React.FC = () => {

0 commit comments

Comments
 (0)