Merge pull request #114 from pzaino/develop

Some code refactoring to clean up a bit

services/api/console.go

@@ -78,7 +78,7 @@ func addSource(sqlQuery string, params addSourceRequest) (ConsoleResponse, error
 		defaultConfig := map[string]string{}
 		defaultConfigJSON, err := json.Marshal(defaultConfig)
 		if err != nil {
-			return results, fmt.Errorf("Failed to marshal default Config: %w", err)
+			return results, fmt.Errorf("failed to marshal default Config: %w", err)
 		}
 		params.Config = string(defaultConfigJSON)
 	} else {
@@ -87,12 +87,12 @@ func addSource(sqlQuery string, params addSourceRequest) (ConsoleResponse, error
 		var jsonRaw map[string]interface{}
 		if err := json.Unmarshal([]byte(params.Config), &jsonRaw); err != nil {
 			// Handle invalid JSON
-			return results, fmt.Errorf("Config field contains invalid JSON: %w", err)
+			return results, fmt.Errorf("config field contains invalid JSON: %w", err)
 		}
 		// Re-marshal to ensure the JSON is in a standardized format (optional)
 		configJSON, err := json.Marshal(jsonRaw)
 		if err != nil {
-			return results, fmt.Errorf("Failed to marshal Config field: %w", err)
+			return results, fmt.Errorf("failed to marshal Config field: %w", err)
 		}
 		params.Config = string(configJSON)
 	}

services/api/helpers.go

@@ -10,13 +10,14 @@ import (
 )
 
 // handleErrorAndRespond encapsulates common error handling and JSON response logic.
-func handleErrorAndRespond(w http.ResponseWriter, err error, results interface{}, errMsg string, errCode int) {
+func handleErrorAndRespond(w http.ResponseWriter, err error, results interface{}, errMsg string, errCode int, successCode int) {
 	if err != nil {
 		cmn.DebugMsg(cmn.DbgLvlDebug3, errMsg, err)
 		http.Error(w, err.Error(), errCode)
 		return
 	}
 	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(successCode) // Explicitly set the success status code
 	if err := json.NewEncoder(w).Encode(results); err != nil {
 		// Log the error and send a generic error message to the client
 		cmn.DebugMsg(cmn.DbgLvlDebug3, "Error encoding JSON response: %v", err)

services/api/main.go

@@ -81,11 +81,13 @@ func initAPIv1() {
 	searchHandlerWithMiddlewares := SecurityHeadersMiddleware(RateLimitMiddleware(http.HandlerFunc(searchHandler)))
 	scrImgSrchHandlerWithMiddlewares := SecurityHeadersMiddleware(RateLimitMiddleware(http.HandlerFunc(scrImgSrchHandler)))
 	netInfoHandlerWithMiddlewares := SecurityHeadersMiddleware(RateLimitMiddleware(http.HandlerFunc(netInfoHandler)))
+	httpInfoHandlerWithMiddlewares := SecurityHeadersMiddleware(RateLimitMiddleware(http.HandlerFunc(httpInfoHandler)))
 	addSourceHandlerWithMiddlewares := SecurityHeadersMiddleware(RateLimitMiddleware(http.HandlerFunc(addSourceHandler)))
 	removeSourceHandlerWithMiddlewares := SecurityHeadersMiddleware(RateLimitMiddleware(http.HandlerFunc(removeSourceHandler)))
 
 	http.Handle("/v1/search", searchHandlerWithMiddlewares)
 	http.Handle("/v1/netinfo", netInfoHandlerWithMiddlewares)
+	http.Handle("/v1/httpinfo", httpInfoHandlerWithMiddlewares)
 	http.Handle("/v1/screenshot", scrImgSrchHandlerWithMiddlewares)
 
 	if config.API.EnableConsole {
@@ -120,61 +122,79 @@ func SecurityHeadersMiddleware(next http.Handler) http.Handler {
 
 // searchHandler handles the traditional search requests
 func searchHandler(w http.ResponseWriter, r *http.Request) {
+	successCode := http.StatusOK
 	query, err := extractQueryOrBody(r)
 	if err != nil {
-		handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in search request", http.StatusBadRequest)
+		handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in search request", http.StatusBadRequest, successCode)
 		return
 	}
 
 	results, err := performSearch(query)
-	handleErrorAndRespond(w, err, results, "Error performing search: %v", http.StatusInternalServerError)
+	handleErrorAndRespond(w, err, results, "Error performing search: %v", http.StatusInternalServerError, successCode)
 }
 
 // scrImgSrchHandler handles the search requests for screenshot images
 func scrImgSrchHandler(w http.ResponseWriter, r *http.Request) {
+	successCode := http.StatusOK
 	query, err := extractQueryOrBody(r)
 	if err != nil {
-		handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in screenshot search request", http.StatusBadRequest)
+		handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in screenshot search request", http.StatusBadRequest, successCode)
 		return
 	}
 
 	results, err := performScreenshotSearch(query, getQType(r.Method != "POST"))
 
-	handleErrorAndRespond(w, err, results, "Error performing screenshot search: %v", http.StatusInternalServerError)
+	handleErrorAndRespond(w, err, results, "Error performing screenshot search: %v", http.StatusInternalServerError, successCode)
 }
 
 // netInfoHandler handles the network information requests
 func netInfoHandler(w http.ResponseWriter, r *http.Request) {
+	successCode := http.StatusOK
 	query, err := extractQueryOrBody(r)
 	if err != nil {
-		handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in netinfo search request", http.StatusBadRequest)
+		handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in netinfo search request", http.StatusBadRequest, successCode)
 		return
 	}
 
 	results, err := performNetInfoSearch(query, getQType(r.Method != "POST"))
-	handleErrorAndRespond(w, err, results, "Error performing netinfo search: %v", http.StatusInternalServerError)
+	handleErrorAndRespond(w, err, results, "Error performing netinfo search: %v", http.StatusInternalServerError, successCode)
+}
+
+// httpInfoHandler handles the http information requests
+func httpInfoHandler(w http.ResponseWriter, r *http.Request) {
+	successCode := http.StatusOK
+	query, err := extractQueryOrBody(r)
+	if err != nil {
+		handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in httpinfo search request", http.StatusBadRequest, successCode)
+		return
+	}
+
+	results, err := performHTTPInfoSearch(query, getQType(r.Method != "POST"))
+	handleErrorAndRespond(w, err, results, "Error performing httpinfo search: %v", http.StatusInternalServerError, successCode)
 }
 
 // addSourceHandler handles the addition of new sources
 func addSourceHandler(w http.ResponseWriter, r *http.Request) {
+	successCode := http.StatusCreated
 	query, err := extractQueryOrBody(r)
 	if err != nil {
-		handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in addSource request", http.StatusBadRequest)
+		handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in addSource request", http.StatusBadRequest, successCode)
 		return
 	}
 
 	results, err := performAddSource(query, getQType(r.Method != "POST"))
-	handleErrorAndRespond(w, err, results, "Error performing addSource: %v", http.StatusInternalServerError)
+	handleErrorAndRespond(w, err, results, "Error performing addSource: %v", http.StatusInternalServerError, successCode)
 }
 
 // removeSourceHandler handles the removal of sources
 func removeSourceHandler(w http.ResponseWriter, r *http.Request) {
+	successCode := http.StatusNoContent
 	query, err := extractQueryOrBody(r)
 	if err != nil {
-		handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in removeSource request", http.StatusBadRequest)
+		handleErrorAndRespond(w, err, nil, "Missing parameter 'q' in removeSource request", http.StatusBadRequest, successCode)
 		return
 	}
 
 	results, err := performRemoveSource(query, getQType(r.Method != "POST"))
-	handleErrorAndRespond(w, err, results, "Error performing removeSource: %v", http.StatusInternalServerError)
+	handleErrorAndRespond(w, err, results, "Error performing removeSource: %v", http.StatusInternalServerError, successCode)
 }

services/api/search_engine.go

@@ -34,6 +34,7 @@ const (
 	sqlQueryParamsLabel = "SQL params: %v"
 	dbConnErrorLabel    = "Error connecting to the database: %v"
 	SearchLabel         = "Performing search for: %s"
+	noQueryProvided     = "no query provided"
 )
 
 func tokenize(input string) []string {
@@ -459,7 +460,7 @@ func parseScreenshotQuery(input string) (string, []interface{}, error) {
 	if len(req.URL) > 0 {
 		query = "%" + req.URL + "%"
 	} else {
-		return "", nil, errors.New("no query provided")
+		return "", nil, errors.New(noQueryProvided)
 	}
 
 	// Parse the user input
@@ -600,7 +601,7 @@ func parseNetInfoQuery(input string) (string, []interface{}, error) {
 	if len(req.Title) > 0 {
 		query = req.Title
 	} else {
-		return "", nil, errors.New("no query provided")
+		return "", nil, errors.New(noQueryProvided)
 	}
 
 	// Parse the user input
@@ -622,3 +623,141 @@ func parseNetInfoQuery(input string) (string, []interface{}, error) {
 
 	return sqlQuery, sqlParams, nil
 }
+
+// performNetInfoSearch performs a search for network information.
+func performHTTPInfoSearch(query string, qType int) (HTTPInfoResponse, error) {
+	// Initialize the database handler
+	db, err := cdb.NewHandler(config)
+	if err != nil {
+		return HTTPInfoResponse{}, err
+	}
+
+	// Connect to the database
+	err = db.Connect(config)
+	if err != nil {
+		cmn.DebugMsg(cmn.DbgLvlError, dbConnErrorLabel, err)
+		return HTTPInfoResponse{}, err
+	}
+	defer db.Close()
+
+	cmn.DebugMsg(cmn.DbgLvlDebug, SearchLabel, query)
+
+	// Parse the user input
+	var sqlQuery string
+	var sqlParams []interface{}
+	if qType == 1 {
+		// it's a GET request, so we need to interpret the q parameter
+		sqlQuery, sqlParams, err = parseHTTPInfoGetQuery(query)
+		if err != nil {
+			return HTTPInfoResponse{}, err
+		}
+	} else {
+		// It's a POST request, so we can use the standard JSON parsing
+		sqlQuery, sqlParams, err = parseHTTPInfoQuery(query)
+		if err != nil {
+			return HTTPInfoResponse{}, err
+		}
+	}
+	cmn.DebugMsg(cmn.DbgLvlDebug1, sqlQueryLabel, sqlQuery)
+	cmn.DebugMsg(cmn.DbgLvlDebug1, sqlQueryParamsLabel, sqlParams)
+
+	// Execute the query
+	rows, err := db.ExecuteQuery(sqlQuery, sqlParams...)
+	if err != nil {
+		return HTTPInfoResponse{}, err
+	}
+	defer rows.Close()
+
+	var results HTTPInfoResponse
+	for rows.Next() {
+		var row HTTPInfoRow
+		var detailsJSON []byte // Use a byte slice to hold the JSONB column data
+
+		// Adjust Scan to match the expected columns returned by your query
+		if err := rows.Scan(&row.CreatedAt, &row.LastUpdatedAt, &detailsJSON); err != nil {
+			return HTTPInfoResponse{}, err
+		}
+
+		// Assuming that detailsJSON contains an array of neti.NetInfo,
+		// you need to unmarshal the JSON into the NetInfo struct.
+		if err := json.Unmarshal(detailsJSON, &row.Details); err != nil {
+			return HTTPInfoResponse{}, err // Handle JSON unmarshal error
+		}
+
+		// Append the row to the results
+		results.Items = append(results.Items, row)
+	}
+
+	// Ensure to check rows.Err() after the loop to catch any error that occurred during iteration.
+	if err := rows.Err(); err != nil {
+		return HTTPInfoResponse{}, err
+	}
+
+	return results, nil
+}
+
+func parseHTTPInfoGetQuery(input string) (string, []interface{}, error) {
+	// Prepare the query body
+	queryBody := `
+	SELECT
+		hi.created_at,
+		hi.last_updated_at,
+		hi.details
+	FROM
+		HTTPInfo hi
+	JOIN
+        HTTPInfoIndex hii ON hi.httpinfo_id = hii.httpinfo_id
+	JOIN
+        SearchIndex si ON hii.index_id = si.index_id
+	LEFT JOIN
+		KeywordIndex ki ON si.index_id = ki.index_id
+	LEFT JOIN
+		Keywords k ON ki.keyword_id = k.keyword_id
+	WHERE
+	`
+	sqlQuery, sqlParams, err := parseAdvancedQuery(queryBody, input)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return sqlQuery, sqlParams, nil
+}
+
+func parseHTTPInfoQuery(input string) (string, []interface{}, error) {
+	var query string
+	var err error
+	var sqlParams []interface{}
+
+	// Unmarshal the JSON document
+	var req QueryRequest
+	err = json.Unmarshal([]byte(input), &req)
+	if err != nil {
+		return "", nil, err
+	}
+
+	// Extract the query from the request
+	if len(req.Title) > 0 {
+		query = req.Title
+	} else {
+		return "", nil, errors.New(noQueryProvided)
+	}
+
+	// Parse the user input
+	sqlQuery := `
+	SELECT
+		hi.created_at,
+		hi.last_updated_at,
+		hi.details
+	FROM
+		HTTPInfo hi
+	JOIN
+        HTTPInfoIndex hii ON hi.httpinfo_id = hii.httpinfo_id
+	JOIN
+        SearchIndex si ON hii.index_id = si.index_id
+	WHERE
+		si.page_url LIKE $1;
+	`
+	sqlParams = append(sqlParams, query)
+
+	return sqlQuery, sqlParams, nil
+}

services/api/types.go

@@ -135,3 +135,14 @@ type NetInfoRow struct {
 type NetInfoResponse struct {
 	Items []NetInfoRow `json:"items"`
 }
+
+type HTTPInfoRow struct {
+	CreatedAt     string       `json:"created_at"`
+	LastUpdatedAt string       `json:"last_updated_at"`
+	Details       neti.NetInfo `json:"details"`
+}
+
+// NetInfoResponse represents the structure of the network information response
+type HTTPInfoResponse struct {
+	Items []HTTPInfoRow `json:"items"`
+}