wl: add results filtering

.gitignore

@@ -8,6 +8,7 @@ test.csv
 test.json
 foo/
 .DS_Store
+filter.txt
 # Binaries for programs and plugins
 *.exe
 *.exe~

README.md

@@ -12,6 +12,9 @@ two different ways to make a port scanner
 ## tas
 throw against site (pull down archived links from Wayback Machine, run against the site, see what status codes currently are)
 
+## wl
+supply a url, get a wordlist (for that page only). adjust your results by filtering out terms, adding a minimum length, or requiring a certain instance count.
+
 # python
 ## tas
 throw against site (pull down archived links from Wayback Machine, run against the site, see what status codes currently are) (not concurrent...yet)

wl/data.go

@@ -25,6 +25,12 @@ func (wm *WordMap) add(w string) {
 	wm.words[w]++
 }
 
+func (wm *WordMap) delete(key string) {
+	wm.mu.Lock()
+	defer wm.mu.Unlock()
+	delete(wm.words, key)
+}
+
 func (wm *WordMap) sort() []string {
 	keys := make([]string, 0, len(wm.words))
 	for key := range wm.words {

wl/main.go

@@ -8,6 +8,7 @@ import (
 )
 
 type config struct {
+	filter string
 	minCount  int
 	minLength int
 	timeout   int
@@ -22,6 +23,7 @@ type wl struct {
 
 func main() {
 	var config config
+	flag.StringVar(&config.filter, "f", "", "file name containing words to filter out of results")
 	flag.IntVar(&config.minCount, "c", 0, "minimum count to include word in results")
 	flag.IntVar(&config.minLength, "len", 0, "minimum word length to consider")
 	flag.IntVar(&config.timeout, "t", 5000, "request timeout (in ms)")
@@ -55,7 +57,11 @@ func main() {
 		}(word)
 	}
 	wg.Wait()
-
+
+	if config.filter != "" {
+		w.filterTerms()
+	}
+
 	keys := w.wordMap.sort()
 	keysCount := w.dropLowCount(keys)
 

wl/process.go

@@ -1,6 +1,9 @@
 package main
 
 import (
+	"bufio"
+	"log"
+	"os"
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
@@ -48,3 +51,29 @@ func (w *wl) dropLowCount(keys []string) []string {
 	}
 	return keys
 }
+
+func (w *wl) filterTerms() {
+	terms, err := w.readInput(w.config.filter)
+	if err != nil {
+		log.Println(err)
+	}
+	for _, term := range terms {
+		if _, ok := w.wordMap.words[term]; ok {
+			w.wordMap.delete(term)
+		}
+	}
+}
+
+func (w *wl) readInput(name string) ([]string, error) {
+	f, err := os.Open(name)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	var terms []string
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		terms = append(terms, scanner.Text()) 
+	}
+	return terms, scanner.Err()
+}