Skip to content

Commit 8ac9e3d

Browse files
demophoondemophoon
committed
Add tracker removal support
This commit allows users to specify a list of hosts to perform tracker removal from the shortened urls to preserve privacy of anyone using the link.
1 parent 38d74c5 commit 8ac9e3d

File tree

4 files changed

+161
-33
lines changed

4 files changed

+161
-33
lines changed

‎pkg/config/config.go

+4
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,10 @@ type Config struct {
5959

6060
// Auth
6161
AuthBackend *AuthBackend `mapstructure:"auth" yaml:"auth,omitempty"`
62+
63+
// Tracker Removal
64+
RemoveQueryParametersMatchingHosts *[]string `mapstructure:"remove_query_parameters_matching_hosts" yaml:"remove_query_parameters_matching_hosts,omitempty"`
65+
ResolveURLMatchingHosts *[]string `mapstructure:"resolve_urls_matching_hosts" yaml:"resolve_urls_matching_hosts,omitempty"`
6266
}
6367

6468
func New() *Config {

‎service/main.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,13 @@ import (
88
type Server struct {
99
state server.ServerState
1010
storage server.ServerStorage
11+
config *config.Config
1112
}
1213

1314
func New(c *config.Config) *Server {
14-
server := &Server{}
15+
server := &Server{
16+
config: c,
17+
}
1518
return server
1619
}
1720

‎service/shrl.go

+2-32
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@ import (
44
"context"
55

66
pb "github.com/demophoon/shrls/server/gen"
7-
"github.com/mat/besticon/v3/besticon"
8-
log "github.com/sirupsen/logrus"
97
)
108

119
func (s Server) GetShrl(ctx context.Context, req *pb.GetShrlRequest) (*pb.GetShrlResponse, error) {
@@ -43,6 +41,7 @@ func (s Server) ListShrls(ctx context.Context, req *pb.ListShrlsRequest) (*pb.Li
4341
}
4442

4543
func (s Server) PutShrl(ctx context.Context, req *pb.PutShrlRequest) (*pb.PutShrlResponse, error) {
44+
s.enhanceShortUrl(req.Shrl)
4645
shrl, err := s.state.UpdateShrl(ctx, req.Shrl)
4746
if err != nil {
4847
return nil, err
@@ -53,38 +52,9 @@ func (s Server) PutShrl(ctx context.Context, req *pb.PutShrlRequest) (*pb.PutShr
5352
}, nil
5453
}
5554

56-
func (s Server) resolveFavicon(u *pb.ShortURL) {
57-
switch u.Content.Content.(type) {
58-
case *pb.ExpandedURL_Url:
59-
loc := u.Content.GetUrl().Url
60-
61-
bi := besticon.New()
62-
icf := bi.NewIconFinder()
63-
64-
icons, err := icf.FetchIcons(loc)
65-
66-
if err == nil {
67-
for _, icon := range icons {
68-
if icon.Width >= 16 && icon.Height >= 16 {
69-
u.Content.Content = &pb.ExpandedURL_Url{
70-
Url: &pb.Redirect{
71-
Url: loc,
72-
Favicon: icon.ImageData,
73-
},
74-
}
75-
log.Tracef("favicon found for %s at %s", loc, icon.URL)
76-
break
77-
}
78-
}
79-
} else {
80-
log.Warnf("Unable to fetch favicon from %s: %s", loc, err)
81-
}
82-
}
83-
}
84-
8555
func (s Server) PostShrl(ctx context.Context, req *pb.PostShrlRequest) (*pb.PostShrlResponse, error) {
8656
u := req.Shrl
87-
s.resolveFavicon(u)
57+
s.enhanceShortUrl(u)
8858

8959
shrl, err := s.state.CreateShrl(ctx, u)
9060
if err != nil {

‎service/shrl_enhancers.go

+151
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
package service
2+
3+
import (
4+
"net/http"
5+
"net/url"
6+
"regexp"
7+
"strings"
8+
9+
"github.com/mat/besticon/v3/besticon"
10+
log "github.com/sirupsen/logrus"
11+
12+
pb "github.com/demophoon/shrls/server/gen"
13+
)
14+
15+
func (s Server) enhanceShortUrl(u *pb.ShortURL) {
16+
// Fetch Favicon
17+
s.resolveFavicon(u)
18+
19+
// Resolve Redirects
20+
s.resolveRedirects(u)
21+
22+
// Strip Url Parameters
23+
s.removeQueryParameters(u)
24+
}
25+
26+
func (s Server) resolveRedirects(u *pb.ShortURL) {
27+
switch u.Content.Content.(type) {
28+
case *pb.ExpandedURL_Url:
29+
if s.config.ResolveURLMatchingHosts == nil {
30+
return
31+
}
32+
33+
loc := u.Content.GetUrl().Url
34+
parsed, err := url.Parse(loc)
35+
if err != nil {
36+
log.Warnf("Couldn't parse url when removing query parameters from %s: %s", loc, err)
37+
}
38+
39+
s := *s.config.ResolveURLMatchingHosts
40+
regex_str := strings.Join(s, "|")
41+
log.Debugf("Regexp string: %s", regex_str)
42+
matcher, err := regexp.Compile(regex_str)
43+
if err != nil {
44+
log.Errorf("Unable to determine resolve_urls_matching_hosts option: %s", err)
45+
return
46+
}
47+
if !matcher.Match([]byte(parsed.Host)) {
48+
return
49+
}
50+
51+
nextLocation := parsed.String()
52+
i := 0
53+
for i < 25 {
54+
client := &http.Client{
55+
CheckRedirect: func(req *http.Request, via []*http.Request) error {
56+
return http.ErrUseLastResponse
57+
},
58+
}
59+
60+
resp, err := client.Head(nextLocation)
61+
if err != nil {
62+
log.Errorf("Unable to resolve url: %s", err)
63+
return
64+
}
65+
66+
if resp.StatusCode >= 300 && resp.StatusCode < 400 {
67+
nextLocation = resp.Header.Get("Location")
68+
i += 1
69+
} else {
70+
break
71+
}
72+
}
73+
74+
u.Content.Content = &pb.ExpandedURL_Url{
75+
Url: &pb.Redirect{
76+
Url: nextLocation,
77+
Favicon: u.Content.GetUrl().Favicon,
78+
},
79+
}
80+
81+
}
82+
}
83+
84+
func (s Server) removeQueryParameters(u *pb.ShortURL) {
85+
switch u.Content.Content.(type) {
86+
case *pb.ExpandedURL_Url:
87+
if s.config.RemoveQueryParametersMatchingHosts == nil {
88+
return
89+
}
90+
91+
loc := u.Content.GetUrl().Url
92+
parsed, err := url.Parse(loc)
93+
if err != nil {
94+
log.Warnf("Couldn't parse url when removing query parameters from %s: %s", loc, err)
95+
}
96+
97+
s := *s.config.RemoveQueryParametersMatchingHosts
98+
regex_str := strings.Join(s, "|")
99+
log.Debugf("Regexp string: %s", regex_str)
100+
matcher, err := regexp.Compile(regex_str)
101+
if err != nil {
102+
log.Errorf("Unable to determine remove_query_parameters_matching_hosts option: %s", err)
103+
return
104+
}
105+
if !matcher.Match([]byte(parsed.Host)) {
106+
return
107+
}
108+
109+
stripped := url.URL{
110+
Scheme: parsed.Scheme,
111+
User: parsed.User,
112+
Host: parsed.Host,
113+
Path: parsed.Path,
114+
}
115+
u.Content.Content = &pb.ExpandedURL_Url{
116+
Url: &pb.Redirect{
117+
Url: stripped.String(),
118+
Favicon: u.Content.GetUrl().Favicon,
119+
},
120+
}
121+
}
122+
}
123+
124+
func (s Server) resolveFavicon(u *pb.ShortURL) {
125+
switch u.Content.Content.(type) {
126+
case *pb.ExpandedURL_Url:
127+
loc := u.Content.GetUrl().Url
128+
129+
bi := besticon.New()
130+
icf := bi.NewIconFinder()
131+
132+
icons, err := icf.FetchIcons(loc)
133+
134+
if err == nil {
135+
for _, icon := range icons {
136+
if icon.Width >= 16 && icon.Height >= 16 {
137+
u.Content.Content = &pb.ExpandedURL_Url{
138+
Url: &pb.Redirect{
139+
Url: loc,
140+
Favicon: icon.ImageData,
141+
},
142+
}
143+
log.Tracef("favicon found for %s at %s", loc, icon.URL)
144+
break
145+
}
146+
}
147+
} else {
148+
log.Warnf("Unable to fetch favicon from %s: %s", loc, err)
149+
}
150+
}
151+
}

0 commit comments

Comments
 (0)