diff --git a/blob.go b/blob.go index 590df99..58e6bbf 100644 --- a/blob.go +++ b/blob.go @@ -1,13 +1,13 @@ -package cache +package blob import ( "bufio" "bytes" "context" wof_cache "github.com/whosonfirst/go-cache" + "github.com/whosonfirst/go-ioutil" "gocloud.dev/blob" "io" - "io/ioutil" "sync/atomic" ) @@ -57,7 +57,7 @@ func (c *BlobCache) Name() string { return "blob" } -func (c *BlobCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (c *BlobCache) Get(ctx context.Context, key string) (io.ReadSeekCloser, error) { fh, err := c.bucket.NewReader(ctx, key, nil) @@ -67,10 +67,10 @@ func (c *BlobCache) Get(ctx context.Context, key string) (io.ReadCloser, error) } atomic.AddInt64(&c.hits, 1) - return fh, nil + return ioutil.NewReadSeekCloser(fh) } -func (c *BlobCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.ReadCloser, error) { +func (c *BlobCache) Set(ctx context.Context, key string, fh io.ReadSeekCloser) (io.ReadSeekCloser, error) { var wr_opts *blob.WriterOptions @@ -79,7 +79,7 @@ func (c *BlobCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.R if v != nil { wr_opts = v.(*blob.WriterOptions) } - + bucket_wr, err := c.bucket.NewWriter(ctx, key, wr_opts) if err != nil { @@ -114,7 +114,7 @@ func (c *BlobCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.R atomic.AddInt64(&c.sets, 1) r.Reset(b.Bytes()) - return ioutil.NopCloser(r), nil + return ioutil.NewReadSeekCloser(r) } func (c *BlobCache) Unset(ctx context.Context, key string) error { diff --git a/blob_test.go b/blob_test.go new file mode 100644 index 0000000..2ca2de1 --- /dev/null +++ b/blob_test.go @@ -0,0 +1,28 @@ +package blob + +import ( + "context" + "github.com/whosonfirst/go-cache" + _ "gocloud.dev/blob/memblob" + "testing" +) + +func TestFSCache(t *testing.T) { + + ctx := context.Background() + + c, err := cache.NewCache(ctx, "mem://") + + if err != nil { + t.Fatalf("Failed to create mem:// cache, %v", err) + } + + opts := &testCacheOptions{} + + // This is defined in testing.go + err = testCache(ctx, c, opts) + + if err != nil { + t.Fatalf("Cache tests failed, %v", err) + } +} diff --git a/doc.go b/doc.go new file mode 100644 index 0000000..ce51d37 --- /dev/null +++ b/doc.go @@ -0,0 +1,2 @@ +// package blob implements the Who's On First `go-cache` interface using Google's GoCloud `Blob` objects. +package blob diff --git a/go.mod b/go.mod index 4feb264..42756f3 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/whosonfirst/go-cache-blob go 1.16 require ( - github.com/whosonfirst/go-cache v0.3.0 + github.com/whosonfirst/go-cache v0.5.0 + github.com/whosonfirst/go-ioutil v1.0.0 gocloud.dev v0.23.0 ) diff --git a/go.sum b/go.sum index 7123592..89fc133 100644 --- a/go.sum +++ b/go.sum @@ -308,6 +308,10 @@ github.com/whosonfirst/go-cache v0.1.0 h1:w6o4pWT/q2q/uclV72EU3D9CCKOgU+w+D+GKpa github.com/whosonfirst/go-cache v0.1.0/go.mod h1:M5Iche+26mVKFph49/pxS5W4ULZ9Dr01mebPqswG8QU= github.com/whosonfirst/go-cache v0.3.0 h1:QwboIxWu3Ab2gPikTMAJT2kukRr5c5yr/+yoMFRo8vQ= github.com/whosonfirst/go-cache v0.3.0/go.mod h1:uYn6IpxBxuSdULs15zkHHkOHTW8i4AS4A3jTv8wcbGg= +github.com/whosonfirst/go-cache v0.5.0 h1:s8o2FafYxI1TGJGqwi3G0XVD5yY1dvzwSe9dE5C+KfA= +github.com/whosonfirst/go-cache v0.5.0/go.mod h1:YawO5K9kQa3XnrtdfwZ6OS4gwuO87S06FqIPblMqZf4= +github.com/whosonfirst/go-ioutil v1.0.0 h1:sYpgJx7Wsp76e7PFGa8KKQBvWQW3+HMCWSJbKdD5m14= +github.com/whosonfirst/go-ioutil v1.0.0/go.mod h1:2dS1vWdAIkiHDvDF8fYyjv6k2NISmwaIjJJeEDBEdvg= github.com/whosonfirst/go-whosonfirst-cache v0.1.0 h1:ryWTsHj7gAEjwHC/WjsjROpFflsz3SCa7W9Qnk4EU7k= github.com/whosonfirst/go-whosonfirst-cache v0.1.0/go.mod h1:P5Tx34j2M50qX/kTfXY516apwGzJGkFSBYQI7TGArKw= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/testing.go b/testing.go new file mode 100644 index 0000000..2a1d584 --- /dev/null +++ b/testing.go @@ -0,0 +1,110 @@ +package blob + +// Eventually this will be deprecated and all of these methods will be publicly +// available in go-cache/testing.go + +import ( + "bytes" + "context" + "fmt" + "github.com/whosonfirst/go-cache" + "io" + _ "log" +) + +type testCacheOptions struct { + AllowCacheMiss bool +} + +func testCache(ctx context.Context, c cache.Cache, opts *testCacheOptions) error { + + k := "test" + v := "test" + + fh1, err := cache.ReadSeekCloserFromString(v) + + if err != nil { + return fmt.Errorf("Failed to create ReadSeekCloser, %v", err) + } + + fh2, err := c.Set(ctx, k, fh1) + + if err != nil { + return fmt.Errorf("Failed to set %s (%s), %v", k, v, err) + } + + equals, err := compareReadSeekClosers(fh1, fh2) + + if err != nil { + return fmt.Errorf("Failed to compare filehandles, %v", err) + } + + if !equals { + return fmt.Errorf("Filehandles 1 and 2 failed equality test") + } + + fh3, err := c.Get(ctx, k) + + if err != nil && !cache.IsCacheMiss(err) { + return fmt.Errorf("Failed to get cache value for '%s', %v", k, err) + } + + if err != nil { + + if !opts.AllowCacheMiss { + return fmt.Errorf("Unexpected cache miss for '%s'", k) + } + + } else { + + equals, err = compareReadSeekClosers(fh1, fh3) + + if err != nil { + return fmt.Errorf("Failed to compare filehandles 1 and 3, %v", err) + } + + if !equals { + return fmt.Errorf("Filehandles 1 and 3 failed equality test") + } + } + + return nil +} + +func compareReadSeekClosers(fh1 io.ReadSeekCloser, fh2 io.ReadSeekCloser) (bool, error) { + + fh1.Seek(0, 0) + fh2.Seek(0, 0) + + defer func() { + fh1.Seek(0, 0) + fh2.Seek(0, 0) + }() + + b1, err := io.ReadAll(fh1) + + if err != nil { + return false, fmt.Errorf("Failed to read fh1, %v", err) + } + + fh1.Seek(0, 0) + + b2, err := io.ReadAll(fh2) + + if err != nil { + return false, fmt.Errorf("Failed to read fh2, %v", err) + } + + fh2.Seek(0, 0) + + // log.Printf("1 '%s'\n", string(b1)) + // log.Printf("2 '%s'\n", string(b2)) + + equals := bytes.Compare(b1, b2) + + if equals != 0 { + return false, nil + } + + return true, nil +} diff --git a/vendor/github.com/whosonfirst/go-cache/README.md b/vendor/github.com/whosonfirst/go-cache/README.md index a7bc5e8..82b96df 100644 --- a/vendor/github.com/whosonfirst/go-cache/README.md +++ b/vendor/github.com/whosonfirst/go-cache/README.md @@ -1,6 +1,6 @@ # go-cache -There are many interfaces for caching things. This one is ours. It reads and writes `io.ReadCloser` instances. +There are many interfaces for caching things. This one is ours. It reads and writes `io.ReadSeekCloser` instances. _This package supersedes [go-whosonfirst-cache](https://github.com/whosonfirst/go-whosonfirst-cache) which will be retired soon._ @@ -29,9 +29,9 @@ func main() { log.Fatal(err) } - str, _ := cache.SetString(c, "some-key", "some-value") + str, _ := cache.SetString(ctx, c, "some-key", "some-value") - str2, _ := cache.GetString(c, "some-key") + str2, _ := cache.GetString(ctx, c, "some-key") log.Println(str2) } @@ -41,7 +41,7 @@ Two things to note: * The use of the `fs://` scheme rather than the more conventional `file://`. This is deliberate so as not to overlap with the [Go Cloud](https://gocloud.dev/howto/blob/) `Blob` package's file handler. -* The use of the `cache.GetString` and `cache.SetString` methods. The `cache.Cache` interface expects `io.ReadCloser` instances so these methods are shortcuts to hide the boilerplate code necessary to work with `io.ReadCloser` interfaces. +* The use of the `cache.GetString` and `cache.SetString` methods. The `cache.Cache` interface expects `io.ReadSeekCloser` instances so these methods are shortcuts to hide the boilerplate code necessary to work with `io.ReadSeekCloser` interfaces. There is also a handy `null://` cache which doesn't do anything at all (expect implement the `cache.Cache` interface). For example: @@ -58,8 +58,8 @@ There is also a handy `null://` cache which doesn't do anything at all (expect i type Cache interface { Name() string Close(context.Context) error - Get(context.Context, string) (io.ReadCloser, error) - Set(context.Context, string, io.ReadCloser) (io.ReadCloser, error) + Get(context.Context, string) (io.ReadSeekCloser, error) + Set(context.Context, string, io.ReadSeekCloser) (io.ReadSeekCloser, error) Unset(context.Context, string) error Hits() int64 Misses() int64 diff --git a/vendor/github.com/whosonfirst/go-cache/cache.go b/vendor/github.com/whosonfirst/go-cache/cache.go index b9c9097..52893b7 100644 --- a/vendor/github.com/whosonfirst/go-cache/cache.go +++ b/vendor/github.com/whosonfirst/go-cache/cache.go @@ -11,8 +11,8 @@ import ( type Cache interface { Close(context.Context) error Name() string - Get(context.Context, string) (io.ReadCloser, error) - Set(context.Context, string, io.ReadCloser) (io.ReadCloser, error) + Get(context.Context, string) (io.ReadSeekCloser, error) + Set(context.Context, string, io.ReadSeekCloser) (io.ReadSeekCloser, error) Unset(context.Context, string) error Hits() int64 Misses() int64 diff --git a/vendor/github.com/whosonfirst/go-cache/doc.go b/vendor/github.com/whosonfirst/go-cache/doc.go new file mode 100644 index 0000000..edcb1ce --- /dev/null +++ b/vendor/github.com/whosonfirst/go-cache/doc.go @@ -0,0 +1,3 @@ +// package cache provides a storage-independent interface, and common implementations, for caching things. +// There are many interfaces for caching things. This one is ours. It reads and writes `io.ReadSeekCloser` instances. +package cache diff --git a/vendor/github.com/whosonfirst/go-cache/fs.go b/vendor/github.com/whosonfirst/go-cache/fs.go index caec34c..38d411b 100644 --- a/vendor/github.com/whosonfirst/go-cache/fs.go +++ b/vendor/github.com/whosonfirst/go-cache/fs.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "errors" + "github.com/whosonfirst/go-ioutil" "io" _ "log" "net/url" @@ -81,7 +82,7 @@ func (c *FSCache) Name() string { return "fs" } -func (c *FSCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (c *FSCache) Get(ctx context.Context, key string) (io.ReadSeekCloser, error) { c.mu.RLock() defer c.mu.RUnlock() @@ -126,7 +127,7 @@ func (c *FSCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { return fh, nil } -func (c *FSCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.ReadCloser, error) { +func (c *FSCache) Set(ctx context.Context, key string, fh io.ReadSeekCloser) (io.ReadSeekCloser, error) { c.mu.Lock() defer c.mu.Unlock() @@ -173,7 +174,8 @@ func (c *FSCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.Rea return nil, err } - return NewReadCloser(b.Bytes()), nil + br := bytes.NewReader(b.Bytes()) + return ioutil.NewReadSeekCloser(br) } func (c *FSCache) Unset(ctx context.Context, key string) error { diff --git a/vendor/github.com/whosonfirst/go-cache/go.mod b/vendor/github.com/whosonfirst/go-cache/go.mod index 9476799..15fe5ab 100644 --- a/vendor/github.com/whosonfirst/go-cache/go.mod +++ b/vendor/github.com/whosonfirst/go-cache/go.mod @@ -5,4 +5,5 @@ go 1.16 require ( github.com/aaronland/go-roster v0.0.2 github.com/patrickmn/go-cache v2.1.0+incompatible + github.com/whosonfirst/go-ioutil v1.0.0 ) diff --git a/vendor/github.com/whosonfirst/go-cache/go.sum b/vendor/github.com/whosonfirst/go-cache/go.sum index 1cf3ac5..04e6ea8 100644 --- a/vendor/github.com/whosonfirst/go-cache/go.sum +++ b/vendor/github.com/whosonfirst/go-cache/go.sum @@ -5,5 +5,7 @@ github.com/aaronland/go-roster v0.0.2/go.mod h1:AcovpxlG1XxJxX2Fjqlm63fEIBhCjEIB github.com/patrickmn/go-cache v1.0.0 h1:3gD5McaYs9CxjyK5AXGcq8gdeCARtd/9gJDUvVeaZ0Y= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/whosonfirst/go-ioutil v1.0.0 h1:sYpgJx7Wsp76e7PFGa8KKQBvWQW3+HMCWSJbKdD5m14= +github.com/whosonfirst/go-ioutil v1.0.0/go.mod h1:2dS1vWdAIkiHDvDF8fYyjv6k2NISmwaIjJJeEDBEdvg= github.com/whosonfirst/go-whosonfirst-cache v0.1.0 h1:ryWTsHj7gAEjwHC/WjsjROpFflsz3SCa7W9Qnk4EU7k= github.com/whosonfirst/go-whosonfirst-cache v0.1.0/go.mod h1:P5Tx34j2M50qX/kTfXY516apwGzJGkFSBYQI7TGArKw= diff --git a/vendor/github.com/whosonfirst/go-cache/gocache.go b/vendor/github.com/whosonfirst/go-cache/gocache.go index 21b5b5a..faea49f 100644 --- a/vendor/github.com/whosonfirst/go-cache/gocache.go +++ b/vendor/github.com/whosonfirst/go-cache/gocache.go @@ -3,10 +3,11 @@ package cache // https://godoc.org/github.com/patrickmn/go-cache import ( + "bytes" "context" gocache "github.com/patrickmn/go-cache" + "github.com/whosonfirst/go-ioutil" "io" - "io/ioutil" "net/url" "strconv" "sync/atomic" @@ -104,7 +105,7 @@ func (c *GoCache) Name() string { return "gocache" } -func (c *GoCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (c *GoCache) Get(ctx context.Context, key string) (io.ReadSeekCloser, error) { // to do: timings that don't slow everything down the way // go-whosonfirst-timer does now (20170915/thisisaaronland) @@ -119,11 +120,12 @@ func (c *GoCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { atomic.AddInt64(&c.hits, 1) body := data.([]byte) + br := bytes.NewReader(body) - return NewReadCloser(body), nil + return ioutil.NewReadSeekCloser(br) } -func (c *GoCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.ReadCloser, error) { +func (c *GoCache) Set(ctx context.Context, key string, fh io.ReadSeekCloser) (io.ReadSeekCloser, error) { /* @@ -137,7 +139,7 @@ func (c *GoCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.Rea defer fh.Close() - body, err := ioutil.ReadAll(fh) + body, err := io.ReadAll(fh) if err != nil { return nil, err @@ -146,7 +148,8 @@ func (c *GoCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.Rea c.cache.Set(key, body, gocache.DefaultExpiration) atomic.AddInt64(&c.keys, 1) - return NewReadCloser(body), nil + fh.Seek(0, 0) + return fh, nil } func (c *GoCache) Unset(ctx context.Context, key string) error { diff --git a/vendor/github.com/whosonfirst/go-cache/multi.go b/vendor/github.com/whosonfirst/go-cache/multi.go index 4e36557..a72f282 100644 --- a/vendor/github.com/whosonfirst/go-cache/multi.go +++ b/vendor/github.com/whosonfirst/go-cache/multi.go @@ -1,10 +1,10 @@ package cache import ( - "bytes" "context" + "fmt" "io" - "io/ioutil" + "net/url" "sync/atomic" ) @@ -17,14 +17,41 @@ type MultiCache struct { caches []Cache } -/* func init() { ctx := context.Background() - RegisterCache(ctx, "null", NewMultiCache) + RegisterCache(ctx, "multi", NewMultiCache) } -*/ -func NewMultiCache(ctx context.Context, caches ...Cache) (Cache, error) { +func NewMultiCache(ctx context.Context, str_uri string) (Cache, error) { + + u, err := url.Parse(str_uri) + + if err != nil { + return nil, err + } + + q := u.Query() + + cache_uris := q["cache"] + + caches := make([]Cache, len(cache_uris)) + + for idx, c_uri := range cache_uris { + + c, err := NewCache(ctx, c_uri) + + if err != nil { + return nil, fmt.Errorf("Failed to create cache for '%s', %v", c_uri, err) + } + + caches[idx] = c + } + + return NewMultiCacheWithCaches(ctx, caches...) +} + +func NewMultiCacheWithCaches(ctx context.Context, caches ...Cache) (Cache, error) { + c := &MultiCache{ size: int64(0), hits: int64(0), @@ -32,6 +59,7 @@ func NewMultiCache(ctx context.Context, caches ...Cache) (Cache, error) { evictions: int64(0), caches: caches, } + return c, nil } @@ -53,7 +81,7 @@ func (mc *MultiCache) Name() string { return "multi" } -func (mc *MultiCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (mc *MultiCache) Get(ctx context.Context, key string) (io.ReadSeekCloser, error) { for _, c := range mc.caches { @@ -64,6 +92,19 @@ func (mc *MultiCache) Get(ctx context.Context, key string) (io.ReadCloser, error } atomic.AddInt64(&mc.hits, 1) + + for _, c := range mc.caches { + + // Only set caches that come *before* this cache + + if c.Name() == mc.Name() { + break + } + + c.Set(ctx, key, fh) + fh.Seek(0, 0) + } + return fh, nil } @@ -71,34 +112,21 @@ func (mc *MultiCache) Get(ctx context.Context, key string) (io.ReadCloser, error return nil, new(CacheMiss) } -func (mc *MultiCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.ReadCloser, error) { - - body, err := ioutil.ReadAll(fh) - - if err != nil { - return nil, err - } - - br := bytes.NewReader(body) +func (mc *MultiCache) Set(ctx context.Context, key string, fh io.ReadSeekCloser) (io.ReadSeekCloser, error) { for _, c := range mc.caches { - br.Seek(0, 0) - cl := ioutil.NopCloser(br) - - _, err := c.Set(ctx, key, cl) + _, err := c.Set(ctx, key, fh) if err != nil { return nil, err } + + fh.Seek(0, 0) } atomic.AddInt64(&mc.size, 1) - - br.Seek(0, 0) - cl := ioutil.NopCloser(br) - - return cl, nil + return fh, nil } func (mc *MultiCache) Unset(ctx context.Context, key string) error { diff --git a/vendor/github.com/whosonfirst/go-cache/null.go b/vendor/github.com/whosonfirst/go-cache/null.go index 5212760..9310655 100644 --- a/vendor/github.com/whosonfirst/go-cache/null.go +++ b/vendor/github.com/whosonfirst/go-cache/null.go @@ -31,12 +31,12 @@ func (c *NullCache) Name() string { return "null" } -func (c *NullCache) Get(ctx context.Context, key string) (io.ReadCloser, error) { +func (c *NullCache) Get(ctx context.Context, key string) (io.ReadSeekCloser, error) { atomic.AddInt64(&c.misses, 1) return nil, new(CacheMiss) } -func (c *NullCache) Set(ctx context.Context, key string, fh io.ReadCloser) (io.ReadCloser, error) { +func (c *NullCache) Set(ctx context.Context, key string, fh io.ReadSeekCloser) (io.ReadSeekCloser, error) { return fh, nil } diff --git a/vendor/github.com/whosonfirst/go-cache/string.go b/vendor/github.com/whosonfirst/go-cache/string.go new file mode 100644 index 0000000..7611d2f --- /dev/null +++ b/vendor/github.com/whosonfirst/go-cache/string.go @@ -0,0 +1,56 @@ +package cache + +import ( + "context" + "github.com/whosonfirst/go-ioutil" + "io" + "strings" +) + +func ReadSeekCloserFromString(v string) (io.ReadSeekCloser, error) { + sr := strings.NewReader(v) + return ioutil.NewReadSeekCloser(sr) +} + +func SetString(ctx context.Context, c Cache, k string, v string) (string, error) { + + fh, err := ReadSeekCloserFromString(v) + + if err != nil { + return "", err + } + + fh, err = c.Set(ctx, k, fh) + + if err != nil { + return "", err + } + + defer fh.Close() + + return toString(fh) +} + +func GetString(ctx context.Context, c Cache, k string) (string, error) { + + fh, err := c.Get(ctx, k) + + if err != nil { + return "", err + } + + defer fh.Close() + + return toString(fh) +} + +func toString(fh io.Reader) (string, error) { + + b, err := io.ReadAll(fh) + + if err != nil { + return "", err + } + + return string(b), nil +} diff --git a/vendor/github.com/whosonfirst/go-cache/testing.go b/vendor/github.com/whosonfirst/go-cache/testing.go new file mode 100644 index 0000000..06aa409 --- /dev/null +++ b/vendor/github.com/whosonfirst/go-cache/testing.go @@ -0,0 +1,106 @@ +package cache + +import ( + "bytes" + "context" + "fmt" + "io" + _ "log" +) + +type testCacheOptions struct { + AllowCacheMiss bool +} + +func testCache(ctx context.Context, c Cache, opts *testCacheOptions) error { + + k := "test" + v := "test" + + fh1, err := ReadSeekCloserFromString(v) + + if err != nil { + return fmt.Errorf("Failed to create ReadSeekCloser, %v", err) + } + + fh2, err := c.Set(ctx, k, fh1) + + if err != nil { + return fmt.Errorf("Failed to set %s (%s), %v", k, v, err) + } + + equals, err := compareReadSeekClosers(fh1, fh2) + + if err != nil { + return fmt.Errorf("Failed to compare filehandles, %v", err) + } + + if !equals { + return fmt.Errorf("Filehandles 1 and 2 failed equality test") + } + + fh3, err := c.Get(ctx, k) + + if err != nil && !IsCacheMiss(err) { + return fmt.Errorf("Failed to get cache value for '%s', %v", k, err) + } + + if err != nil { + + if !opts.AllowCacheMiss { + return fmt.Errorf("Unexpected cache miss for '%s'", k) + } + + } else { + + equals, err = compareReadSeekClosers(fh1, fh3) + + if err != nil { + return fmt.Errorf("Failed to compare filehandles 1 and 3, %v", err) + } + + if !equals { + return fmt.Errorf("Filehandles 1 and 3 failed equality test") + } + } + + return nil +} + +func compareReadSeekClosers(fh1 io.ReadSeekCloser, fh2 io.ReadSeekCloser) (bool, error) { + + fh1.Seek(0, 0) + fh2.Seek(0, 0) + + defer func() { + fh1.Seek(0, 0) + fh2.Seek(0, 0) + }() + + b1, err := io.ReadAll(fh1) + + if err != nil { + return false, fmt.Errorf("Failed to read fh1, %v", err) + } + + fh1.Seek(0, 0) + + b2, err := io.ReadAll(fh2) + + if err != nil { + return false, fmt.Errorf("Failed to read fh2, %v", err) + } + + fh2.Seek(0, 0) + + // log.Printf("1 '%s'\n", string(b1)) + // log.Printf("2 '%s'\n", string(b2)) + + equals := bytes.Compare(b1, b2) + + if equals != 0 { + return false, nil + } + + return true, nil +} diff --git a/vendor/github.com/whosonfirst/go-cache/utils.go b/vendor/github.com/whosonfirst/go-cache/utils.go deleted file mode 100644 index 532869d..0000000 --- a/vendor/github.com/whosonfirst/go-cache/utils.go +++ /dev/null @@ -1,59 +0,0 @@ -package cache - -import ( - "bytes" - "context" - "io" - "io/ioutil" -) - -func NewReadCloser(b []byte) io.ReadCloser { - r := bytes.NewReader(b) - return ioutil.NopCloser(r) -} - -func NewReadCloserFromString(s string) io.ReadCloser { - return NewReadCloser([]byte(s)) -} - -func SetString(c Cache, k string, v string) (string, error) { - - ctx := context.Background() - - r := NewReadCloserFromString(v) - fh, err := c.Set(ctx, k, r) - - if err != nil { - return "", err - } - - defer fh.Close() - - return toString(fh) -} - -func GetString(c Cache, k string) (string, error) { - - ctx := context.Background() - - fh, err := c.Get(ctx, k) - - if err != nil { - return "", err - } - - defer fh.Close() - - return toString(fh) -} - -func toString(fh io.Reader) (string, error) { - - b, err := ioutil.ReadAll(fh) - - if err != nil { - return "", err - } - - return string(b), nil -} diff --git a/vendor/github.com/whosonfirst/go-ioutil/.gitignore b/vendor/github.com/whosonfirst/go-ioutil/.gitignore new file mode 100644 index 0000000..afa44cd --- /dev/null +++ b/vendor/github.com/whosonfirst/go-ioutil/.gitignore @@ -0,0 +1,11 @@ +*~ +pkg +src +!vendor/src +bin +!bin/.gitignore +*.log +*.json +.travis.yml +*.db +testdata/*.txt \ No newline at end of file diff --git a/vendor/github.com/whosonfirst/go-ioutil/LICENSE b/vendor/github.com/whosonfirst/go-ioutil/LICENSE new file mode 100644 index 0000000..7832dde --- /dev/null +++ b/vendor/github.com/whosonfirst/go-ioutil/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2021, Aaron Straup Cope +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the {organization} nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/whosonfirst/go-ioutil/README.md b/vendor/github.com/whosonfirst/go-ioutil/README.md new file mode 100644 index 0000000..4fe61b3 --- /dev/null +++ b/vendor/github.com/whosonfirst/go-ioutil/README.md @@ -0,0 +1,42 @@ +# go-ioutil + +Go package for creating instances conforming to the Go 1.16 `io.ReadSeekCloser` interface from a variety of io.Read* instances that implement some but not all of the `io.Reader`, `io.Seeker` and `io.Closer` interfaces. + +## Documentation + +[![Go Reference](https://pkg.go.dev/badge/github.com/whosonfirst/go-ioutil.svg)](https://pkg.go.dev/github.com/whosonfirst/go-ioutil) + +## Example + +``` +import ( + "bytes" + "github.com/whosonfirst/go-ioutil" + "io" + "log" +) + +func main(){ + + fh, _ := os.Open("README.md") + + rsc, _ := NewReadSeekCloser(fh) + + body, _ := io.ReadAll(rsc) + + rsc.Seek(0, 0) + + body2, _ := io.ReadAll(rsc) + + same := bytes.Equal(body, body2) + log.Printf("Same %t\n", same) + + rsc.Close() +} +``` + +_Error handling removed for brevity._ + +## See also + +* https://golang.org/pkg/io/#ReadSeekCloser \ No newline at end of file diff --git a/vendor/github.com/whosonfirst/go-ioutil/doc.go b/vendor/github.com/whosonfirst/go-ioutil/doc.go new file mode 100644 index 0000000..954c86d --- /dev/null +++ b/vendor/github.com/whosonfirst/go-ioutil/doc.go @@ -0,0 +1,29 @@ +// package ioutil provides methods for creating a new instance conforming to the Go 1.16 io.ReadSeekCloser interface from a variety of io.Read* instances that implement some but not all of the io.Reader, io.Seeker and io.Closer interfaces. +// +// Example +// +// import ( +// "bytes" +// "github.com/whosonfirst/go-ioutil" +// "io" +// "log" +// ) +// +// func main(){ +// +// fh, _ := os.Open("README.md") +// +// rsc, _ := NewReadSeekCloser(fh) +// +// body, _ := io.ReadAll(rsc) +// +// rsc.Seek(0, 0) +// +// body2, _ := io.ReadAll(rsc) +// +// same := bytes.Equal(body, body2) +// log.Printf("Same %t\n", same) +// +// rsc.Close() +// } +package ioutil diff --git a/vendor/github.com/whosonfirst/go-ioutil/go.mod b/vendor/github.com/whosonfirst/go-ioutil/go.mod new file mode 100644 index 0000000..7e37699 --- /dev/null +++ b/vendor/github.com/whosonfirst/go-ioutil/go.mod @@ -0,0 +1,3 @@ +module github.com/whosonfirst/go-ioutil + +go 1.16 diff --git a/vendor/github.com/whosonfirst/go-ioutil/readseekcloser.go b/vendor/github.com/whosonfirst/go-ioutil/readseekcloser.go new file mode 100644 index 0000000..18bb37e --- /dev/null +++ b/vendor/github.com/whosonfirst/go-ioutil/readseekcloser.go @@ -0,0 +1,120 @@ +package ioutil + +// This is only here until there is an equivalent package/construct in the core Go language +// (20210217/thisisaaronland) + +import ( + "bytes" + "fmt" + "io" + "sync" +) + +// Type ReadSeekCloser implements the io.Reader, io.Seeker and io.Closer interfaces. +type ReadSeekCloser struct { + io.Reader + io.Seeker + io.Closer + reader bool + closer bool + seeker bool + fh interface{} + br *bytes.Reader + mu *sync.RWMutex +} + +// Create a new NewReadSeekCloser instance conforming to the Go 1.16 `io.ReadSeekCloser` interface. This method accepts the following types: io.ReadSeekCloser, io.Reader, io.ReadCloser and io.ReadSeeker. +func NewReadSeekCloser(fh interface{}) (io.ReadSeekCloser, error) { + + reader := true + seeker := false + closer := false + + switch fh.(type) { + case io.ReadSeekCloser: + return fh.(io.ReadSeekCloser), nil + case io.Reader: + // pass + case io.ReadCloser: + closer = true + case io.ReadSeeker: + seeker = true + default: + return nil, fmt.Errorf("Invalid or unsupported type") + } + + mu := new(sync.RWMutex) + + rsc := &ReadSeekCloser{ + reader: reader, + seeker: seeker, + closer: closer, + fh: fh, + mu: mu, + } + + return rsc, nil +} + +// Read implements the standard Read interface: it reads data from the pipe, blocking until a writer arrives or the write end is closed. If the write end is closed with an error, that error is returned as err; otherwise err is `io.EOF`. +func (rsc *ReadSeekCloser) Read(p []byte) (n int, err error) { + + if rsc.seeker { + return rsc.fh.(io.Reader).Read(p) + } + + br, err := rsc.bytesReader() + + if err != nil { + return 0, err + } + + return br.Read(p) +} + +// Close closes the reader; subsequent writes to the write half of the pipe will return the error `io.ErrClosedPipe`. +func (rsc *ReadSeekCloser) Close() error { + + if rsc.closer { + return rsc.fh.(io.ReadCloser).Close() + } + + return nil +} + +// Seek implements the `io.Seeker` interface. +func (rsc *ReadSeekCloser) Seek(offset int64, whence int) (int64, error) { + + if rsc.seeker { + return rsc.fh.(io.Seeker).Seek(offset, whence) + } + + br, err := rsc.bytesReader() + + if err != nil { + return 0, err + } + + return br.Seek(offset, whence) +} + +func (rsc *ReadSeekCloser) bytesReader() (*bytes.Reader, error) { + + rsc.mu.Lock() + defer rsc.mu.Unlock() + + if rsc.br != nil { + return rsc.br, nil + } + + body, err := io.ReadAll(rsc.fh.(io.Reader)) + + if err != nil { + return nil, err + } + + br := bytes.NewReader(body) + rsc.br = br + + return br, nil +} diff --git a/vendor/gocloud.dev/blob/memblob/memblob.go b/vendor/gocloud.dev/blob/memblob/memblob.go new file mode 100644 index 0000000..59a9c45 --- /dev/null +++ b/vendor/gocloud.dev/blob/memblob/memblob.go @@ -0,0 +1,381 @@ +// Copyright 2018 The Go Cloud Development Kit Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package memblob provides an in-memory blob implementation. +// Use OpenBucket to construct a *blob.Bucket. +// +// URLs +// +// For blob.OpenBucket memblob registers for the scheme "mem". +// To customize the URL opener, or for more details on the URL format, +// see URLOpener. +// See https://gocloud.dev/concepts/urls/ for background information. +// +// As +// +// memblob does not support any types for As. +package memblob // import "gocloud.dev/blob/memblob" + +import ( + "bytes" + "context" + "crypto/md5" + "errors" + "fmt" + "hash" + "io" + "net/url" + "sort" + "strings" + "sync" + "time" + + "gocloud.dev/blob" + "gocloud.dev/blob/driver" + "gocloud.dev/gcerrors" +) + +const defaultPageSize = 1000 + +var ( + errNotFound = errors.New("blob not found") + errNotImplemented = errors.New("not implemented") +) + +func init() { + blob.DefaultURLMux().RegisterBucket(Scheme, &URLOpener{}) +} + +// Scheme is the URL scheme memblob registers its URLOpener under on +// blob.DefaultMux. +const Scheme = "mem" + +// URLOpener opens URLs like "mem://". +// +// No query parameters are supported. +type URLOpener struct{} + +// OpenBucketURL opens a blob.Bucket based on u. +func (*URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket, error) { + for param := range u.Query() { + return nil, fmt.Errorf("open bucket %v: invalid query parameter %q", u, param) + } + return OpenBucket(nil), nil +} + +// Options sets options for constructing a *blob.Bucket backed by memory. +type Options struct{} + +type blobEntry struct { + Content []byte + Attributes *driver.Attributes +} + +type bucket struct { + mu sync.Mutex + blobs map[string]*blobEntry +} + +// openBucket creates a driver.Bucket backed by memory. +func openBucket(_ *Options) driver.Bucket { + return &bucket{ + blobs: map[string]*blobEntry{}, + } +} + +// OpenBucket creates a *blob.Bucket backed by memory. +func OpenBucket(opts *Options) *blob.Bucket { + return blob.NewBucket(openBucket(opts)) +} + +func (b *bucket) Close() error { + return nil +} + +func (b *bucket) ErrorCode(err error) gcerrors.ErrorCode { + switch err { + case errNotFound: + return gcerrors.NotFound + case errNotImplemented: + return gcerrors.Unimplemented + default: + return gcerrors.Unknown + } +} + +// ListPaged implements driver.ListPaged. +// The implementation largely mirrors the one in fileblob. +func (b *bucket) ListPaged(ctx context.Context, opts *driver.ListOptions) (*driver.ListPage, error) { + b.mu.Lock() + defer b.mu.Unlock() + + // pageToken is a returned NextPageToken, set below; it's the last key of the + // previous page. + var pageToken string + if len(opts.PageToken) > 0 { + pageToken = string(opts.PageToken) + } + pageSize := opts.PageSize + if pageSize == 0 { + pageSize = defaultPageSize + } + + var keys []string + for key := range b.blobs { + keys = append(keys, key) + } + sort.Strings(keys) + + // If opts.Delimiter != "", lastPrefix contains the last "directory" key we + // added. It is used to avoid adding it again; all files in this "directory" + // are collapsed to the single directory entry. + var lastPrefix string + var result driver.ListPage + for _, key := range keys { + // Skip keys that don't match the Prefix. + if !strings.HasPrefix(key, opts.Prefix) { + continue + } + + entry := b.blobs[key] + obj := &driver.ListObject{ + Key: key, + ModTime: entry.Attributes.ModTime, + Size: entry.Attributes.Size, + MD5: entry.Attributes.MD5, + } + + // If using Delimiter, collapse "directories". + if opts.Delimiter != "" { + // Strip the prefix, which may contain Delimiter. + keyWithoutPrefix := key[len(opts.Prefix):] + // See if the key still contains Delimiter. + // If no, it's a file and we just include it. + // If yes, it's a file in a "sub-directory" and we want to collapse + // all files in that "sub-directory" into a single "directory" result. + if idx := strings.Index(keyWithoutPrefix, opts.Delimiter); idx != -1 { + prefix := opts.Prefix + keyWithoutPrefix[0:idx+len(opts.Delimiter)] + // We've already included this "directory"; don't add it. + if prefix == lastPrefix { + continue + } + // Update the object to be a "directory". + obj = &driver.ListObject{ + Key: prefix, + IsDir: true, + } + lastPrefix = prefix + } + } + + // If there's a pageToken, skip anything before it. + if pageToken != "" && obj.Key <= pageToken { + continue + } + + // If we've already got a full page of results, set NextPageToken and return. + if len(result.Objects) == pageSize { + result.NextPageToken = []byte(result.Objects[pageSize-1].Key) + return &result, nil + } + result.Objects = append(result.Objects, obj) + } + return &result, nil +} + +// As implements driver.As. +func (b *bucket) As(i interface{}) bool { return false } + +// As implements driver.ErrorAs. +func (b *bucket) ErrorAs(err error, i interface{}) bool { return false } + +// Attributes implements driver.Attributes. +func (b *bucket) Attributes(ctx context.Context, key string) (*driver.Attributes, error) { + b.mu.Lock() + defer b.mu.Unlock() + + entry, found := b.blobs[key] + if !found { + return nil, errNotFound + } + return entry.Attributes, nil +} + +// NewRangeReader implements driver.NewRangeReader. +func (b *bucket) NewRangeReader(ctx context.Context, key string, offset, length int64, opts *driver.ReaderOptions) (driver.Reader, error) { + b.mu.Lock() + defer b.mu.Unlock() + + entry, found := b.blobs[key] + if !found { + return nil, errNotFound + } + + if opts.BeforeRead != nil { + if err := opts.BeforeRead(func(interface{}) bool { return false }); err != nil { + return nil, err + } + } + r := bytes.NewReader(entry.Content) + if offset > 0 { + if _, err := r.Seek(offset, io.SeekStart); err != nil { + return nil, err + } + } + var ior io.Reader = r + if length >= 0 { + ior = io.LimitReader(r, length) + } + return &reader{ + r: ior, + attrs: driver.ReaderAttributes{ + ContentType: entry.Attributes.ContentType, + ModTime: entry.Attributes.ModTime, + Size: entry.Attributes.Size, + }, + }, nil +} + +type reader struct { + r io.Reader + attrs driver.ReaderAttributes +} + +func (r *reader) Read(p []byte) (int, error) { + return r.r.Read(p) +} + +func (r *reader) Close() error { + return nil +} + +func (r *reader) Attributes() *driver.ReaderAttributes { + return &r.attrs +} + +func (r *reader) As(i interface{}) bool { return false } + +// NewTypedWriter implements driver.NewTypedWriter. +func (b *bucket) NewTypedWriter(ctx context.Context, key string, contentType string, opts *driver.WriterOptions) (driver.Writer, error) { + if key == "" { + return nil, errors.New("invalid key (empty string)") + } + b.mu.Lock() + defer b.mu.Unlock() + + if opts.BeforeWrite != nil { + if err := opts.BeforeWrite(func(interface{}) bool { return false }); err != nil { + return nil, err + } + } + md := map[string]string{} + for k, v := range opts.Metadata { + md[k] = v + } + return &writer{ + ctx: ctx, + b: b, + key: key, + contentType: contentType, + metadata: md, + opts: opts, + md5hash: md5.New(), + }, nil +} + +type writer struct { + ctx context.Context + b *bucket + key string + contentType string + metadata map[string]string + opts *driver.WriterOptions + buf bytes.Buffer + // We compute the MD5 hash so that we can store it with the file attributes, + // not for verification. + md5hash hash.Hash +} + +func (w *writer) Write(p []byte) (n int, err error) { + if _, err := w.md5hash.Write(p); err != nil { + return 0, err + } + return w.buf.Write(p) +} + +func (w *writer) Close() error { + // Check if the write was cancelled. + if err := w.ctx.Err(); err != nil { + return err + } + + md5sum := w.md5hash.Sum(nil) + content := w.buf.Bytes() + now := time.Now() + entry := &blobEntry{ + Content: content, + Attributes: &driver.Attributes{ + CacheControl: w.opts.CacheControl, + ContentDisposition: w.opts.ContentDisposition, + ContentEncoding: w.opts.ContentEncoding, + ContentLanguage: w.opts.ContentLanguage, + ContentType: w.contentType, + Metadata: w.metadata, + Size: int64(len(content)), + CreateTime: now, + ModTime: now, + MD5: md5sum, + ETag: fmt.Sprintf("\"%x-%x\"", now.UnixNano(), len(content)), + }, + } + w.b.mu.Lock() + defer w.b.mu.Unlock() + if prev := w.b.blobs[w.key]; prev != nil { + entry.Attributes.CreateTime = prev.Attributes.CreateTime + } + w.b.blobs[w.key] = entry + return nil +} + +// Copy implements driver.Copy. +func (b *bucket) Copy(ctx context.Context, dstKey, srcKey string, opts *driver.CopyOptions) error { + b.mu.Lock() + defer b.mu.Unlock() + + if opts.BeforeCopy != nil { + return opts.BeforeCopy(func(interface{}) bool { return false }) + } + v := b.blobs[srcKey] + if v == nil { + return errNotFound + } + b.blobs[dstKey] = v + return nil +} + +// Delete implements driver.Delete. +func (b *bucket) Delete(ctx context.Context, key string) error { + b.mu.Lock() + defer b.mu.Unlock() + + if b.blobs[key] == nil { + return errNotFound + } + delete(b.blobs, key) + return nil +} + +func (b *bucket) SignedURL(ctx context.Context, key string, opts *driver.SignedURLOptions) (string, error) { + return "", errNotImplemented +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 09c7adb..519814c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -12,9 +12,12 @@ github.com/golang/protobuf/ptypes/timestamp github.com/googleapis/gax-go/v2 # github.com/patrickmn/go-cache v2.1.0+incompatible github.com/patrickmn/go-cache -# github.com/whosonfirst/go-cache v0.3.0 +# github.com/whosonfirst/go-cache v0.5.0 ## explicit github.com/whosonfirst/go-cache +# github.com/whosonfirst/go-ioutil v1.0.0 +## explicit +github.com/whosonfirst/go-ioutil # go.opencensus.io v0.23.0 go.opencensus.io go.opencensus.io/internal @@ -35,6 +38,7 @@ go.opencensus.io/trace/tracestate ## explicit gocloud.dev/blob gocloud.dev/blob/driver +gocloud.dev/blob/memblob gocloud.dev/gcerrors gocloud.dev/internal/gcerr gocloud.dev/internal/oc