tests: tweak timings so we cancel during HTTP processing

There are 3 possibilities:
* We cancel before the HTTP request (possibly even before the dial has completed)
* We cancel during the delay in our slow server
* We cancel after request processing

Because our intervals were [0, 1s] and [0.5s, 2s] I think we
previously could have hit all three cases.

Likely there is a bug lurking in one of these three cases,
but let's start by making the delays such that we are
always in the second situation (assuming dialing localhost
takes < 1s)

We now delay the HTTP response by 3 seconds, and cancel
after a random value in the interval [1s, 2s)

tests/proxy_test.go

@@ -105,17 +105,26 @@ type delayedServer struct {
 	maxWait time.Duration
 }
 
-func newDelayedServer() *delayedServer {
+// randomDuration returns a random duration in the [min, max) interval
+func randomDuration(min, max time.Duration) time.Duration {
+	d := min
+	if max != min {
+		d += time.Millisecond * time.Duration(rand.Int63n(int64(max-min)))
+	}
+	return d
+}
+
+func newDelayedServer(minWait time.Duration, maxWait time.Duration) *delayedServer {
 	return &delayedServer{
-		minWait: 500 * time.Millisecond,
-		maxWait: 2 * time.Second,
+		minWait: minWait,
+		maxWait: maxWait,
 	}
 }
 
-var _ = newDelayedServer() // Suppress unused lint error.
+var _ = newDelayedServer(time.Second, time.Second) // Suppress unused lint error.
 
 func (s *delayedServer) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
-	delay := time.Duration(rand.Int63n(int64(s.maxWait-s.minWait))) + s.minWait /* #nosec G404 */
+	delay := randomDuration(s.minWait, s.maxWait)
 	time.Sleep(delay)
 	w.Write([]byte("hello"))
 }
@@ -360,7 +369,8 @@ func TestProxyDial_RequestCancelled_GRPC(t *testing.T) {
 func TestProxyDial_RequestCancelled_Concurrent_GRPC(t *testing.T) {
 	expectCleanShutdown(t)
 
-	slowServer := newDelayedServer()
+	// An HTTP server that always waits 3 seconds before replying
+	slowServer := newDelayedServer(3*time.Second, 3*time.Second)
 	server := httptest.NewServer(slowServer)
 	defer server.Close()
 
@@ -420,8 +430,12 @@ func TestProxyDial_RequestCancelled_Concurrent_GRPC(t *testing.T) {
 	const concurrentConns = 50
 	wg.Add(concurrentConns)
 	for i := 0; i < concurrentConns; i++ {
-		cancelDelayMs := rand.Int63n(1000) + 5 /* #nosec G404 */
-		go dialFn(i, time.Duration(cancelDelayMs)*time.Millisecond)
+		// The delay before we cancel the HTTP request:
+		// * at least 1 second for the HTTP connection to establish
+		// * less than 2 seconds so that we cancel during the 3 second delay in our slowServer
+		// TODO: Can we try to cancel during the dial?  Or after the HTTP request has returned?
+		cancelDelay := randomDuration(time.Second, 2*time.Second)
+		go dialFn(i, cancelDelay)
 	}
 	wg.Wait()