Fix default User-Agent when using custom headers

When using non-nil hdr parameter with c.Request method, and not setting
User-Agent header explicitly, it would be left unset (when it should've
been set to c.UserAgent). This means that Go HTTP client would use the default
"Go-http-client/2.0" string.

Suppressing the header altogether is still possible by setting it to
empty string.

Author: WGH-

colly.go

@@ -558,7 +558,10 @@ func (c *Collector) scrape(u, method string, depth int, requestData io.Reader, c
 	}
 
 	if hdr == nil {
-		hdr = http.Header{"User-Agent": []string{c.UserAgent}}
+		hdr = http.Header{}
+	}
+	if _, ok := hdr["User-Agent"]; !ok {
+		hdr.Set("User-Agent", c.UserAgent)
 	}
 	rc, ok := requestData.(io.ReadCloser)
 	if !ok && requestData != nil {

colly_test.go

@@ -1009,6 +1009,17 @@ func TestUserAgent(t *testing.T) {
 			t.Errorf("mismatched User-Agent (nil hdr): got=%q want=%q", got, want)
 		}
 	}()
+	func() {
+		c := NewCollector(UserAgent(exampleUserAgent1))
+		c.OnResponse(func(resp *Response) {
+			receivedUserAgent = string(resp.Body)
+		})
+
+		c.Request("GET", ts.URL+"/user_agent", nil, nil, http.Header{})
+		if got, want := receivedUserAgent, exampleUserAgent1; got != want {
+			t.Errorf("mismatched User-Agent (non-nil hdr): got=%q want=%q", got, want)
+		}
+	}()
 	func() {
 		c := NewCollector(UserAgent(exampleUserAgent1))
 		c.OnResponse(func(resp *Response) {