1 package http
2
3 import (
4 "context"
5 "crypto/tls"
6 "flag"
7 "fmt"
8 "io/ioutil"
9 "net"
10 "net/http"
11 "strings"
12 "time"
13
14 "golang.conradwood.net/go-easyops/errors"
15 "golang.conradwood.net/go-easyops/prometheus"
16 "google.golang.org/grpc/codes"
17 )
18
19 var (
20 header_timeout = flag.Duration("ge_http_header_timeout", time.Duration(3)*time.Second, "how long to wait for headers after establishing tcp connection")
21 durationSummary = prometheus.NewSummaryVec(
22 prometheus.SummaryOpts{
23 Name: "goeasyops_httpclient_duration",
24 Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
25 Help: "V=1 unit=s DESC=execution time of successful http calls",
26 }, []string{"name"},
27 )
28 callcounter = prometheus.NewCounterVec(
29 prometheus.CounterOpts{
30 Name: "goeasyops_httpclient_total_calls",
31 Help: "V=1 unit=ops DESC=total number of outbound http calls",
32 }, []string{"name"},
33 )
34 failcounter = prometheus.NewCounterVec(
35 prometheus.CounterOpts{
36 Name: "goeasyops_httpclient_failures",
37 Help: "V=1 unit=ops DESC=number of failed outbound http calls",
38 }, []string{"name"},
39 )
40
41 tr = &http.Transport{
42 TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
43 MaxIdleConns: 50,
44 MaxIdleConnsPerHost: 10,
45 IdleConnTimeout: 3 * time.Second,
46 ResponseHeaderTimeout: *header_timeout,
47 ExpectContinueTimeout: 5 * time.Second,
48 DialContext: (&net.Dialer{
49 Timeout: 5 * time.Second,
50 KeepAlive: 30 * time.Second,
51 DualStack: true,
52 }).DialContext,
53 }
54 mytr = &transport{t: tr}
55 )
56
57 func init() {
58 prometheus.MustRegister(durationSummary, callcounter, failcounter)
59 }
60
61 type HTTP struct {
62 MetricName string
63 username string
64 password string
65 err error
66 headers map[string]string
67 jar *Cookies
68 transport *transport
69 debug bool
70 }
71
72 func (h *HTTP) Debugf(format string, args ...interface{}) {
73 if !*debug && !h.debug {
74 return
75 }
76 s := fmt.Sprintf(format, args...)
77 fmt.Printf("[go-easyops/http] %s", s)
78 }
79 func (h *HTTP) SetDebug(b bool) {
80 h.debug = b
81 }
82 func (h *HTTP) SetTimeout(dur time.Duration) {
83 if h.transport == nil {
84 h.transport = &transport{t: &http.Transport{
85 TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
86 MaxIdleConns: 50,
87 MaxIdleConnsPerHost: 10,
88 IdleConnTimeout: dur,
89 ResponseHeaderTimeout: dur,
90 ExpectContinueTimeout: dur,
91 DialContext: (&net.Dialer{
92 Timeout: dur,
93 KeepAlive: 30 * time.Second,
94 DualStack: true,
95 }).DialContext,
96 },
97 }
98 } else {
99 h.transport.t.IdleConnTimeout = dur
100 h.transport.t.ResponseHeaderTimeout = dur
101 h.transport.t.ExpectContinueTimeout = dur
102 h.transport.t.DialContext = (&net.Dialer{
103 Timeout: dur,
104 KeepAlive: 30 * time.Second,
105 DualStack: true,
106 }).DialContext
107 }
108 }
109 func (h *HTTP) promLabels() prometheus.Labels {
110 return prometheus.Labels{"name": h.MetricName}
111 }
112 func (h *HTTP) doMetric() bool {
113 return h.MetricName != ""
114 }
115 func WithAuth(username string, password string) *HTTP {
116 res := &HTTP{username: username, password: password}
117 if username == "" {
118 res.err = fmt.Errorf("Missing username")
119 }
120 return res
121 }
122 func (h *HTTP) SetHeader(key string, value string) {
123 if h.headers == nil {
124 h.headers = make(map[string]string)
125 }
126 h.headers[key] = value
127 }
128
129 func (h *HTTP) Head(url string) *HTTPResponse {
130 hr := &HTTPResponse{ht: h}
131 if h.err != nil {
132 hr.err = h.err
133 return hr
134 }
135 req, err := http.NewRequest("HEAD", url, nil)
136 if err != nil {
137 hr.err = err
138 return hr
139 }
140 h.do(hr, req, true)
141 return hr
142 }
143 func (h *HTTP) GetStream(url string) *HTTPResponse {
144 hr := &HTTPResponse{ht: h}
145 if h.err != nil {
146 hr.err = h.err
147 return hr
148 }
149 req, err := http.NewRequest("GET", url, nil)
150 if err != nil {
151 hr.err = err
152 return hr
153 }
154 return h.do(hr, req, false)
155 }
156 func (h *HTTP) Get(url string) *HTTPResponse {
157 h.Debugf("Get request to \"%s\"\n", url)
158 hr := &HTTPResponse{ht: h}
159 if h.err != nil {
160 hr.err = h.err
161 return hr
162 }
163 req, err := http.NewRequest("GET", url, nil)
164 if err != nil {
165 hr.err = err
166 return hr
167 }
168 h.do(hr, req, true)
169 return hr
170 }
171 func (h *HTTP) Delete(url string, body []byte) *HTTPResponse {
172 hr := &HTTPResponse{ht: h}
173 if h.err != nil {
174 hr.err = h.err
175 return hr
176 }
177 b := strings.NewReader(string(body))
178 req, err := http.NewRequest("DELETE", url, b)
179 if err != nil {
180 hr.err = err
181 return hr
182 }
183 h.do(hr, req, true)
184 return hr
185 }
186 func (h *HTTP) Post(url string, body []byte) *HTTPResponse {
187 hr := &HTTPResponse{ht: h}
188 if h.err != nil {
189 hr.err = h.err
190 return hr
191 }
192 h.Debugf("Body: \"%s\"\n", string(body))
193 b := strings.NewReader(string(body))
194 req, err := http.NewRequest("POST", url, b)
195 if err != nil {
196 hr.err = err
197 return hr
198 }
199 h.do(hr, req, true)
200 return hr
201 }
202 func (h *HTTP) Put(url string, body string) *HTTPResponse {
203 hr := &HTTPResponse{ht: h}
204 if h.err != nil {
205 hr.err = h.err
206 return hr
207 }
208 b := strings.NewReader(body)
209 req, err := http.NewRequest("PUT", url, b)
210 if err != nil {
211 hr.err = err
212 return hr
213 }
214 h.do(hr, req, true)
215 return hr
216 }
217
218
219 func Get(url string) ([]byte, error) {
220 h := &HTTP{}
221 res := h.Get(url)
222 err := res.Error()
223 if err != nil {
224 return nil, err
225 }
226 code := res.HTTPCode()
227 if code < 200 || code >= 300 {
228 return nil, fmt.Errorf("failed to download %s, HTTP Status: %d", url, code)
229 }
230 return res.Body(), nil
231 }
232 func Post(url string, body []byte) ([]byte, error) {
233 h := &HTTP{}
234 res := h.Post(url, body)
235 return res.Body(), res.Error()
236 }
237 func Put(url string, body string) ([]byte, error) {
238 h := &HTTP{}
239 res := h.Put(url, body)
240 return res.Body(), res.Error()
241 }
242
243 func (h *HTTP) Cookies() []*http.Cookie {
244 if h.jar == nil {
245 return nil
246 }
247 return h.jar.cookies
248 }
249 func (h *HTTP) Cookie(name string) *http.Cookie {
250 if h.jar == nil {
251 return nil
252 }
253 for _, c := range h.jar.cookies {
254 if c.Name == name {
255 return c
256 }
257 }
258 return nil
259 }
260 func (h *HTTP) SetCreds(username, password string) {
261 h.username = username
262 h.password = password
263 }
264 func (h *HTTP) do(hr *HTTPResponse, req *http.Request, readbody bool) *HTTPResponse {
265 cr := &cred_producer{host: req.Host}
266 if h.username != "" {
267 h.Debugf("Setting username %s\n", h.username)
268 cr.AddUsernamePassword(h.username, h.password)
269 }
270 retry_counter := 0
271 retry:
272 retry_counter++
273 if h.jar == nil {
274 h.jar = &Cookies{}
275 }
276
277 ctx := context.Background()
278 username := "none"
279 var creds *creds
280 if retry_counter > 1 {
281 creds = cr.GetCredentials()
282 if creds != nil {
283 username = creds.username
284 req.SetBasicAuth(creds.username, creds.password)
285 }
286 }
287 h.Debugf("request attempt #%d started (username=%s, cookies=%d)\n", retry_counter, username, len(h.jar.cookies))
288 tr := mytr
289 if hr.ht.transport != nil {
290 tr = hr.ht.transport
291 }
292 hclient := &http.Client{Transport: tr, Jar: h.jar}
293 h.jar.Print()
294 if h.headers != nil {
295 for k, v := range h.headers {
296 h.Debugf("Header \"%s\" = \"%s\"\n", k, v)
297 req.Header.Set(k, v)
298
299 if strings.ToLower(k) == "host" {
300 req.Host = v
301 }
302
303 }
304 }
305 h.Debugf("Sending %d cookies\n", len(h.jar.cookies))
306
307 for _, c := range h.jar.cookies {
308 h.Debugf("Adding cookie %s\n", c.Name)
309 req.Header.Add("Cookie", fmt.Sprintf("%s=%s", c.Name, c.Value))
310 }
311 started := time.Now()
312 if h.doMetric() {
313 callcounter.With(h.promLabels()).Inc()
314 }
315 h.Debugf("http: %s %s\n", req.Method, req.URL)
316 resp, err := hclient.Do(req)
317 if resp != nil {
318 if resp.StatusCode == 401 {
319 if creds != nil {
320 if *debug {
321 fmt.Printf("url failed for user %s\n", creds.username)
322 }
323 goto retry
324 }
325 }
326 }
327 if resp != nil {
328 hr.httpCode = resp.StatusCode
329 hr.finalurl = resp.Request.URL.String()
330 }
331 if err != nil {
332 if h.doMetric() {
333 failcounter.With(h.promLabels()).Inc()
334 }
335 hr.err = err
336 return hr
337 }
338
339 h.Debugf("Request to %s complete (code=%d)\n", hr.FinalURL(), hr.HTTPCode())
340
341 hr.header = make(map[string]string)
342 for k, va := range resp.Header {
343 if len(va) == 0 {
344 continue
345 }
346 k = strings.ToLower(k)
347 for _, v := range va {
348 hr.allheaders = append(hr.allheaders, &header{Name: k, Value: v})
349 }
350 hr.header[k] = va[0]
351 }
352 hr.resp = resp
353 hr.received_cookies = resp.Cookies()
354 h.Debugf("Received %d cookies\n", len(hr.received_cookies))
355
356 if readbody {
357 defer resp.Body.Close()
358 pbody, err := ioutil.ReadAll(resp.Body)
359 if err != nil {
360 hr.err = err
361 return hr
362 }
363 hr.setBody(pbody)
364 }
365 if resp.StatusCode == 404 {
366 if h.doMetric() {
367 failcounter.With(h.promLabels()).Inc()
368 }
369 h.err = errors.Error(ctx, codes.NotFound, "not found", "%s not found", req.URL)
370 } else if resp.StatusCode > 299 || resp.StatusCode < 200 {
371 if h.doMetric() {
372 failcounter.With(h.promLabels()).Inc()
373 }
374 h.err = fmt.Errorf("Http to \"%s\" failed with code %d", req.URL, resp.StatusCode)
375 }
376 if h.err == nil {
377 durationSummary.With(h.promLabels()).Observe(time.Since(started).Seconds())
378 }
379 return hr
380 }
381
382 type transport struct {
383 t *http.Transport
384 }
385
386
387
388 func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) {
389 if *debug {
390
391
392 fmt.Printf("URL: %s\n", req.URL)
393 }
394 return t.t.RoundTrip(req)
395 }
396
View as plain text