HTTP logger with slog

I wanted to create a logging middleware for HTTP client.

Working code example can be found on GitHub

The basic structure is

type LoggingTransport struct {
   rt http.RoundTripper
}

func (t *LoggingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
    // do before request is sent, ex. start timer, log request
    resp, err := t.rt.RoundTrip(r)
    // do after the response is received, ex. end timer, log response
    return resp, err
}

I found this code in Kubernetes source and I liked this implementation, I decided to adjust it to my needs.

Using WithOptions pattern

In this example, I am using WithOptions pattern. This pattern sets sane defaults and allows to customize structs.

type Option func(transport *LoggingTransport)

func NewLoggingTransport(options ...Option) *LoggingTransport {
	t := &LoggingTransport{
		rt:             http.DefaultTransport,
		logger:         slog.Default(),
		detailedTiming: false,
	}

	for _, option := range options {
		option(t)
	}

	return t
}

func WithRoundTripper(rt http.RoundTripper) Option {
	return func(t *LoggingTransport) {
		t.rt = rt
	}
}

func WithLogger(logger *slog.Logger) Option {
	return func(t *LoggingTransport) {
		t.logger = logger
	}
}

func WithDetailedTiming(level slog.Level) Option {
	return func(t *LoggingTransport) {
		t.detailedTiming = true
		t.detailedTimingLevel = level
	}
}

Creating requestInfo

This is a struct copied from Kubernetes source. It has all info needed for logging. It also stores durations from net/http/httptrace

// requestInfo keeps track of information about a request/response combination
type requestInfo struct {
	RequestHeaders http.Header
	RequestMethod  string
	RequestURL     string

	ResponseStatus  string
	ResponseHeaders http.Header
	ResponseErr     error

	muTrace          sync.Mutex // Protect trace fields
	DNSLookup        time.Duration
	Dialing          time.Duration
	GetConnection    time.Duration
	TLSHandshake     time.Duration
	ServerProcessing time.Duration
	ConnectionReused bool

	Duration time.Duration
}

func newRequestInfo(r *http.Request) *requestInfo {
	return &requestInfo{
		RequestURL:     r.URL.String(),
		RequestMethod:  r.Method,
		RequestHeaders: r.Header,
	}
}

Logging request info

func (t *LoggingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	rCtx := r.Context()

	reqInfo := newRequestInfo(r)

	methodAttr := slog.String("method", reqInfo.RequestMethod)
	urlAttr := slog.String("url", reqInfo.RequestURL)
	t.logger.DebugContext(rCtx, "request info", methodAttr, urlAttr)

	var headers []any
	for key, values := range reqInfo.RequestHeaders {
		for _, value := range values {
			value = maskValue(key, value)
			headers = append(headers, slog.String(key, value))
		}
	}

	t.logger.DebugContext(rCtx, "request headers", headers...)
// ...
}

Measuring the detailed HTTP timing

If the detailed timing is enabling, the code here measures all the durations with net/http/httptrace package.

// ...
startTime := time.Now()

if t.detailedTiming {
	var getConn, dnsStart, dialStart, tlsStart, serverStart time.Time
	var host string
	trace := &httptrace.ClientTrace{
		// DNS
		DNSStart: func(info httptrace.DNSStartInfo) {
			reqInfo.muTrace.Lock()
			defer reqInfo.muTrace.Unlock()
			dnsStart = time.Now()
			host = info.Host
		},
		DNSDone: func(info httptrace.DNSDoneInfo) {
			reqInfo.muTrace.Lock()
			defer reqInfo.muTrace.Unlock()
			reqInfo.DNSLookup = time.Since(dnsStart)
			t.logger.Log(rCtx, t.detailedTimingLevel, "HTTP Trace", slog.String("DNS_lookup", host), slog.String("resolved", fmt.Sprintf("%v", info.Addrs)))
		},
		// Dial
		ConnectStart: func(network, addr string) {
			reqInfo.muTrace.Lock()
			defer reqInfo.muTrace.Unlock()
			dialStart = time.Now()
		},
		ConnectDone: func(network, addr string, err error) {
			reqInfo.muTrace.Lock()
			defer reqInfo.muTrace.Unlock()
			reqInfo.Dialing = time.Since(dialStart)
			if err != nil {
				t.logger.Log(rCtx, t.detailedTimingLevel, "HTTP Trace: Dial failed", slog.String("network", network), slog.String("addr", addr), slog.Any("error", err))
			} else {
				t.logger.Log(rCtx, t.detailedTimingLevel, "HTTP Trace: Dial succeed", slog.String("network", network), slog.String("addr", addr))
			}
		},
		// TLS
		TLSHandshakeStart: func() {
			tlsStart = time.Now()
		},
		TLSHandshakeDone: func(_ tls.ConnectionState, _ error) {
			reqInfo.muTrace.Lock()
			defer reqInfo.muTrace.Unlock()
			reqInfo.TLSHandshake = time.Since(tlsStart)
		},
		// Connection (it can be DNS + Dial or just the time to get one from the connection pool)
		GetConn: func(hostPort string) {
			getConn = time.Now()
		},
		GotConn: func(info httptrace.GotConnInfo) {
			reqInfo.muTrace.Lock()
			defer reqInfo.muTrace.Unlock()
			reqInfo.GetConnection = time.Since(getConn)
			reqInfo.ConnectionReused = info.Reused
		},
		// Server Processing (time since we wrote the request until first byte is received)
		WroteRequest: func(info httptrace.WroteRequestInfo) {
			reqInfo.muTrace.Lock()
			defer reqInfo.muTrace.Unlock()
			serverStart = time.Now()
		},
		GotFirstResponseByte: func() {
			reqInfo.muTrace.Lock()
			defer reqInfo.muTrace.Unlock()
			reqInfo.ServerProcessing = time.Since(serverStart)
		},
	}
	r = r.WithContext(httptrace.WithClientTrace(r.Context(), trace))
}
// ...

Logging response

// ...
resp, err := t.rt.RoundTrip(r)
reqInfo.Duration = time.Since(startTime)

reqInfo.complete(resp, err)

t.logger.InfoContext(rCtx, "response", methodAttr, urlAttr, slog.String("status", reqInfo.ResponseStatus), slog.Int64("Duration_ms", reqInfo.Duration.Nanoseconds()/int64(time.Millisecond)))

if t.detailedTiming {
	var stats []slog.Attr
	if !reqInfo.ConnectionReused {
		stats = append(stats, slog.Int64("DNSLookup_ms", reqInfo.DNSLookup.Nanoseconds()/int64(time.Millisecond)))
		stats = append(stats, slog.Int64("Dial_ms", reqInfo.Dialing.Nanoseconds()/int64(time.Millisecond)))
		stats = append(stats, slog.Int64("TLSHandshake_ms", reqInfo.TLSHandshake.Nanoseconds()/int64(time.Millisecond)))
	} else {
		stats = append(stats, slog.Int64("GetConnection_ms", reqInfo.GetConnection.Nanoseconds()/int64(time.Millisecond)))
	}
	if reqInfo.ServerProcessing != 0 {
		stats = append(stats, slog.Int64("ServerProcessing_ms", reqInfo.ServerProcessing.Nanoseconds()/int64(time.Millisecond)))
	}
	stats = append(stats, slog.Int64("Duration_ms", reqInfo.Duration.Nanoseconds()/int64(time.Millisecond)))
	t.logger.LogAttrs(rCtx, t.detailedTimingLevel, "HTTP statistics", stats...)

	var responseHeaders []slog.Attr
	for key, values := range reqInfo.ResponseHeaders {
		for _, value := range values {
			value = maskValue(key, value)
			responseHeaders = append(responseHeaders, slog.String(key, value))
		}
	}
	t.logger.LogAttrs(rCtx, slog.LevelDebug, "response headers", responseHeaders...)
}

return resp, err
}

Creating the http.Client

logger := slog.New(slog.NewTextHandler(os.Stderr, opts))
slog.SetDefault(logger)

ctx := context.Background()

// creating http.Client with LoggingTransport
// using WithOptions pattern
client := http.Client{Transport: NewLoggingTransport(WithLogger(logger), WithDetailedTiming(LevelTrace))}
req, err := http.NewRequestWithContext(ctx, "GET", "https://httpbin.org/get", nil)
if err != nil {
	slog.ErrorContext(ctx, "Error creating request", slog.Any("error", err))
	return
}
req.Header.Add("Accept", "application/json")
req.Header.Add("Authorization", "Bearer: XXX")

// doing two requests to check the connection reusing
resp, err := client.Do(req)
resp, err = client.Do(req)
if err != nil {
	slog.ErrorContext(ctx, "Error creating request", slog.Any("error", err))
	return
}

bytes, err := io.ReadAll(resp.Body)
if err != nil {
	slog.ErrorContext(ctx, "Error reading body", slog.Any("error", err))
	return
}

fmt.Printf("body: %s\n", string(bytes))

In this example, we are calling https://httpbin.org/get twice. In second run, we are seeing that the connection is reused. GetConnection_ms=0 ServerProcessing_ms=119 Duration_ms=119

Results

time=2024-04-24T11:06:01.913+02:00 level=DEBUG msg="request info" method=GET url=https://httpbin.org/get
time=2024-04-24T11:06:01.913+02:00 level=DEBUG msg="request headers" Authorization=<masked> Accept=application/json
time=2024-04-24T11:06:02.709+02:00 level=DEBUG msg=response method=GET url=https://httpbin.org/get status="200 OK" Duration_ms=796
time=2024-04-24T11:06:02.709+02:00 level=DEBUG msg="response headers" Content-Type=application/json Content-Length=344 Server=gunicorn/19.9.0 Access-Control-Allow-Origin=* Access-Control-Allow-Credentials=true Date="Wed, 24 Apr 2024 09:06:02 GMT"
time=2024-04-24T11:06:02.709+02:00 level=DEBUG msg="request info" method=GET url=https://httpbin.org/get
time=2024-04-24T11:06:02.709+02:00 level=DEBUG msg="request headers" Accept=application/json Authorization=<masked>
time=2024-04-24T11:06:02.833+02:00 level=DEBUG msg=response method=GET url=https://httpbin.org/get status="200 OK" Duration_ms=123
time=2024-04-24T11:06:02.833+02:00 level=DEBUG msg="response headers" Content-Length=344 Server=gunicorn/19.9.0 Access-Control-Allow-Origin=* Access-Control-Allow-Credentials=true Date="Wed, 24 Apr 2024 09:06:02 GMT" Content-Type=application/json
body: {
  "args": {}, 
  "headers": {
    "Accept": "application/json", 
    "Accept-Encoding": "gzip", 
    "Authorization": "Bearer: XXX", 
    "Host": "httpbin.org", 
    "User-Agent": "Go-http-client/2.0", 
    "X-Amzn-Trace-Id": "Root=1-6628cb7a-40016ef64cca01b00c936393"
  }, 
  "origin": "xxx", 
  "url": "https://httpbin.org/get"
}

With detailed timing

time=2024-04-24T11:04:56.793+02:00 level=DEBUG msg="request info" method=GET url=https://httpbin.org/get
time=2024-04-24T11:04:56.794+02:00 level=DEBUG msg="request headers" Accept=application/json Authorization=<masked>
time=2024-04-24T11:04:56.842+02:00 level=TRACE msg="HTTP Trace" DNS_lookup=httpbin.org resolved="[{54.91.228.73 } {3.221.38.252 } {34.236.135.224 } {52.200.250.127 } {34.196.110.25 } {174.129.50.9 } {3.211.223.136 } {23.23.165.157 }]"
time=2024-04-24T11:04:56.968+02:00 level=TRACE msg="HTTP Trace: Dial succeed" network=tcp addr=54.91.228.73:443
time=2024-04-24T11:04:57.341+02:00 level=DEBUG msg=response method=GET url=https://httpbin.org/get status="200 OK" Duration_ms=547
time=2024-04-24T11:04:57.341+02:00 level=TRACE msg="HTTP statistics" DNSLookup_ms=47 Dial_ms=125 TLSHandshake_ms=251 ServerProcessing_ms=121 Duration_ms=547
time=2024-04-24T11:04:57.341+02:00 level=DEBUG msg="response headers" Content-Type=application/json Content-Length=344 Server=gunicorn/19.9.0 Access-Control-Allow-Origin=* Access-Control-Allow-Credentials=true Date="Wed, 24 Apr 2024 09:04:57 GMT"
time=2024-04-24T11:04:57.341+02:00 level=DEBUG msg="request info" method=GET url=https://httpbin.org/get
time=2024-04-24T11:04:57.341+02:00 level=DEBUG msg="request headers" Accept=application/json Authorization=<masked>
time=2024-04-24T11:04:57.461+02:00 level=DEBUG msg=response method=GET url=https://httpbin.org/get status="200 OK" Duration_ms=119
time=2024-04-24T11:04:57.461+02:00 level=TRACE msg="HTTP statistics" GetConnection_ms=0 ServerProcessing_ms=119 Duration_ms=119
time=2024-04-24T11:04:57.461+02:00 level=DEBUG msg="response headers" Access-Control-Allow-Credentials=true Date="Wed, 24 Apr 2024 09:04:57 GMT" Content-Type=application/json Content-Length=344 Server=gunicorn/19.9.0 Access-Control-Allow-Origin=*
body: {
  "args": {}, 
  "headers": {
    "Accept": "application/json", 
    "Accept-Encoding": "gzip", 
    "Authorization": "Bearer: XXX", 
    "Host": "httpbin.org", 
    "User-Agent": "Go-http-client/2.0", 
    "X-Amzn-Trace-Id": "Root=1-6628cb39-643d17a8277c8fea613f82dd"
  }, 
  "origin": "xxx", 
  "url": "https://httpbin.org/get"
}

Join the Newsletter

Subscribe to get my latest content by email.

    I won't send you spam. Unsubscribe at any time.

    For comments, please send me 📧 an email