Skip to content

Commit c4e7ffc

Browse files
authored
feat(dsl): do not include HTTP body by default (#11)
By making it explicit that we want to include the HTTP body into a measurement we highlight that including the body should only be done when the actual content of the body matters. It makes sense, for example, for Web Connectivity but it does not make sense to do so in many other contexts. Noticed when working on ooni/probe#2502
1 parent 32dae99 commit c4e7ffc

9 files changed

Lines changed: 212 additions & 21 deletions

File tree

pkg/dsl/httpcore.go

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -70,16 +70,17 @@ func (op *httpTransactionOperation) Run(ctx context.Context, rtx Runtime, conn *
7070

7171
// create configuration
7272
config := &httpTransactionConfig{
73-
AcceptHeader: model.HTTPHeaderAccept,
74-
AcceptLanguageHeader: model.HTTPHeaderAcceptLanguage,
75-
HostHeader: conn.Domain,
76-
RefererHeader: "",
77-
RequestMethod: "GET",
78-
ResponseBodySnapshotSize: 1 << 19,
79-
URLHost: conn.Domain,
80-
URLPath: "/",
81-
URLScheme: conn.Scheme,
82-
UserAgentHeader: model.HTTPHeaderUserAgent,
73+
AcceptHeader: model.HTTPHeaderAccept,
74+
AcceptLanguageHeader: model.HTTPHeaderAcceptLanguage,
75+
HostHeader: conn.Domain,
76+
IncludeResponseBodySnapshot: false,
77+
RefererHeader: "",
78+
RequestMethod: "GET",
79+
ResponseBodySnapshotSize: 1 << 19,
80+
URLHost: conn.Domain,
81+
URLPath: "/",
82+
URLScheme: conn.Scheme,
83+
UserAgentHeader: model.HTTPHeaderUserAgent,
8384
}
8485
for _, option := range op.options {
8586
option(config)
@@ -106,7 +107,12 @@ func (op *httpTransactionOperation) Run(ctx context.Context, rtx Runtime, conn *
106107

107108
// mediate the transaction execution via the trace, which gets a chance
108109
// to generate HTTP observations for this transaction
109-
resp, body, err := conn.Trace.HTTPTransaction(conn, req, config.ResponseBodySnapshotSize)
110+
resp, body, err := conn.Trace.HTTPTransaction(
111+
conn,
112+
config.IncludeResponseBodySnapshot,
113+
req,
114+
config.ResponseBodySnapshotSize,
115+
)
110116

111117
// save trace-collected observations (if any)
112118
rtx.SaveObservations(conn.Trace.ExtractObservations()...)

pkg/dsl/httpmodel.go

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,14 @@ func HTTPTransactionOptionHost(value string) HTTPTransactionOption {
5555
}
5656
}
5757

58+
// HTTPTransactionOptionIncludeResponseBodySnapshot controls whether to include the
59+
// response body snapshot in the JSON measurement; the default is false.
60+
func HTTPTransactionOptionIncludeResponseBodySnapshot(value bool) HTTPTransactionOption {
61+
return func(c *httpTransactionConfig) {
62+
c.IncludeResponseBodySnapshot = value
63+
}
64+
}
65+
5866
// HTTPTransactionOptionMethod sets the method.
5967
func HTTPTransactionOptionMethod(value string) HTTPTransactionOption {
6068
return func(c *httpTransactionConfig) {
@@ -104,7 +112,7 @@ func HTTPTransactionOptionUserAgent(value string) HTTPTransactionOption {
104112
}
105113
}
106114

107-
// TODO(bassosimone): we should probably autogenerate the config, the functional optionl
115+
// TODO(bassosimone): we should probably autogenerate the config, the functional optional
108116
// setters, and the conversion from config to list of options.
109117

110118
type httpTransactionConfig struct {
@@ -117,6 +125,10 @@ type httpTransactionConfig struct {
117125
// HostHeader is the host header to use.
118126
HostHeader string `json:"host_header,omitempty"`
119127

128+
// IncludeResponseBodySnapshot tells the engine to include the response body snapshot
129+
// we have read inside the JSON measurement.
130+
IncludeResponseBodySnapshot bool `json:"include_response_body_snapshot,omitempty"`
131+
120132
// RefererHeader is the referer header to use.
121133
RefererHeader string `json:"referer_header,omitempty"`
122134

@@ -149,6 +161,9 @@ func (c *httpTransactionConfig) options() (options []HTTPTransactionOption) {
149161
if value := c.HostHeader; value != "" {
150162
options = append(options, HTTPTransactionOptionHost(value))
151163
}
164+
if value := c.IncludeResponseBodySnapshot; value {
165+
options = append(options, HTTPTransactionOptionIncludeResponseBodySnapshot(value))
166+
}
152167
if value := c.RefererHeader; value != "" {
153168
options = append(options, HTTPTransactionOptionReferer(value))
154169
}

pkg/dsl/measurexlite.go

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,16 @@ var _ Trace = &measurexliteTrace{}
9797

9898
// HTTPTransaction implements Trace.
9999
func (t *measurexliteTrace) HTTPTransaction(
100-
conn *HTTPConnection, req *http.Request, maxBodySnapshotSize int) (*http.Response, []byte, error) {
100+
conn *HTTPConnection,
101+
includeResponseBodySnapshot bool,
102+
req *http.Request,
103+
maxBodySnapshotSize int,
104+
) (*http.Response, []byte, error) {
105+
// make sure the response body snapshot size is non-negative
106+
if maxBodySnapshotSize < 0 {
107+
maxBodySnapshotSize = 0
108+
}
109+
101110
// create the beginning-of-transaction observation
102111
started := t.trace.TimeSince(t.trace.ZeroTime)
103112
t.runtime.saveNetworkEvents(measurexlite.NewAnnotationArchivalNetworkEvent(
@@ -151,7 +160,7 @@ func (t *measurexliteTrace) HTTPTransaction(
151160
req,
152161
resp,
153162
int64(maxBodySnapshotSize),
154-
body,
163+
t.maybeIncludeBody(includeResponseBodySnapshot, body),
155164
err,
156165
finished,
157166
))
@@ -166,6 +175,13 @@ func (t *measurexliteTrace) HTTPTransaction(
166175
return resp, body, err
167176
}
168177

178+
func (t *measurexliteTrace) maybeIncludeBody(includeBody bool, body []byte) []byte {
179+
if !includeBody {
180+
return []byte{}
181+
}
182+
return body
183+
}
184+
169185
// Index implements Trace.
170186
func (t *measurexliteTrace) Index() int64 {
171187
return t.trace.Index

pkg/dsl/measurexlite_test.go

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
package dsl
2+
3+
import (
4+
"context"
5+
"errors"
6+
"net/http"
7+
"net/http/httptest"
8+
"net/url"
9+
"testing"
10+
"time"
11+
12+
"github.com/google/go-cmp/cmp"
13+
"github.com/ooni/probe-engine/pkg/model"
14+
"github.com/ooni/probe-engine/pkg/runtimex"
15+
)
16+
17+
// TestMeasurexliteHTTPIncludeResponseBodySnapshot checks whether we include or not
18+
// include a body snapshot into the JSON measurement depending on the settings.
19+
func TestMeasurexliteHTTPIncludeResponseBodySnapshot(t *testing.T) {
20+
// define the expected response body
21+
expectedBody := []byte("Bonsoir, Elliot!\r\n")
22+
23+
// create local test server
24+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
25+
w.Write(expectedBody)
26+
}))
27+
defer server.Close()
28+
29+
// parse the server's URL
30+
URL := runtimex.Try1(url.Parse(server.URL))
31+
32+
// define the function to create a measurement pipeline
33+
makePipeline := func(options ...HTTPTransactionOption) Stage[*Void, *HTTPResponse] {
34+
return Compose4(
35+
NewEndpoint(URL.Host, NewEndpointOptionDomain("example.com")),
36+
TCPConnect(),
37+
HTTPConnectionTCP(),
38+
HTTPTransaction(options...),
39+
)
40+
}
41+
42+
// define the function to return the body in the pipeline
43+
getPipelineBody := func(input Maybe[*HTTPResponse]) ([]byte, error) {
44+
// handle the case of unexpected error
45+
if input.Error != nil {
46+
return nil, input.Error
47+
}
48+
return input.Value.ResponseBodySnapshot, nil
49+
}
50+
51+
// define the function to return the body in the measurement
52+
getMeasurementBody := func(observations *Observations) ([]byte, error) {
53+
if len(observations.Requests) != 1 {
54+
return nil, errors.New("expected a single request entry")
55+
}
56+
return []byte(observations.Requests[0].Response.Body.Value), nil
57+
}
58+
59+
// define the function to run the measurement
60+
measure := func(options ...HTTPTransactionOption) (Maybe[*HTTPResponse], *Observations) {
61+
pipeline := makePipeline(options...)
62+
rtx := NewMeasurexliteRuntime(model.DiscardLogger, &NullMetrics{}, time.Now())
63+
input := NewValue(&Void{})
64+
output := pipeline.Run(context.Background(), rtx, input)
65+
observations := ReduceObservations(rtx.ExtractObservations()...)
66+
return output, observations
67+
}
68+
69+
t.Run("the default should be that of not including the body", func(t *testing.T) {
70+
output, observations := measure( /* empty */ )
71+
72+
t.Run("the pipeline body should contain the body", func(t *testing.T) {
73+
pipeBody, err := getPipelineBody(output)
74+
if err != nil {
75+
t.Fatal(err)
76+
}
77+
t.Log("pipeline body", pipeBody)
78+
if diff := cmp.Diff(expectedBody, pipeBody); diff != "" {
79+
t.Fatal(diff)
80+
}
81+
})
82+
83+
t.Run("the measurement body should be empty", func(t *testing.T) {
84+
measBody, err := getMeasurementBody(observations)
85+
if err != nil {
86+
t.Fatal(err)
87+
}
88+
t.Log("measurement body", measBody)
89+
if len(measBody) != 0 {
90+
t.Fatal("expected empty body")
91+
}
92+
})
93+
})
94+
95+
t.Run("but we can optionally request to include it", func(t *testing.T) {
96+
output, observations := measure(HTTPTransactionOptionIncludeResponseBodySnapshot(true))
97+
98+
t.Run("the pipeline body should contain the body", func(t *testing.T) {
99+
pipeBody, err := getPipelineBody(output)
100+
if err != nil {
101+
t.Fatal(err)
102+
}
103+
t.Log("pipeline body", pipeBody)
104+
if diff := cmp.Diff(expectedBody, pipeBody); diff != "" {
105+
t.Fatal(diff)
106+
}
107+
})
108+
109+
t.Run("the measurement body should contain the body", func(t *testing.T) {
110+
measBody, err := getMeasurementBody(observations)
111+
if err != nil {
112+
t.Fatal(err)
113+
}
114+
t.Log("measurement body", measBody)
115+
if diff := cmp.Diff(expectedBody, measBody); diff != "" {
116+
t.Fatal(diff)
117+
}
118+
})
119+
})
120+
}

pkg/dsl/metrics.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ type Metrics interface {
1414
Success(name string)
1515
}
1616

17-
// NullMetrics implements [Metrics] but ignores events.
17+
// NullMetrics implements [Metrics] but ignores events. The zero value of
18+
// this structure is ready to use.
1819
type NullMetrics struct{}
1920

2021
// Error implements Metrics.

pkg/dsl/quicmodel.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ type QUICConnection struct {
3232
// QUICHandshakeOption is an option for configuring the QUIC handshake.
3333
type QUICHandshakeOption func(config *quicHandshakeConfig)
3434

35-
// TODO(bassosimone): we should probably autogenerate the config, the functional optionl
35+
// TODO(bassosimone): we should probably autogenerate the config, the functional optional
3636
// setters, and the conversion from config to list of options.
3737

3838
type quicHandshakeConfig struct {

pkg/dsl/runtime.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,11 @@ func (t *minimalTrace) ExtractObservations() []*Observations {
156156

157157
// HTTPTransaction implements Trace.
158158
func (t *minimalTrace) HTTPTransaction(
159-
conn *HTTPConnection, req *http.Request, maxBodySnapshotSize int) (*http.Response, []byte, error) {
159+
conn *HTTPConnection,
160+
includeResponseBodySnapshot bool,
161+
req *http.Request,
162+
maxBodySnapshotSize int,
163+
) (*http.Response, []byte, error) {
160164
// perform round trip
161165
resp, err := conn.Transport.RoundTrip(req)
162166
if err != nil {

pkg/dsl/tlsmodel.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ type TLSConnection struct {
2929
// TLSHandshakeOption is an option for configuring the TLS handshake.
3030
type TLSHandshakeOption func(config *tlsHandshakeConfig)
3131

32-
// TODO(bassosimone): we should probably autogenerate the config, the functional optionl
32+
// TODO(bassosimone): we should probably autogenerate the config, the functional optional
3333
// setters, and the conversion from config to list of options.
3434

3535
type tlsHandshakeConfig struct {

pkg/dsl/trace.go

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,38 @@ type Trace interface {
1111
// ExtractObservations removes and returns the observations saved so far.
1212
ExtractObservations() []*Observations
1313

14-
// HTTPTransaction executes and measures an HTTP transaction. The n argument controls
15-
// the maximum response body snapshot size that we are willing to read.
16-
HTTPTransaction(c *HTTPConnection, r *http.Request, n int) (*http.Response, []byte, error)
14+
// HTTPTransaction executes and measures an HTTP transaction.
15+
//
16+
// Arguments:
17+
//
18+
// - conn is the HTTP connection to use;
19+
//
20+
// - includeResponseBodySnapshot controls whether to include the response body
21+
// snapshot into the JSON measurement;
22+
//
23+
// - request is the HTTP request;
24+
//
25+
// - responseBodySnapshotSize controls the maximum number of bytes of the
26+
// body that we are willing to read (to avoid reading unbounded bodies).
27+
//
28+
// Return values:
29+
//
30+
// - resp is the HTTP response;
31+
//
32+
// - body is the HTTP response body (which MAY be empty if the response body
33+
// snapshot size value is zero or negative);
34+
//
35+
// - err is the error that occurred (nil on success).
36+
HTTPTransaction(
37+
conn *HTTPConnection,
38+
includeResponseBodySnapshot bool,
39+
request *http.Request,
40+
responseBodySnapshotSize int,
41+
) (
42+
resp *http.Response,
43+
body []byte,
44+
err error,
45+
)
1746

1847
// Index is the unique index of this trace.
1948
Index() int64

0 commit comments

Comments
 (0)