Skip to content

Commit 6ef3fba

Browse files
bassosimonebassosimone
authored
feat(enginenetx): support HTTP and HTTPS proxies (#1282)
This diff completes the work we have been doing for a few days now and provides HTTP and HTTPS proxy support, in addition to SOCKS5 support, for the engine-specific network. We did this work in the context of ooni/probe#2531 and ooni/probe#1955. BTW, the fact that tests used `measurexlite` and tracing is very nice here. It means the idea to write `measurexlite` based on context and tracing was good and could be used beyond its original design goals.
1 parent d0ea69d commit 6ef3fba

File tree

4 files changed

+259
-28
lines changed

4 files changed

+259
-28
lines changed

‎internal/enginenetx/http.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -48,13 +48,13 @@ func NewHTTPTransport(
4848
resolver model.Resolver,
4949
) *HTTPTransport {
5050
dialer := netxlite.NewDialerWithResolver(logger, resolver)
51-
dialer = netxlite.MaybeWrapWithProxyDialer(dialer, proxyURL)
5251
handshaker := netxlite.NewTLSHandshakerStdlib(logger)
5352
tlsDialer := netxlite.NewTLSDialer(dialer, handshaker)
54-
// TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport
55-
// function, but we can probably avoid using it, given that this code is
56-
// not using tracing and does not care about those quirks.
57-
txp := netxlite.NewHTTPTransport(logger, dialer, tlsDialer)
53+
txp := netxlite.NewHTTPTransportWithOptions(
54+
logger, dialer, tlsDialer,
55+
netxlite.HTTPTransportOptionDisableCompression(false),
56+
netxlite.HTTPTransportOptionProxyURL(proxyURL), // nil implies "no proxy"
57+
)
5858
txp = bytecounter.WrapHTTPTransport(txp, counter)
5959
return &HTTPTransport{txp}
6060
}

‎internal/enginenetx/http_test.go

+247-18
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,263 @@
1-
package enginenetx
1+
package enginenetx_test
22

33
import (
4+
"context"
5+
"net"
6+
"net/http"
7+
"net/url"
48
"testing"
9+
"time"
510

11+
"github.com/apex/log"
612
"github.com/ooni/probe-cli/v3/internal/bytecounter"
13+
"github.com/ooni/probe-cli/v3/internal/enginenetx"
14+
"github.com/ooni/probe-cli/v3/internal/measurexlite"
715
"github.com/ooni/probe-cli/v3/internal/model"
16+
"github.com/ooni/probe-cli/v3/internal/netemx"
817
"github.com/ooni/probe-cli/v3/internal/netxlite"
18+
"github.com/ooni/probe-cli/v3/internal/testingsocks5"
19+
"github.com/ooni/probe-cli/v3/internal/testingx"
920
)
1021

11-
func TestHTTPTransport(t *testing.T) {
22+
func TestHTTPTransportWAI(t *testing.T) {
23+
t.Run("is WAI when not using any proxy", func(t *testing.T) {
24+
env := netemx.MustNewScenario(netemx.InternetScenario)
25+
defer env.Close()
1226

13-
// TODO(bassosimone): we should replace this integration test with netemx
14-
// as soon as we can sever the hard link between netxlite and this pkg
15-
t.Run("is working as intended", func(t *testing.T) {
16-
txp := NewHTTPTransport(
17-
bytecounter.New(), model.DiscardLogger, nil, netxlite.NewStdlibResolver(model.DiscardLogger))
18-
client := txp.NewHTTPClient()
19-
resp, err := client.Get("https://www.google.com/robots.txt")
20-
if err != nil {
21-
t.Fatal(err)
22-
}
23-
defer resp.Body.Close()
24-
if resp.StatusCode != 200 {
25-
t.Fatal("unexpected status code")
26-
}
27+
env.Do(func() {
28+
txp := enginenetx.NewHTTPTransport(
29+
bytecounter.New(),
30+
model.DiscardLogger,
31+
nil,
32+
netxlite.NewStdlibResolver(model.DiscardLogger),
33+
)
34+
client := txp.NewHTTPClient()
35+
resp, err := client.Get("https://www.example.com/")
36+
if err != nil {
37+
t.Fatal(err)
38+
}
39+
t.Logf("%+v", resp)
40+
defer resp.Body.Close()
41+
if resp.StatusCode != 200 {
42+
t.Fatal("unexpected status code")
43+
}
44+
})
45+
})
46+
47+
t.Run("is WAI when using a SOCKS5 proxy", func(t *testing.T) {
48+
// create internet measurement scenario
49+
env := netemx.MustNewScenario(netemx.InternetScenario)
50+
defer env.Close()
51+
52+
// create a proxy using the client's TCP/IP stack
53+
proxy := testingsocks5.MustNewServer(
54+
log.Log,
55+
&netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}},
56+
&net.TCPAddr{
57+
IP: net.ParseIP(env.ClientStack.IPAddress()),
58+
Port: 9050,
59+
},
60+
)
61+
defer proxy.Close()
62+
63+
env.Do(func() {
64+
txp := enginenetx.NewHTTPTransport(
65+
bytecounter.New(),
66+
model.DiscardLogger,
67+
&url.URL{
68+
Scheme: "socks5",
69+
Host: net.JoinHostPort(env.ClientStack.IPAddress(), "9050"),
70+
Path: "/",
71+
},
72+
netxlite.NewStdlibResolver(model.DiscardLogger),
73+
)
74+
client := txp.NewHTTPClient()
75+
76+
// To make sure we're connecting to the expected endpoint, we're going to use
77+
// measurexlite and tracing to observe the destination endpoints
78+
trace := measurexlite.NewTrace(0, time.Now())
79+
ctx := netxlite.ContextWithTrace(context.Background(), trace)
80+
81+
// create request using the above context
82+
//
83+
// Implementation note: we cannot use HTTPS with netem here as explained
84+
// by the https://github.com/ooni/probe/issues/2536 issue.
85+
req, err := http.NewRequestWithContext(ctx, "GET", "http://www.example.com/", nil)
86+
if err != nil {
87+
t.Fatal(err)
88+
}
89+
90+
resp, err := client.Do(req)
91+
if err != nil {
92+
t.Fatal(err)
93+
}
94+
t.Logf("%+v", resp)
95+
defer resp.Body.Close()
96+
if resp.StatusCode != 200 {
97+
t.Fatal("unexpected status code")
98+
}
99+
100+
// make sure that we only connected to the SOCKS5 proxy
101+
tcpConnects := trace.TCPConnects()
102+
if len(tcpConnects) <= 0 {
103+
t.Fatal("expected at least one TCP connect")
104+
}
105+
for idx, entry := range tcpConnects {
106+
t.Logf("%d: %+v", idx, entry)
107+
if entry.IP != env.ClientStack.IPAddress() {
108+
t.Fatal("unexpected IP address")
109+
}
110+
if entry.Port != 9050 {
111+
t.Fatal("unexpected port")
112+
}
113+
}
114+
})
115+
})
116+
117+
t.Run("is WAI when using an HTTP proxy", func(t *testing.T) {
118+
// create internet measurement scenario
119+
env := netemx.MustNewScenario(netemx.InternetScenario)
120+
defer env.Close()
121+
122+
// create a proxy using the client's TCP/IP stack
123+
proxy := testingx.MustNewHTTPServerEx(
124+
&net.TCPAddr{IP: net.ParseIP(env.ClientStack.IPAddress()), Port: 8080},
125+
env.ClientStack,
126+
testingx.NewHTTPProxyHandler(log.Log, &netxlite.Netx{
127+
Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}}),
128+
)
129+
defer proxy.Close()
130+
131+
env.Do(func() {
132+
txp := enginenetx.NewHTTPTransport(
133+
bytecounter.New(),
134+
model.DiscardLogger,
135+
&url.URL{
136+
Scheme: "http",
137+
Host: net.JoinHostPort(env.ClientStack.IPAddress(), "8080"),
138+
Path: "/",
139+
},
140+
netxlite.NewStdlibResolver(model.DiscardLogger),
141+
)
142+
client := txp.NewHTTPClient()
143+
144+
// To make sure we're connecting to the expected endpoint, we're going to use
145+
// measurexlite and tracing to observe the destination endpoints
146+
trace := measurexlite.NewTrace(0, time.Now())
147+
ctx := netxlite.ContextWithTrace(context.Background(), trace)
148+
149+
// create request using the above context
150+
//
151+
// Implementation note: we cannot use HTTPS with netem here as explained
152+
// by the https://github.com/ooni/probe/issues/2536 issue.
153+
req, err := http.NewRequestWithContext(ctx, "GET", "http://www.example.com/", nil)
154+
if err != nil {
155+
t.Fatal(err)
156+
}
157+
158+
resp, err := client.Do(req)
159+
if err != nil {
160+
t.Fatal(err)
161+
}
162+
t.Logf("%+v", resp)
163+
defer resp.Body.Close()
164+
if resp.StatusCode != 200 {
165+
t.Fatal("unexpected status code")
166+
}
167+
168+
// make sure that we only connected to the HTTP proxy
169+
tcpConnects := trace.TCPConnects()
170+
if len(tcpConnects) <= 0 {
171+
t.Fatal("expected at least one TCP connect")
172+
}
173+
for idx, entry := range tcpConnects {
174+
t.Logf("%d: %+v", idx, entry)
175+
if entry.IP != env.ClientStack.IPAddress() {
176+
t.Fatal("unexpected IP address")
177+
}
178+
if entry.Port != 8080 {
179+
t.Fatal("unexpected port")
180+
}
181+
}
182+
})
183+
})
184+
185+
t.Run("is WAI when using an HTTPS proxy", func(t *testing.T) {
186+
// create internet measurement scenario
187+
env := netemx.MustNewScenario(netemx.InternetScenario)
188+
defer env.Close()
189+
190+
// create a proxy using the client's TCP/IP stack
191+
proxy := testingx.MustNewHTTPServerTLSEx(
192+
&net.TCPAddr{IP: net.ParseIP(env.ClientStack.IPAddress()), Port: 4443},
193+
env.ClientStack,
194+
testingx.NewHTTPProxyHandler(log.Log, &netxlite.Netx{
195+
Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}}),
196+
env.ClientStack,
197+
)
198+
defer proxy.Close()
199+
200+
env.Do(func() {
201+
txp := enginenetx.NewHTTPTransport(
202+
bytecounter.New(),
203+
model.DiscardLogger,
204+
&url.URL{
205+
Scheme: "https",
206+
Host: net.JoinHostPort(env.ClientStack.IPAddress(), "4443"),
207+
Path: "/",
208+
},
209+
netxlite.NewStdlibResolver(model.DiscardLogger),
210+
)
211+
client := txp.NewHTTPClient()
212+
213+
// To make sure we're connecting to the expected endpoint, we're going to use
214+
// measurexlite and tracing to observe the destination endpoints
215+
trace := measurexlite.NewTrace(0, time.Now())
216+
ctx := netxlite.ContextWithTrace(context.Background(), trace)
217+
218+
// create request using the above context
219+
//
220+
// Implementation note: we cannot use HTTPS with netem here as explained
221+
// by the https://github.com/ooni/probe/issues/2536 issue.
222+
req, err := http.NewRequestWithContext(ctx, "GET", "http://www.example.com/", nil)
223+
if err != nil {
224+
t.Fatal(err)
225+
}
226+
227+
resp, err := client.Do(req)
228+
if err != nil {
229+
t.Fatal(err)
230+
}
231+
t.Logf("%+v", resp)
232+
defer resp.Body.Close()
233+
if resp.StatusCode != 200 {
234+
t.Fatal("unexpected status code")
235+
}
236+
237+
// make sure that we only connected to the HTTPS proxy
238+
tcpConnects := trace.TCPConnects()
239+
if len(tcpConnects) <= 0 {
240+
t.Fatal("expected at least one TCP connect")
241+
}
242+
for idx, entry := range tcpConnects {
243+
t.Logf("%d: %+v", idx, entry)
244+
if entry.IP != env.ClientStack.IPAddress() {
245+
t.Fatal("unexpected IP address")
246+
}
247+
if entry.Port != 4443 {
248+
t.Fatal("unexpected port")
249+
}
250+
}
251+
})
27252
})
28253

29254
t.Run("NewHTTPClient returns a client with a cookie jar", func(t *testing.T) {
30-
txp := NewHTTPTransport(
31-
bytecounter.New(), model.DiscardLogger, nil, netxlite.NewStdlibResolver(model.DiscardLogger))
255+
txp := enginenetx.NewHTTPTransport(
256+
bytecounter.New(),
257+
model.DiscardLogger,
258+
nil,
259+
netxlite.NewStdlibResolver(model.DiscardLogger),
260+
)
32261
client := txp.NewHTTPClient()
33262
if client.Jar == nil {
34263
t.Fatal("expected non-nil cookie jar")

‎internal/netemx/qaenv.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -140,8 +140,8 @@ type QAEnv struct {
140140
// clientNICWrapper is the OPTIONAL wrapper for the client NIC.
141141
clientNICWrapper netem.LinkNICWrapper
142142

143-
// clientStack is the client stack to use.
144-
clientStack *netem.UNetStack
143+
// ClientStack is the client stack to use.
144+
ClientStack *netem.UNetStack
145145

146146
// closables contains all entities where we have to take care of closing.
147147
closables []io.Closer
@@ -197,7 +197,7 @@ func MustNewQAEnv(options ...QAEnvOption) *QAEnv {
197197
env := &QAEnv{
198198
baseLogger: config.logger,
199199
clientNICWrapper: config.clientNICWrapper,
200-
clientStack: nil,
200+
ClientStack: nil,
201201
closables: []io.Closer{},
202202
emulateAndroidGetaddrinfo: &atomic.Bool{},
203203
ispResolverConfig: netem.NewDNSConfig(),
@@ -208,7 +208,7 @@ func MustNewQAEnv(options ...QAEnvOption) *QAEnv {
208208
}
209209

210210
// create all the required internals
211-
env.clientStack = env.mustNewClientStack(config)
211+
env.ClientStack = env.mustNewClientStack(config)
212212
env.closables = append(env.closables, env.mustNewNetStacks(config)...)
213213

214214
return env
@@ -306,7 +306,7 @@ func (env *QAEnv) EmulateAndroidGetaddrinfo(value bool) {
306306
// Do executes the given function such that [netxlite] code uses the
307307
// underlying clientStack rather than ordinary networking code.
308308
func (env *QAEnv) Do(function func()) {
309-
var stack netem.UnderlyingNetwork = env.clientStack
309+
var stack netem.UnderlyingNetwork = env.ClientStack
310310
if env.emulateAndroidGetaddrinfo.Load() {
311311
stack = &androidStack{stack}
312312
}

‎internal/netxlite/maybeproxy.go

+2
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ type proxyDialer struct {
2222

2323
// MaybeWrapWithProxyDialer returns the original dialer if the proxyURL is nil
2424
// and otherwise returns a wrapped dialer that implements proxying.
25+
//
26+
// Deprecated: do not use this function in new code.
2527
func MaybeWrapWithProxyDialer(dialer model.Dialer, proxyURL *url.URL) model.Dialer {
2628
if proxyURL == nil {
2729
return dialer

0 commit comments

Comments
 (0)