Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cleanup(enginenetx): remove previous policies #1593

Merged
merged 134 commits into from
May 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
134 commits
Select commit Hold shift + click to select a range
80b5e1f
doc(enginenetx): improve documentation
bassosimone Apr 15, 2024
62c3917
fix(enginenetx): mix bridges and DNS tactics
bassosimone Apr 15, 2024
0d2b0f2
fix: update test name and add comment
bassosimone Apr 15, 2024
aee17cf
feat: test for the mixed policies case
bassosimone Apr 15, 2024
7576fc7
feat(enginenetx): add support for filtering tactics
bassosimone Apr 15, 2024
8120c06
refactor: rename function
bassosimone Apr 15, 2024
3207255
feat: improve TCP connect statistics
bassosimone Apr 15, 2024
5618b72
fix previous
bassosimone Apr 15, 2024
2153e36
feat: start to prepare for filtering endpoints
bassosimone Apr 16, 2024
e5cdbb0
ongoing work while documenting and clarifying
bassosimone Apr 16, 2024
089f70b
x
bassosimone Apr 16, 2024
20e71e8
[ci skip]
bassosimone Apr 16, 2024
b5b2e49
[ci skip]
bassosimone Apr 16, 2024
94eb284
[ci skip]
bassosimone Apr 16, 2024
90601c6
[ci skip]
bassosimone Apr 16, 2024
ef8fdfe
[ci skip]
bassosimone Apr 16, 2024
aa65cb7
[ci skip]
bassosimone Apr 16, 2024
28a6265
[ci skip]
bassosimone Apr 16, 2024
f452208
[ci skip]
bassosimone Apr 16, 2024
119e610
[ci skip]
bassosimone Apr 16, 2024
f3fb1dd
[ci skip]
bassosimone Apr 16, 2024
ce6ec84
[ci skip]
bassosimone Apr 16, 2024
4f63b60
[ci skip]
bassosimone Apr 16, 2024
45e655c
x
bassosimone Apr 16, 2024
8e2a1f3
x
bassosimone Apr 16, 2024
e2aed07
x
bassosimone Apr 16, 2024
08fbf48
[ci skip]
bassosimone Apr 16, 2024
7c6ab4b
[ci skip]
bassosimone Apr 16, 2024
c843241
[ci skip]
bassosimone Apr 16, 2024
7f16577
[ci skip]
bassosimone Apr 16, 2024
4b0c768
[ci skip]
bassosimone Apr 16, 2024
7ea130d
[ci skip]
bassosimone Apr 16, 2024
1cbc109
[ci skip]
bassosimone Apr 16, 2024
7edfbb8
[ci skip]
bassosimone Apr 16, 2024
21f9b90
[ci skip]
bassosimone Apr 16, 2024
8e5fee9
[ci skip]
bassosimone Apr 16, 2024
4f8cf91
[ci skip]
bassosimone Apr 16, 2024
436fb50
[ci skip]
bassosimone Apr 16, 2024
b6aebc2
[ci skip]
bassosimone Apr 16, 2024
77b03bd
[ci skip]
bassosimone Apr 16, 2024
f565e68
[ci skip]
bassosimone Apr 16, 2024
e9eee04
[ci skip]
bassosimone Apr 16, 2024
393968d
[ci skip]
bassosimone Apr 16, 2024
665b961
[ci skip]
bassosimone Apr 16, 2024
fb651c7
[ci skip]
bassosimone Apr 16, 2024
cb0dbfc
[ci skip]
bassosimone Apr 16, 2024
bfc0a1d
[ci skip]
bassosimone Apr 16, 2024
02660de
[ci skip]
bassosimone Apr 16, 2024
f707616
[ci skip]
bassosimone Apr 16, 2024
492ab69
[ci skip]
bassosimone Apr 16, 2024
3b63fbd
[ci skip]
bassosimone Apr 16, 2024
0c06b53
[ci skip]
bassosimone Apr 16, 2024
8691606
[ci skip]
bassosimone Apr 16, 2024
20f800a
[ci skip]
bassosimone Apr 16, 2024
e02c5d4
[ci skip]
bassosimone Apr 16, 2024
dd128d8
[ci skip]
bassosimone Apr 16, 2024
8c5bc60
[ci skip]
bassosimone Apr 16, 2024
6e47258
[ci skip]
bassosimone Apr 16, 2024
018fec4
[ci skip]
bassosimone Apr 16, 2024
c834599
[ci skip]
bassosimone Apr 16, 2024
ccd26c4
[ci skip]
bassosimone Apr 16, 2024
0b32b47
[ci skip]
bassosimone Apr 16, 2024
6066a7f
[ci skip]
bassosimone Apr 16, 2024
6c83c25
[ci skip]
bassosimone Apr 16, 2024
19fc4b5
[ci skip]
bassosimone Apr 16, 2024
856d261
[ci skip]
bassosimone Apr 16, 2024
2b7a881
[ci skip]
bassosimone Apr 16, 2024
b7327e2
[ci skip]
bassosimone Apr 16, 2024
3399fec
[ci skip]
bassosimone Apr 16, 2024
baef14f
[ci skip]
bassosimone Apr 16, 2024
15d28f2
[ci skip]
bassosimone Apr 16, 2024
0baaf9b
[ci skip]
bassosimone Apr 16, 2024
49bbf25
[ci skip]
bassosimone Apr 16, 2024
d739ddd
[ci skip]
bassosimone Apr 16, 2024
4896f68
[ci skip]
bassosimone Apr 16, 2024
8bdbbaf
x
bassosimone Apr 16, 2024
4e3a8af
the design document should now be good
bassosimone Apr 16, 2024
fd81cf7
x
bassosimone Apr 16, 2024
8a93533
x
bassosimone Apr 16, 2024
2949035
remove more code that we probably don't need
bassosimone Apr 16, 2024
c075625
[ci skip]
bassosimone Apr 17, 2024
8d14587
[ci skip]
bassosimone Apr 17, 2024
7a2a360
[ci skip]
bassosimone Apr 17, 2024
dfed510
[ci skip]
bassosimone Apr 17, 2024
be41fa9
[ci skip]
bassosimone Apr 17, 2024
5f7302a
[ci skip]
bassosimone Apr 17, 2024
7443710
[ci skip]
bassosimone Apr 17, 2024
2944ea0
[ci skip]
bassosimone Apr 17, 2024
43c1e7c
[ci skip]
bassosimone Apr 17, 2024
d6159d6
[ci skip]
bassosimone Apr 17, 2024
dcf4a03
[ci skip]
bassosimone Apr 17, 2024
f037eb3
[ci skip]
bassosimone Apr 17, 2024
586c450
[ci skip]
bassosimone Apr 17, 2024
de31fd5
[ci skip]
bassosimone Apr 17, 2024
8a3eef7
[ci skip]
bassosimone Apr 17, 2024
697f6b3
[ci skip]
bassosimone Apr 17, 2024
e652147
[ci skip]
bassosimone Apr 17, 2024
4baca70
[ci skip]
bassosimone Apr 17, 2024
fa1c237
[ci skip]
bassosimone Apr 17, 2024
75eb2df
Merge branch 'master' into issue/2704
bassosimone Apr 17, 2024
fd9c568
Merge branch 'master' into issue/2704
bassosimone Apr 17, 2024
d0f721a
x
bassosimone Apr 17, 2024
7058b7d
x
bassosimone Apr 17, 2024
4e628db
x
bassosimone Apr 17, 2024
3b0e110
x
bassosimone Apr 17, 2024
72e1336
Merge branch 'master' into issue/2704
bassosimone Apr 17, 2024
32178c6
x
bassosimone Apr 17, 2024
e7c7541
x
bassosimone Apr 17, 2024
f16784c
x
bassosimone Apr 17, 2024
28bcd5f
[ci skip]
bassosimone Apr 17, 2024
da06bbb
[ci skip]
bassosimone Apr 17, 2024
67ae4d9
[ci skip]
bassosimone Apr 17, 2024
02ee95d
[ci skip]
bassosimone Apr 17, 2024
3877575
fix: adapt test after remix change
bassosimone Apr 18, 2024
af87c6e
doc: add table of contents
bassosimone Apr 18, 2024
1a50266
doc: rename the goals section to be more clear
bassosimone Apr 18, 2024
139f10c
Update internal/enginenetx/DESIGN.md
bassosimone Apr 26, 2024
1d8a786
Update internal/enginenetx/DESIGN.md
bassosimone Apr 26, 2024
a161fa1
Merge branch 'master' into issue/2704
bassosimone Apr 26, 2024
bffc811
Apply suggestions from @hellais
bassosimone May 7, 2024
e9765df
Merge branch 'master' into issue/2704
bassosimone May 7, 2024
9d2c47f
feat: allow chaining policies after the DNS
bassosimone May 8, 2024
f7509d9
feat: implement the mix-with-interleaving policy
bassosimone May 8, 2024
ab3926e
fix: we actually need to get rid of fallbacks
bassosimone May 8, 2024
d4f60b1
refactor(bridgespolicy.go): use free functions
bassosimone May 8, 2024
b0f88bd
feat: introduce fallback-less v2 policies
bassosimone May 8, 2024
35d667f
Shove all the changes I have together
bassosimone May 8, 2024
2582614
chore: add more debugging info
bassosimone May 9, 2024
ba05f4c
x
bassosimone May 9, 2024
1e818f4
x
bassosimone May 9, 2024
f61e96b
Merge branch 'master' into issue/2704
bassosimone May 9, 2024
fb70e04
x
bassosimone May 9, 2024
4614120
x
bassosimone May 9, 2024
4c4ae84
x
bassosimone May 9, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 0 additions & 61 deletions internal/enginenetx/bridgespolicy.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ package enginenetx
import (
"context"
"math/rand"
"slices"
"time"
)

Expand All @@ -32,66 +31,6 @@ func (p *bridgesPolicyV2) LookupTactics(ctx context.Context, domain, port string
return bridgesTacticsForDomain(domain, port)
}

// bridgesPolicy is a policy where we use bridges for communicating
// with the OONI backend, i.e., api.ooni.io.
//
// A bridge is an IP address that can route traffic from and to
// the OONI backend and accepts any SNI.
//
// The zero value is invalid; please, init MANDATORY fields.
type bridgesPolicy struct {
// Fallback is the MANDATORY fallback policy.
Fallback httpsDialerPolicy
}

var _ httpsDialerPolicy = &bridgesPolicy{}

// LookupTactics implements httpsDialerPolicy.
func (p *bridgesPolicy) LookupTactics(ctx context.Context, domain, port string) <-chan *httpsDialerTactic {
return mixSequentially(
// emit bridges related tactics first which are empty if there are
// no bridges for the givend domain and port
bridgesTacticsForDomain(domain, port),

// now fallback to get more tactics (typically here the fallback
// uses the DNS and obtains some extra tactics)
//
// we wrap whatever the underlying policy returns us with some
// extra logic for better communicating with test helpers
p.maybeRewriteTestHelpersTactics(p.Fallback.LookupTactics(ctx, domain, port)),
)
}

func (p *bridgesPolicy) maybeRewriteTestHelpersTactics(input <-chan *httpsDialerTactic) <-chan *httpsDialerTactic {
out := make(chan *httpsDialerTactic)

go func() {
defer close(out) // tell the parent when we're done

for tactic := range input {
// When we're not connecting to a TH, pass the policy down the chain unmodified
if !slices.Contains(testHelpersDomains, tactic.VerifyHostname) {
out <- tactic
continue
}

// This is the case where we're connecting to a test helper. Let's try
// to produce policies hiding the SNI to censoring middleboxes.
for _, sni := range bridgesDomainsInRandomOrder() {
out <- &httpsDialerTactic{
Address: tactic.Address,
InitialDelay: 0, // set when dialing
Port: tactic.Port,
SNI: sni,
VerifyHostname: tactic.VerifyHostname,
}
}
}
}()

return out
}

func bridgesTacticsForDomain(domain, port string) <-chan *httpsDialerTactic {
out := make(chan *httpsDialerTactic)

Expand Down
240 changes: 0 additions & 240 deletions internal/enginenetx/bridgespolicy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,7 @@ package enginenetx

import (
"context"
"errors"
"testing"

"github.com/ooni/probe-cli/v3/internal/mocks"
"github.com/ooni/probe-cli/v3/internal/model"
)

func TestBridgesPolicyV2(t *testing.T) {
Expand Down Expand Up @@ -63,239 +59,3 @@ func TestBridgesPolicyV2(t *testing.T) {
}
})
}

func TestBridgesPolicy(t *testing.T) {
t.Run("for domains for which we don't have bridges and DNS failure", func(t *testing.T) {
expected := errors.New("mocked error")
p := &bridgesPolicy{
Fallback: &dnsPolicy{
Logger: model.DiscardLogger,
Resolver: &mocks.Resolver{
MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
return nil, expected
},
},
},
}

ctx := context.Background()
tactics := p.LookupTactics(ctx, "www.example.com", "443")

var count int
for range tactics {
count++
}

if count != 0 {
t.Fatal("expected to see zero tactics")
}
})

t.Run("for domains for which we don't have bridges and DNS success", func(t *testing.T) {
p := &bridgesPolicy{
Fallback: &dnsPolicy{
Logger: model.DiscardLogger,
Resolver: &mocks.Resolver{
MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
return []string{"93.184.216.34"}, nil
},
},
},
}

ctx := context.Background()
tactics := p.LookupTactics(ctx, "www.example.com", "443")

var count int
for tactic := range tactics {
count++

if tactic.Port != "443" {
t.Fatal("the port should always be 443")
}
if tactic.Address != "93.184.216.34" {
t.Fatal("the host should always be 93.184.216.34")
}

if tactic.InitialDelay != 0 {
t.Fatal("unexpected InitialDelay")
}

if tactic.SNI != "www.example.com" {
t.Fatal("the SNI field should always be like `www.example.com`")
}

if tactic.VerifyHostname != "www.example.com" {
t.Fatal("the VerifyHostname field should always be like `www.example.com`")
}
}

if count != 1 {
t.Fatal("expected to see one tactic")
}
})

t.Run("for the api.ooni.io domain with DNS failure", func(t *testing.T) {
expected := errors.New("mocked error")
p := &bridgesPolicy{
Fallback: &dnsPolicy{
Logger: model.DiscardLogger,
Resolver: &mocks.Resolver{
MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
return nil, expected
},
},
},
}

ctx := context.Background()
tactics := p.LookupTactics(ctx, "api.ooni.io", "443")

// since the DNS fails, we should only see tactics generated by bridges
var count int
for tactic := range tactics {
count++

if tactic.Port != "443" {
t.Fatal("the port should always be 443")
}
if tactic.Address != "162.55.247.208" {
t.Fatal("the host should always be 162.55.247.208")
}

if tactic.InitialDelay != 0 {
t.Fatal("unexpected InitialDelay")
}

if tactic.SNI == "api.ooni.io" {
t.Fatal("we should not see the `api.ooni.io` SNI on the wire")
}

if tactic.VerifyHostname != "api.ooni.io" {
t.Fatal("the VerifyHostname field should always be like `api.ooni.io`")
}
}

if count <= 0 {
t.Fatal("expected to see at least one tactic")
}
})

t.Run("for the api.ooni.io domain with DNS success", func(t *testing.T) {
p := &bridgesPolicy{
Fallback: &dnsPolicy{
Logger: model.DiscardLogger,
Resolver: &mocks.Resolver{
MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
return []string{"130.192.91.211"}, nil
},
},
},
}

ctx := context.Background()
tactics := p.LookupTactics(ctx, "api.ooni.io", "443")

// since the DNS succeeds we should see bridge tactics mixed with DNS tactics
var (
bridgesCount int
dnsCount int
overallCount int
)
const expectedDNSEntryCount = 153 // yikes!
for tactic := range tactics {
overallCount++

t.Log(overallCount, tactic)

if tactic.Port != "443" {
t.Fatal("the port should always be 443")
}

switch {
case overallCount == expectedDNSEntryCount:
if tactic.Address != "130.192.91.211" {
t.Fatal("the host should be 130.192.91.211 for count ==", expectedDNSEntryCount)
}

if tactic.SNI != "api.ooni.io" {
t.Fatal("we should see the `api.ooni.io` SNI on the wire for count ==", expectedDNSEntryCount)
}

dnsCount++

default:
if tactic.Address != "162.55.247.208" {
t.Fatal("the host should be 162.55.247.208 for count !=", expectedDNSEntryCount)
}

if tactic.SNI == "api.ooni.io" {
t.Fatal("we should not see the `api.ooni.io` SNI on the wire for count !=", expectedDNSEntryCount)
}

bridgesCount++
}

if tactic.InitialDelay != 0 {
t.Fatal("unexpected InitialDelay")
}

if tactic.VerifyHostname != "api.ooni.io" {
t.Fatal("the VerifyHostname field should always be like `api.ooni.io`")
}
}

if overallCount <= 0 {
t.Fatal("expected to see at least one tactic")
}
if dnsCount != 1 {
t.Fatal("expected to see exactly one DNS based tactic")
}
if bridgesCount <= 0 {
t.Fatal("expected to see at least one bridge tactic")
}
})

t.Run("for test helper domains", func(t *testing.T) {
for _, domain := range testHelpersDomains {
t.Run(domain, func(t *testing.T) {
expectedAddrs := []string{"164.92.180.7"}

p := &bridgesPolicy{
Fallback: &dnsPolicy{
Logger: model.DiscardLogger,
Resolver: &mocks.Resolver{
MockLookupHost: func(ctx context.Context, domain string) ([]string, error) {
return expectedAddrs, nil
},
},
},
}

ctx := context.Background()
for tactic := range p.LookupTactics(ctx, domain, "443") {

if tactic.Address != "164.92.180.7" {
t.Fatal("unexpected .Address")
}

if tactic.InitialDelay != 0 {
t.Fatal("unexpected .InitialDelay")
}

if tactic.Port != "443" {
t.Fatal("unexpected .Port")
}

if tactic.SNI == domain {
t.Fatal("unexpected .Domain")
}

if tactic.VerifyHostname != domain {
t.Fatal("unexpected .VerifyHostname")
}
}
})
}
})
}
Loading
Loading