diff --git a/.gitlab/ci/test.yml b/.gitlab/ci/test.yml index f0e484b9403dc55e18a536f8116aec6f7f201941..ea4496877032388739c2a6b5467196cb658d8150 100644 --- a/.gitlab/ci/test.yml +++ b/.gitlab/ci/test.yml @@ -17,6 +17,9 @@ .tests-acceptance-deamon: extends: .tests-common + before_script: + - apt-get update + - apt-get install net-tools -y script: - echo "Running just the acceptance tests daemonized (tmpdir)...." - TEST_DAEMONIZE=tmpdir make acceptance @@ -50,6 +53,9 @@ test-acceptance:1.16: race: extends: .tests-common + before_script: + - apt-get update + - apt-get install net-tools -y script: - echo "Running race detector" - make setup diff --git a/go.mod b/go.mod index be92224753bf686c07b477224daf4b85cd3c1288..b35c04af0b32f7fcbbf9717502b08be9a7c662ba 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/andybalholm/brotli v1.0.3 github.com/cenkalti/backoff/v4 v4.0.2 github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/fatih/color v1.9.0 // indirect github.com/golang/mock v1.3.1 github.com/golangci/golangci-lint v1.27.0 github.com/gorilla/context v1.1.1 @@ -17,9 +18,11 @@ require ( github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 github.com/karlseguin/ccache/v2 v2.0.6 github.com/karrick/godirwalk v1.10.12 + github.com/mozilla/mig v0.0.0-20190913234010-9e7e4f525805 github.com/namsral/flag v1.7.4-pre github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pires/go-proxyproto v0.2.0 + github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.6.0 github.com/rs/cors v1.7.0 github.com/sirupsen/logrus v1.7.0 diff --git a/go.sum b/go.sum index 1d1b21d9e39687273568a7886481ca02a36e1a9a..2d76033f049a456e77a27d0dc7732f7eb2e5dcec 100644 --- a/go.sum +++ b/go.sum @@ -81,8 +81,9 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -312,8 +313,9 @@ github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaa github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -331,6 +333,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/mozilla/mig v0.0.0-20190913234010-9e7e4f525805 h1:aR0FDMk+x7dz50dqbvd5EPobe2cCHvRbUKGD6HhSXhk= +github.com/mozilla/mig v0.0.0-20190913234010-9e7e4f525805/go.mod h1:2c03209qVdj62h4aUMg8KfnbIGC83caKZB8/4F/7YV4= github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nakabonne/nestif v0.3.0 h1:+yOViDGhg8ygGrmII72nV9B/zGxY188TYpfolntsaPw= @@ -369,8 +373,9 @@ github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw github.com/pires/go-proxyproto v0.2.0 h1:WyYKlv9pkt77b+LjMvPfwrsAxviaGCFhG4KDIy1ofLY= github.com/pires/go-proxyproto v0.2.0/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9Gw5rnCjcwzAY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= diff --git a/helpers.go b/helpers.go index 17b464d0f29bd2cd0d4947f37edb509478e1a6c9..a36044ebd768d48c5248b8678a3dbe08aa557d18 100644 --- a/helpers.go +++ b/helpers.go @@ -1,8 +1,12 @@ package main import ( + "fmt" + "log" "net" "os" + "os/exec" + "time" "gitlab.com/gitlab-org/labkit/errortracking" ) @@ -12,6 +16,18 @@ import ( func createSocket(addr string) (net.Listener, *os.File) { l, err := net.Listen("tcp", addr) if err != nil { + log.Printf("failed to net.Listen: %+v\n", addr) + cmd := exec.Command("netstat", "-plnut") + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Start() + if err != nil { + fmt.Printf("NETSTAT FAILED: %+v\n", err) + } + //}() + time.Sleep(2 * time.Second) + fatal(err, "could not create socket") } diff --git a/test/acceptance/config_test.go b/test/acceptance/config_test.go index aa5689695afda4fe055180e9e6dcbc82d05e9df8..2d3d1a649e7b9929ba6862a97ba415762ac8844f 100644 --- a/test/acceptance/config_test.go +++ b/test/acceptance/config_test.go @@ -4,6 +4,9 @@ import ( "fmt" "net" "net/http" + "os" + "os/exec" + "sync" "testing" "github.com/stretchr/testify/require" @@ -50,6 +53,20 @@ func TestMixedConfigSources(t *testing.T) { func TestMultipleListenersFromEnvironmentVariables(t *testing.T) { skipUnlessEnabled(t) + wg := sync.WaitGroup{} + wg.Add(2) + go func() { + defer wg.Done() + + fmt.Println("checking netstat...") + cmd := exec.Command("netstat", "-plnut") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Start() + if err != nil { + fmt.Printf("NETSTAT FAILED: %+v\n", err) + } + }() listenSpecs := []ListenSpec{{"http", "127.0.0.1", "37001"}, {"http", "127.0.0.1", "37002"}} envVarValue := fmt.Sprintf("LISTEN_HTTP=%s,%s", net.JoinHostPort("127.0.0.1", "37001"), net.JoinHostPort("127.0.0.1", "37002")) @@ -60,12 +77,17 @@ func TestMultipleListenersFromEnvironmentVariables(t *testing.T) { withEnv([]string{envVarValue}), ) - for _, listener := range listenSpecs { - require.NoError(t, listener.WaitUntilRequestSucceeds(nil)) - rsp, err := GetPageFromListener(t, listener, "group.gitlab-example.com", "project/") + go func() { + for _, listener := range listenSpecs { + require.NoError(t, listener.WaitUntilRequestSucceeds(nil)) + rsp, err := GetPageFromListener(t, listener, "group.gitlab-example.com", "project/") - require.NoError(t, err) - rsp.Body.Close() - require.Equal(t, http.StatusOK, rsp.StatusCode) - } + require.NoError(t, err) + rsp.Body.Close() + require.Equal(t, http.StatusOK, rsp.StatusCode) + } + wg.Done() + }() + + wg.Wait() } diff --git a/test/acceptance/helpers_test.go b/test/acceptance/helpers_test.go index 2b95cf4bda63c9b5e7b4c045ea0fb9941084cca3..5325aa873438b403de0286d874270665743fe104 100644 --- a/test/acceptance/helpers_test.go +++ b/test/acceptance/helpers_test.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "io/ioutil" + "log" "net" "net/http" "net/http/httptest" @@ -20,6 +21,7 @@ import ( "testing" "time" + "github.com/mozilla/mig/modules/netstat" "github.com/pires/go-proxyproto" "github.com/stretchr/testify/require" "golang.org/x/net/nettest" @@ -254,6 +256,21 @@ func RunPagesProcessWithStubGitLabServer(t *testing.T, opts ...processOption) *L logBuf, cleanup := runPagesProcess(t, processCfg.wait, processCfg.pagesBinary, processCfg.listeners, "", processCfg.envs, processCfg.extraArgs...) + if strings.Contains(logBuf.String(), "bind: address already in use") { + log.Println("FAIL:") + cleanup() + for _, listener := range processCfg.listeners { + _, elements, err := netstat.HasListeningPort(listener.Port) + log.Printf("listening port: %q - err: %+v", listener.Port, err) + for _, element := range elements { + log.Printf("HasListeningPort: %+v", element) + } + } + + //t.Log("retrying once...") + //logBuf, cleanup = runPagesProcess(t, processCfg.wait, processCfg.pagesBinary, processCfg.listeners, "", processCfg.envs, processCfg.extraArgs...) } + } + t.Cleanup(func() { source.Close() chdirCleanup() @@ -347,6 +364,14 @@ func runPagesProcess(t *testing.T, wait bool, pagesBinary string, listeners []Li if wait { for _, spec := range listeners { if err := spec.WaitUntilRequestSucceeds(waitCh); err != nil { + //go func() { + log.Printf("failed to open listener: %+v\n", spec) + cmd := exec.Command("netstat", "-plnut") + cmd.Stdout = out + cmd.Stderr = out + cmd.Start() + //}() + time.Sleep(time.Second) cleanup() t.Fatal(err) }