github proxy tests

.travis.yml

@@ -6,13 +6,13 @@ go_import_path: github.com/src-d/gitcollector
 matrix:
   fast_finish: true
 
+services:
+  - docker
+
 script:
   - GO111MODULE=on
   - make ci-script
 
-services:
-  - docker
-
 jobs:
   include:
     - os: linux

_testdata/server.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDI4WJC4LZ5VHsx
+EEVG6IbsyqlHwmcH5X9ywRAc0iLW/5rzkF5FG5g6caODlSp63e40t6Bc45XUdw3n
++o8Nv/wdEkiozhROca4mPGpV71uDQJaNrV7sp+yflQa6kYr2Wibhk8BaBJEeNfdE
+bAAF4hZx6KNLRIMpfGeimNRqVfA+QNfhwP7D9vG8rqxJW9aj6YKxhBonvGumY6kZ
+p4sqxEHdb/yTTHR329PsBbT7PJ9Zid9TUy1DttiyZ2UiYPFaYGE5YYiMOqm8s3aL
+Mr8WqGCtwqdg1YjeV9Ht9HpDLgTBgivwXQ5b3oobUpbrGkmx52HzjB1HkbxUpK2N
+KF4ux/TpAgMBAAECggEAeZwQe4JElaEyTxqVhf7hJ9rsrjefXiVuiY8dbQU9+ECX
+ktoUZ+vztXX0UoVsSSOvwJRl7bYNj6lfbs+96MoQG51lBWR04KRrlFRA37NfCE+C
+lVSewdBLLLTwdOQFZ2xeW3OyIAzhuZNcFTz4qpcblWwL51YbvXLF3qZ8+b77CQPv
+UoRNtXeGS/cB3rFj8L4HfSiRs6TumXZS8RCQPmAhn83qmVfxkF4wdHtxnUDNNTRs
+jBWgtkCLWoCJcc/KHqjOJF81u2UdWW6hl2ZB+CmlLZbvxULFYD2+folIR/whwgnV
+sP+8p/aQGo60udLZUz2f/+9WW0POuZfXwzqVlMP+MQKBgQDxXEranOlI9eOT6dWn
+V2O4quV13iWgiTxyxv1/bWIQKH9N0VD8M7cTomcs/ryLawItvWDC41SOz9QSrKis
+PPTXyJKBPaZOp4j0EIDIzlla0vCBdAIbxI7G0nFLTcElqgziQWDLepOjGH4u6bsd
+ZlMYsOi5j3fRGw3q0fqhFoMZ7QKBgQDVEIocKnpkAOBJOfTaSY+nR0DWWBBu13YE
+/4CKt6u2jmPPiM3JhqnILK/uydBWLB1IKaedJsMadZ0zfGcrfUwR1lx1ZppWvj01
+avIdutkihXlSXJdgM8ZhV6M4jT3i9J4ggVaSVk/3lIHiZePxkjUzoM8pMRbxwJHr
+RxqOi3Q3bQKBgQDLYiMtjpXCRnmMNbo1Nh2xCkkrhpOccKiAa5BHpavBw+5qa5zb
+uNRn9i0NLGdC8Fw4oOF9uESTxyuExPupkAAeKUTTJVkT5HcyewmarM99Ck1iOJhi
+P8K0ZbbUZuQAUghjTAY90L/B6qzB6XEkeApDvyeM3glwKmUS3/hbb/XvjQKBgEM5
+srnsVS2UcS/pj24IY8D95GE87AJtDRnKm/4osY8qIifrHDMHwC4em2wWVRu7U4nf
+Srl07347udvy1oKMSCIgpo+MTgosgBeIpNfoinYv8vWfrUt3SpKQGjnPpsUjNXlA
+FdaaUSTmwIx0ZZVk8lBV3sc6a+jfP6sihiiPq52JAoGBAOHYF7nbkJUbiyBVdS6n
+ER9lUbim6ViDqUjsjKIHHphbIs9Rh+GrlA4WPVcHVZmhXIDSmon67fxAclTjGI16
+enPrTpk+oMDPFkruoUme72JPlOvRFhpLeiBfzMffPuKEqObXYnyxGcP7ya3IAY3R
+5k3GLdCvSDvDsHza2OxVErD1
+-----END PRIVATE KEY-----

_testdata/server.pem

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIICzjCCAbagAwIBAgIUShAAlHP7BagLuY+xCv3a5rnhvjQwDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE5MDkyNDEwMzMwMFoXDTI5MDky
+MTEwMzMwMFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF
+AAOCAQ8AMIIBCgKCAQEAyOFiQuC2eVR7MRBFRuiG7MqpR8JnB+V/csEQHNIi1v+a
+85BeRRuYOnGjg5Uqet3uNLegXOOV1HcN5/qPDb/8HRJIqM4UTnGuJjxqVe9bg0CW
+ja1e7Kfsn5UGupGK9lom4ZPAWgSRHjX3RGwABeIWceijS0SDKXxnopjUalXwPkDX
+4cD+w/bxvK6sSVvWo+mCsYQaJ7xrpmOpGaeLKsRB3W/8k0x0d9vT7AW0+zyfWYnf
+U1MtQ7bYsmdlImDxWmBhOWGIjDqpvLN2izK/FqhgrcKnYNWI3lfR7fR6Qy4EwYIr
+8F0OW96KG1KW6xpJsedh84wdR5G8VKStjSheLsf06QIDAQABoxgwFjAUBgNVHREE
+DTALgglsb2NhbGhvc3QwDQYJKoZIhvcNAQELBQADggEBALaAuARTKmRCeBgCae98
+2jgepX1dxtEL0oHWBUNW28pZ0Oko+yFBVZzIDlmKTNT0qVgNgCfwZx3werNG7T9h
+x3annlQ0MGMGWqGi2kHif5VP/Tw7vHDo8BtIeN16BDhZ/5HtSOTYa9ZrD3lHVI1O
+ATE/gw7gC0OhPEvDYs+ob9iQ6exaHIOJwjo0XrRiUOyOr7dt31y2Zj53jxOHdNIJ
+oOqU7JzixJcNDb9wSk4rUURLQLyZrjQiFrhYiLPC0PLEkRSJxgnqMIf6AhGaub3S
+boINjFa12llUn04Gw3TB7xNGmoevdhZ0WwpFajepepTc+VqXs24PehwGvWWt9EJ+
+zg8=
+-----END CERTIFICATE-----

discovery/github_test.go

@@ -3,15 +3,20 @@ package discovery
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/src-d/gitcollector/testutils"
+
 	"github.com/google/go-github/v28/github"
 	"github.com/stretchr/testify/require"
 )
 
+const couldNotFindNewRepos = "couldn't find new repositories"
+
 func TestGitHub(t *testing.T) {
 	var req = require.New(t)
 
@@ -20,7 +25,7 @@ func TestGitHub(t *testing.T) {
 		timeToStop = 5 * time.Second
 	)
 
-	token, _ := testToken()
+	token, _ := getToken()
 	queue := make(chan *github.Repository, 50)
 	advertiseRepos := func(
 		_ context.Context,
@@ -88,7 +93,7 @@ func TestGitHubSkipForks(t *testing.T) {
 	var req = require.New(t)
 	const org = "src-d"
 
-	token, skip := testToken()
+	token, skip := getToken()
 	if skip != nil {
 		t.Skip(skip.Error())
 	}
@@ -138,7 +143,152 @@ func TestGitHubSkipForks(t *testing.T) {
 	}
 }
 
-func testToken() (string, error) {
+// TODO request rate error ?
+
+// TestProxyMockUps
+// setup https proxy that returns error responses with different status codes
+// <expected> check that errors returned correspond to expected ones
+func TestProxyMockUps(t *testing.T) {
+	for _, tst := range []struct {
+		name        string
+		code        int
+		errContains string
+	}{
+		{"301", http.StatusMovedPermanently, "Moved Permanently"},
+		{"400", http.StatusBadRequest, "Bad Request"},
+		{"403", http.StatusForbidden, "Forbidden"},
+		{"404", http.StatusNotFound, "Not Found"},
+		{"500", http.StatusInternalServerError, "Internal Server Error"},
+		{"501", http.StatusNotImplemented, "Not Implemented"},
+		{"502", http.StatusBadGateway, "Bad Gateway"},
+		{"503", http.StatusServiceUnavailable, "Service Unavailable"},
+	} {
+		tst := tst
+		t.Run(tst.name, func(t *testing.T) {
+			testProxyMockUp(t, tst.code, tst.errContains)
+		})
+	}
+}
+
+func testProxyMockUp(t *testing.T, code int, errContains string) {
+	const org = "bblfsh"
+
+	healthyTransport := http.DefaultTransport
+	defer func() { http.DefaultTransport = healthyTransport }()
+
+	proxy, err := testutils.NewProxy(
+		healthyTransport,
+		&testutils.Options{
+			Code:    code,
+			KeyPath: "../_testdata/server.key",
+			PemPath: "../_testdata/server.pem",
+		})
+	require.NoError(t, err)
+
+	require.NoError(t, proxy.Start())
+	defer func() { proxy.Stop() }()
+
+	require.NoError(t, testutils.SetTransportProxy())
+
+	token, _ := getToken()
+	queue := make(chan *github.Repository, 50)
+	advertiseRepos := func(
+		_ context.Context,
+		repos []*github.Repository,
+	) error {
+		time.Sleep(time.Minute)
+		for _, repo := range repos {
+			queue <- repo
+		}
+
+		return nil
+	}
+
+	discovery := NewGitHub(
+		advertiseRepos,
+		NewGHOrgReposIter(org, &GHReposIterOpts{
+			TimeNewRepos:   1 * time.Second,
+			ResultsPerPage: 100,
+			AuthToken:      token,
+		}),
+		&GitHubOpts{
+			MaxJobBuffer:     50,
+			AdvertiseTimeout: time.Second,
+		},
+	)
+
+	err = discovery.Start()
+	require.Error(t, err)
+	require.Contains(t, err.Error(), errContains)
+}
+
+type advertiseCase struct {
+	name           string
+	advErr         error
+	delay, timeout time.Duration
+	errContains    string
+}
+
+// TestAdvertiseErrors checks basic advertise cases:
+// 1) advertise function completed successfully
+// 2) advertise function has returned an error
+// 3) advertise function is being evaluated longer than AdvertiseTimeout
+func TestAdvertiseErrors(t *testing.T) {
+	for _, tst := range []advertiseCase{
+		{"NoAdvertiseError", nil, 0, 0, couldNotFindNewRepos},
+		{"AdvertiseError", fmt.Errorf("advertise err"), 0, 0, "advertise err"},
+		{"AdvertiseTimeout", nil, 2 * time.Second, time.Second, "context deadline exceeded"},
+	} {
+		tst := tst
+		t.Run(tst.name, func(t *testing.T) {
+			testAdvertise(t, tst)
+		})
+	}
+}
+
+func testAdvertise(t *testing.T, ac advertiseCase) {
+	const org = "bblfsh"
+
+	token, _ := getToken()
+	queue := make(chan *github.Repository, 50)
+	advertiseRepos := func(
+		ctx context.Context,
+		repos []*github.Repository,
+	) error {
+		time.Sleep(ac.delay)
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+
+		for _, repo := range repos {
+			queue <- repo
+		}
+
+		return ac.advErr
+	}
+
+	discovery := NewGitHub(
+		advertiseRepos,
+		NewGHOrgReposIter(org, &GHReposIterOpts{
+			TimeNewRepos:   1 * time.Second,
+			ResultsPerPage: 100,
+			AuthToken:      token,
+		}),
+		&GitHubOpts{
+			MaxJobBuffer:     50,
+			AdvertiseTimeout: ac.timeout,
+		},
+	)
+
+	err := discovery.Start()
+	require.Error(t, err)
+	require.Contains(t, err.Error(), ac.errContains)
+}
+
+func getToken() (string, error) {
 	token := os.Getenv("GITHUB_TOKEN")
 	ci := os.Getenv("TRAVIS")
 	var err error