initial work on oauth

build.sh

@@ -13,14 +13,15 @@ export GO15VENDOREXPERIMENT=1
 
 usage() {
   echo
-  echo "Usage: $0 [-t target ] [-a arch ] [ -p prefix ] [-h] [-d]"
+  echo "Usage: $0 [-t target ] [-a arch ] [ -p prefix ] [-h] [-d] [-r]"
   echo "Options:"
   echo "-h Show this screen"
   echo "-t (linux|darwin) Target OS Default:(linux)"
   echo "-a (amd64|386) Arch Default:(amd64)"
   echo "-d debug output"
   echo "-b build only, do not generate packages"
   echo "-p build prefix Default:(/usr/local)"
+  echo "-r build with race detector"
   echo "-s release subversion"
   echo
 }
@@ -130,7 +131,7 @@ function build() {
   prefix="$4"
   ldflags="-X main.AppVersion=${RELEASE_VERSION}"
   echo "Building via $(go version)"
-  gobuild="go build -ldflags \"$ldflags\" -o $builddir/orchestrator${prefix}/orchestrator/orchestrator go/cmd/orchestrator/main.go"
+  gobuild="go build ${opt_race} -ldflags \"$ldflags\" -o $builddir/orchestrator${prefix}/orchestrator/orchestrator go/cmd/orchestrator/main.go"
 
   case $os in
     'linux')
@@ -162,7 +163,8 @@ function main() {
 }
 
 build_only=0
-while getopts a:t:p:s:dbh flag; do
+opt_race=
+while getopts a:t:p:s:dbhr flag; do
   case $flag in
   a)
     arch="$OPTARG"
@@ -184,6 +186,9 @@ while getopts a:t:p:s:dbh flag; do
   p)
     prefix="$OPTARG"
     ;;
+  r)
+    opt_race="-race"
+    ;;
   s)
     RELEASE_VERSION="${RELEASE_VERSION}_${OPTARG}"
     ;;

doc/Orchestrator-Manual.md

@@ -929,9 +929,9 @@ Cheatsheet:
         instance-status
             Output short status on a given instance (name, replication status, noteable configuration). Example2:
 
-            orchestrator -c replication-status -i instance.to.investigate.com
+            orchestrator -c instance-status -i instance.to.investigate.com
 
-            orchestrator -c replication-status
+            orchestrator -c instance-status
                 -i not given, implicitly assumed local hostname
 
         snapshot-topologies

etc/init.d/orchestrator.bash

@@ -61,8 +61,16 @@ case "$1" in
     cd $DAEMON_PATH
     if [ -f $PIDFILE ]; then
       kill -TERM $PID
-      printf "%s\n" "Ok"
       rm -f $PIDFILE
+      # Wait for orchestrator to stop otherwise restart may fail.
+      # (The newly restarted process may be unable to bind to the
+      # currently bound socket.)
+      while ps -p $PID >/dev/null 2>&1; do
+        printf "."
+        sleep 1
+      done
+      printf "\n"
+      printf "Ok\n"
     else
       printf "%s\n" "pidfile not found"
       exit 1

go/app/http.go

@@ -14,7 +14,6 @@
    limitations under the License.
 */
 
-//
 package app
 
 import (
@@ -26,15 +25,20 @@ import (
 	"github.com/martini-contrib/auth"
 	"github.com/martini-contrib/gzip"
 	"github.com/martini-contrib/render"
+	"github.com/martini-contrib/sessions"
 	"github.com/outbrain/golib/log"
 	"github.com/outbrain/orchestrator/go/config"
 	"github.com/outbrain/orchestrator/go/http"
 	"github.com/outbrain/orchestrator/go/inst"
 	"github.com/outbrain/orchestrator/go/logic"
 	"github.com/outbrain/orchestrator/go/process"
 	"github.com/outbrain/orchestrator/go/ssl"
+
+	martinioauth2 "github.com/martini-contrib/oauth2"
+	golang_oauth2 "golang.org/x/oauth2"
 )
 
+var oauthStateString = process.ProcessToken.Hash
 var sslPEMPassword []byte
 var agentSSLPEMPassword []byte
 
@@ -93,6 +97,19 @@ func standardHttp(discovery bool) {
 				return auth.SecureCompare(username, config.Config.HTTPAuthUser) && auth.SecureCompare(password, config.Config.HTTPAuthPassword)
 			}))
 		}
+	case "oauth":
+		{
+			oauthConf := &golang_oauth2.Config{
+				ClientID:     config.Config.OAuthClientId,
+				ClientSecret: config.Config.OAuthClientSecret,
+				Scopes:       config.Config.OAuthScopes,
+				RedirectURL:  "http://localhost:3000/oauth2callback",
+			}
+			m.Use(sessions.Sessions("oauth_session", sessions.NewCookieStore([]byte("oauth_secret"))))
+			m.Use(martinioauth2.Github(oauthConf))
+			m.Use(martinioauth2.LoginRequired)
+			m.Map(auth.User(""))
+		}
 	default:
 		{
 			// We inject a dummy User object because we have function signatures with User argument in api.go

go/cmd/orchestrator/main.go

@@ -694,9 +694,9 @@ Cheatsheet:
         instance-status
             Output short status on a given instance (name, replication status, noteable configuration). Example2:
 
-            orchestrator -c replication-status -i instance.to.investigate.com
+            orchestrator -c instance-status -i instance.to.investigate.com
 
-            orchestrator -c replication-status
+            orchestrator -c instance-status
                 -i not given, implicitly assumed local hostname
 
         snapshot-topologies
@@ -945,7 +945,7 @@ Cheatsheet:
 
 						orchestrator -c active-nodes
 
-				acceess-token
+				access-token
 						When running HTTP with "AuthenticationMethod" : "token", receive a new access token.
 						This token must be utilized within "AccessTokenUseExpirySeconds" and can then be used
 						until "AccessTokenExpiryMinutes" have passed.

go/config/config.go

@@ -50,6 +50,7 @@ type Configuration struct {
 	MySQLTopologyMaxPoolConnections              int    // Max concurrent connections on any topology instance
 	DatabaselessMode__experimental               bool   // !!!EXPERIMENTAL!!! Orchestrator will execute without speaking to a backend database; super-standalone mode
 	MySQLOrchestratorHost                        string
+	MySQLOrchestratorMaxPoolConnections          int    // The maximum size of the connection pool to the Orchestrator backend.
 	MySQLOrchestratorPort                        uint
 	MySQLOrchestratorDatabase                    string
 	MySQLOrchestratorUser                        string
@@ -92,7 +93,10 @@ type Configuration struct {
 	AuditPurgeDays                               uint   // Days after which audit entries are purged from the database
 	RemoveTextFromHostnameDisplay                string // Text to strip off the hostname on cluster/clusters pages
 	ReadOnly                                     bool
-	AuthenticationMethod                         string            // Type of autherntication to use, if any. "" for none, "basic" for BasicAuth, "multi" for advanced BasicAuth, "proxy" for forwarded credentials via reverse proxy, "token" for token based access
+	AuthenticationMethod                         string // Type of autherntication to use, if any. "" for none, "basic" for BasicAuth, "multi" for advanced BasicAuth, "proxy" for forwarded credentials via reverse proxy, "token" for token based access
+	OAuthClientId                                string
+	OAuthClientSecret                            string
+	OAuthScopes                                  []string
 	HTTPAuthUser                                 string            // Username for HTTP Basic authentication (blank disables authentication)
 	HTTPAuthPassword                             string            // Password for HTTP Basic authentication
 	AuthUserHeader                               string            // HTTP header indicating auth user, when AuthenticationMethod is "proxy"
@@ -199,6 +203,7 @@ func newConfiguration() *Configuration {
 		StatusEndpoint:                               "/api/status",
 		StatusSimpleHealth:                           true,
 		StatusOUVerify:                               false,
+		MySQLOrchestratorMaxPoolConnections:          128,              // limit concurrent conns to backend DB
 		MySQLOrchestratorPort:                        3306,
 		MySQLTopologyMaxPoolConnections:              3,
 		MySQLTopologyUseMutualTLS:                    false,

go/db/db.go

@@ -898,6 +898,15 @@ func OpenOrchestrator() (*sql.DB, error) {
 	db, fromCache, err := sqlutils.GetDB(mysql_uri)
 	if err == nil && !fromCache {
 		initOrchestratorDB(db)
+
+		// do not show the password but do show what we connect to.
+		safe_mysql_uri := fmt.Sprintf("%s:?@tcp(%s:%d)/%s?timeout=%ds", config.Config.MySQLOrchestratorUser,
+			config.Config.MySQLOrchestratorHost, config.Config.MySQLOrchestratorPort, config.Config.MySQLOrchestratorDatabase, config.Config.MySQLConnectTimeoutSeconds)
+		log.Debugf("Connected to orchestrator backend: %v", safe_mysql_uri)
+		if config.Config.MySQLOrchestratorMaxPoolConnections > 0 {
+			log.Debugf("Orchestrator pool SetMaxOpenConns: %d", config.Config.MySQLOrchestratorMaxPoolConnections)
+			db.SetMaxOpenConns(config.Config.MySQLOrchestratorMaxPoolConnections)
+		}
 		db.SetMaxIdleConns(10)
 	}
 	return db, err

go/http/api.go

@@ -1663,7 +1663,7 @@ func (this *HttpAPI) AgentCustomCommand(params martini.Params, r render.Render,
 		return
 	}
 
-	output, err := agent.CustomCommand(params["host"], params["cmd"])
+	output, err := agent.CustomCommand(params["host"], params["command"])
 
 	if err != nil {
 		r.JSON(200, &APIResponse{Code: ERROR, Message: fmt.Sprintf("%+v", err)})

go/http/httpbase.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"github.com/martini-contrib/auth"
+
 	"github.com/outbrain/orchestrator/go/config"
 	"github.com/outbrain/orchestrator/go/process"
 )
@@ -77,6 +78,10 @@ func isAuthorizedForAction(req *http.Request, user auth.User) bool {
 			result, _ := process.TokenIsValid(publicToken, secretToken)
 			return result
 		}
+	case "oauth":
+		{
+			return false
+		}
 	default:
 		{
 			// Default: no authentication method

go/inst/cluster_alias.go

@@ -16,8 +16,6 @@
 
 package inst
 
-import ()
-
 // SetClusterAlias will write (and override) a single cluster name mapping
 func SetClusterAlias(clusterName string, alias string) error {
 	return WriteClusterAlias(clusterName, alias)

go/process/health_dao.go

@@ -183,6 +183,27 @@ func ReadAvailableNodes(onlyHttpNodes bool) ([]string, error) {
 	return res, err
 }
 
+func TokenBelongsToHealthyHttpService(token string) (result bool, err error) {
+	extraInfo := string(OrchestratorExecutionHttpMode)
+
+	query := `
+		select
+			token
+		from
+			node_health
+		where
+			and token = ?
+			and extra_info = ?
+		`
+
+	err = db.QueryOrchestrator(query, sqlutils.Args(token, extraInfo), func(m sqlutils.RowMap) error {
+		// Row exists? We're happy
+		result = true
+		return nil
+	})
+	return result, log.Errore(err)
+}
+
 // Just check to make sure we can connect to the database
 func SimpleHealthTest() (*HealthStatus, error) {
 	health := HealthStatus{Healthy: false, Hostname: ThisHostname, Token: ProcessToken.Hash}