Merge pull request #24 from dmitris/bindata

Standalone executable, binpacking of templates

Author: dmitris

README.md

@@ -65,7 +65,9 @@ In all tests, excepts where specially mentioned, the attack input is assumed to
 
 * xss/dom/yuinode_hash_unencoded?#in=xyz - passing the unencoded hash value to YUI's setHTML function.  PoE (Firefox / Chrome): /xss/dom/yuinode_hash?#in=xyz">/xss/dom/yuinode_hash?#in=xyz</A> - DOM XSS using YUI (decoded location.hash) 
 
-### Adding New Tests
+### Modifying Tests
+
+When modifying, adding or deleting any tests, you need to rerun ```go generate```.
 
 For most of the tests, you need to add a template that contains the "moustache" with {{.In}}.
 
@@ -80,4 +82,4 @@ to the map in the FilterMap function in custom.go.  For example:
 
 To add a new fully custom testcase, add a template (if needed),
 add a mapping of the entrypoint to the handling function to CustomMap in custom.go and implement the custom function with the signature: func(http.ResponseWriter, *http.Request).  For example, for a test case with XSS injection through the Morse code, you could add:  
-```mp["/xss/reflect/morse"] = XssUnsafeMorse```  
+```mp["/xss/reflect/morse"] = XssUnsafeMorse```.  See entrypoints and implementing functions in CustomMap in custom.go.  

webseclab/main.go renamed to cmd/webseclab/main.go

@@ -9,11 +9,9 @@ package main
 import (
 	"flag"
 	"fmt"
-	"log"
 	"net"
 	"net/http"
 	"os"
-	"path"
 	"runtime"
 
 	"github.com/yahoo/webseclab"
@@ -26,46 +24,33 @@ func (fn indexHandler) ServerHTTP(w http.ResponseWriter, r *http.Request) {
 	return
 }
 
+const notice = `Attention: Webseclab is purposedly INSECURE software intended for testing and education.  Use it at your own risk and be careful! Hit Ctrl-C now if you don't understand the risks invovled.
+
+Webseclab executable includes the Go runtime which is covered by the Go license that can be found in http://golang.org/LICENSE.  See https://github.com/yahoo/webseclab for the Webseclab source code, license, and additional information.`
+
 func main() {
 	// use up to 2 CPUs, not more
 	cpus := runtime.NumCPU()
 	if cpus >= 2 {
 		cpus = 2
 	}
 	runtime.GOMAXPROCS(cpus)
-	defaultBase, err := webseclab.TemplateBaseDefault()
-	if err != nil {
-		panic(err)
-	}
-	base := flag.String("base", defaultBase, "base path for webseclab templates")
+
 	port := flag.String("http", ":8080", "port to run the webserver on")
 	noindex := flag.Bool("noindex", false, "do not serve the top index page (/ and /index.html)")
 	cleanup := flag.Bool("cleanup", false, "cleanup only (terminate existing instance and exit)")
+	version := flag.Bool("version", false, "display version number and exit")
 	flag.Parse()
-	fmt.Printf("Using %s as the template base, change it with -base command-line option (run 'webseclab -help' for usage help)\n", *base)
-	var abspath string
-	if path.IsAbs(*base) {
-		abspath = *base
-	} else {
-		pwd, err := os.Getwd()
-		if err != nil {
-			panic(err)
-		}
-		abspath = path.Join(pwd, *base)
-	}
-	if err := webseclab.CheckPath(abspath); err != nil {
-		log.Printf("Unable to open base %s: %s\n", abspath, err)
-		os.Exit(1)
+	if *version {
+		fmt.Println(webseclab.WEBSECLAB_VERSION)
+		os.Exit(0)
 	}
+	fmt.Println(notice, "\n")
 	// if there is another webseclab server running, tell it to quit and make way for us
 	webseclab.KillPredecessor(*port)
 	if *cleanup {
 		return
 	}
-	err = webseclab.ParseTemplates(abspath)
-	if err != nil {
-		panic(err)
-	}
 	ln, err := net.Listen("tcp", *port)
 	if err != nil {
 		panic(err)
@@ -74,12 +59,12 @@ func main() {
 	fmt.Printf("Webseclab starts to listen on %s\n", *port)
 
 	if !*noindex {
-		http.HandleFunc("/index.html", webseclab.MakeIndexFunc(*base, "/index.html"))
+		http.HandleFunc("/index.html", webseclab.MakeIndexFunc("/index.html"))
 	}
 	http.HandleFunc("favicon.ico", func(w http.ResponseWriter, r *http.Request) {
 		return
 	})
-	http.Handle("/", webseclab.MakeMainHandler(*base, *noindex))
+	http.Handle("/", webseclab.MakeMainHandler(*noindex))
 	http.HandleFunc("/exit", webseclab.MakeExitFunc(ln))
 	http.HandleFunc("/ruok", webseclab.Ruok)
 

genstatic.go

@@ -0,0 +1,109 @@
+// Copyright 2015, Yahoo Inc. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// based on https://github.com/golang/tools/blob/master/godoc/static/makestatic.go
+// which carries the following copyright notice:
+
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The LICENSE file for the golang/tools repository can be found in
+// https://github.com/golang/tools/blob/master/LICENSE
+
+// +build ignore
+
+// Command genstatic reads a set of files (Webseclab templates)
+// and writes a Go source file to "static.go"
+// that declares a map of string constants containing contents of the input files.
+// It is intended to be invoked via "go generate" (directive in "templates.go").
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"unicode/utf8"
+)
+
+func main() {
+	if err := makestatic(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func makestatic() error {
+	f, err := os.Create("static.go")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	w := bufio.NewWriter(f)
+	fmt.Fprintf(w, "%v\n\npackage webseclab\n\n", warning)
+	fmt.Fprintf(w, "var Templates = map[string]string{\n")
+	base := "templates/"
+	files, err := getTemplateFiles(base)
+	for _, fn := range files {
+		b, err := ioutil.ReadFile(base + fn)
+		if err != nil {
+			return err
+		}
+		fmt.Fprintf(w, "\t%q: ", fn)
+		if utf8.Valid(b) {
+			fmt.Fprintf(w, "`%s`", sanitize(b))
+		} else {
+			fmt.Fprintf(w, "%q", b)
+		}
+		fmt.Fprintln(w, ",")
+	}
+
+	fmt.Fprintln(w, "}")
+	if err := w.Flush(); err != nil {
+		return err
+	}
+	return f.Close()
+}
+
+// sanitize prepares a valid UTF-8 string as a raw string constant.
+func sanitize(b []byte) []byte {
+	// Replace ` with `+"`"+`
+	b = bytes.Replace(b, []byte("`"), []byte("`+\"`\"+`"), -1)
+
+	// Replace BOM with `+"\xEF\xBB\xBF"+`
+	// (A BOM is valid UTF-8 but not permitted in Go source files.
+	// I wouldn't bother handling this, but for some insane reason
+	// jquery.js has a BOM somewhere in the middle.)
+	return bytes.Replace(b, []byte("\xEF\xBB\xBF"), []byte("`+\"\\xEF\\xBB\\xBF\"+`"), -1)
+}
+
+// copied from templates.go to prevent making it exported
+// getTemplateFiles collects all files under the base dir
+func getTemplateFiles(base string) (files []string, err error) {
+	files = make([]string, 0, 50)
+	visit := func(path string, f os.FileInfo, err error) error {
+		if f == nil {
+			return errors.New("File " + path + " does not exist or not readable")
+		}
+		// don't add common shared files
+		if strings.HasSuffix(path, "common/footer") ||
+			strings.HasSuffix(path, "common/header") {
+			return nil
+		}
+		if f.IsDir() == false {
+			files = append(files, strings.TrimPrefix(path, base))
+		}
+		return err
+	}
+	err = filepath.Walk(base, visit)
+	return files, err
+}
+
+const warning = "// DO NOT EDIT ** This file was generated by \"go generate\" (gen directive in templates.go) ** DO NOT EDIT //"

handler.go

@@ -5,10 +5,15 @@
 package webseclab
 
 import (
+	"errors"
+	"fmt"
 	"html/template"
+	"io"
 	"log"
+	"net"
 	"net/http"
 	"strconv"
+	"strings"
 )
 
 // LabResp wraps the data related to processing results
@@ -48,3 +53,115 @@ func (fn LabHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 }
+
+// MakeMainHandler performs routing between 'standard' (template based with standard parameters) cases
+// and those requiring custom processing.  For standard processing, it unescapes input and prepars an instance of Indata
+// which is then passed to the template execution.  For URLs found in the map of custom processing,
+// the corresponding function is called.
+func MakeMainHandler(noindex bool) LabHandler {
+	return func(w http.ResponseWriter, r *http.Request) *LabResp {
+		// routing - handle a special case, path = "/" (=> index.html)
+		// dump, _ := httputil.DumpRequest(r, true)
+		// fmt.Printf("DEBUG request dump: %q\n", dump)
+		indexFn := MakeIndexFunc("index.html")
+		if r.URL.Path == "/" || r.URL.Path == "/index.html" {
+			if noindex {
+				return &LabResp{
+					Err:  errors.New("index page is prevented with -noindex option"),
+					Code: http.StatusForbidden,
+				}
+
+			}
+			indexFn(w, r)
+			return &LabResp{Err: nil, Code: http.StatusOK}
+		}
+		// be paranoid about the non-IP domains to protect cookies against XSS
+		safe := IsSafeHost(r.Host)
+		if safe == false {
+			ipurl, err := GetIpUrl(r.Host, r.URL)
+			if err != nil {
+				return &LabResp{Err: errors.New("ERROR in GetIpUrl(" + r.URL.String() + "): " + err.Error()),
+					Code: http.StatusInternalServerError}
+			}
+			return &LabResp{Err: errors.New(r.Host + " - not an IP quad pair"),
+				Code:     http.StatusFound,
+				Redirect: ipurl.String()}
+		}
+
+		// check if custom handling is needed
+		funcmap := CustomMap()
+		handler, ok := funcmap[r.URL.Path]
+		if ok {
+			return handler(w, r)
+		}
+		filtermap := FilterMap()
+		filters, ok := filtermap[r.URL.Path]
+		if ok {
+			return HandleFilterBased(w, r, filters)
+		}
+		return DoLabTestStandard(w, r)
+	}
+}
+
+// ack to a ping: "ruok" => "imok\n" (for /ruok monitoring entrypoint)
+func Ruok(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-type", "text/plain; charset=utf-8")
+	fmt.Fprintf(w, "imok\n")
+}
+
+// MakeExitFunc creates an /exit handler
+func MakeExitFunc(ln net.Listener) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		parts := strings.Split(r.RemoteAddr, ":")
+		if len(parts) > 0 {
+			if parts[0] != "127.0.0.1" && parts[0] != "localhost" {
+				w.WriteHeader(http.StatusForbidden)
+				io.WriteString(w, "Access denied. (/exit called from non-local IP: "+parts[0]+")\n")
+				return
+			}
+		}
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		io.WriteString(w, `bye`)
+		log.Println("Received exit request, exiting")
+		ln.Close()
+	}
+}
+
+// MakeIndexFunc creates a function to display the index file (ToC)
+func MakeIndexFunc(page string) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// log.Printf("Request for %s from %s\n", r.URL.String(), r.RemoteAddr)
+		var data = &InData{}
+		// for UI, find out an Ip quad-pair link if we are not on a such already
+		// this is done to protect cookies of our domain against XSS (see ip.go)
+		if IsIp(r.Host) == false {
+			iplink, err := GetIpUrl(r.Host, r.URL)
+			if err != nil {
+				w.WriteHeader(http.StatusInternalServerError)
+				w.Write([]byte(`Internal Server Error`))
+				log.Printf("ERROR - unable to find out my own IP address: r.Host = %s, r.URL = %s\n", r.Host, r.URL.String())
+				return
+			}
+			data.In = iplink.Host
+		}
+		err := DoTemplate(w, "/index.html", data)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(`Internal Server Error`))
+			return
+		}
+		return
+	}
+}
+
+// MakeStaticFunc
+func MakeStaticFunc() LabHandler {
+	return func(w http.ResponseWriter, r *http.Request) *LabResp {
+		err := DoTemplate(w, r.URL.Path, &InData{})
+		if err != nil {
+			log.Printf("Error in MakeStaticFunc, DoTemplate returned Err: %s\n", err)
+			return &LabResp{Err: err, Code: http.StatusInternalServerError}
+		}
+		return &LabResp{Err: nil, Code: http.StatusOK}
+	}
+}

process.go

@@ -86,13 +86,13 @@ func DoTemplate(w http.ResponseWriter, path string, input *InData) (err error) {
 
 func doHtmlTemplate(w http.ResponseWriter, fpath string, input *InData) (err error) {
 	// html/template - context-sensitive escaping
+	fmt.Printf("DMDEBUG - fpath=%s\n", fpath)
 	tmpl, ok := LookupHtmlTemplate(fpath)
 	if ok == false {
 		return errors.New("Error in DoTemplate - html template " + fpath + " not found.")
 	}
 	w.Header().Set("Content-type", "text/html; charset=utf-8")
-	parts := strings.Split(fpath, "/")
-	err = tmpl.ExecuteTemplate(w, parts[len(parts)-1], *input)
+	err = tmpl.ExecuteTemplate(w, fpath, *input)
 	if err != nil {
 		log.Printf("Error in DoTemplate (html) - tmpl.Execute: %s\n", err)
 		return err
@@ -107,8 +107,7 @@ func doTextTemplate(w http.ResponseWriter, fpath string, input *InData) (err err
 		return err
 	}
 	w.Header().Set("Content-type", "text/html; charset=utf-8")
-	parts := strings.Split(fpath, "/")
-	err = tmpl.ExecuteTemplate(w, parts[len(parts)-1], *input)
+	err = tmpl.ExecuteTemplate(w, fpath, *input)
 	if err != nil {
 		log.Printf("Error in DoTemplate (text) - tmpl.Execute's err: %s\n", err)
 		return err

process_test.go

@@ -2,8 +2,6 @@ package webseclab
 
 import (
 	"net/http"
-	"os"
-	"path"
 	"testing"
 )
 
@@ -14,11 +12,7 @@ type dohandler func(w http.ResponseWriter, fpath string, input *InData) (err err
 
 // read the templates
 func init() {
-	pwd, err := os.Getwd()
-	if err != nil {
-		panic(err)
-	}
-	err = ParseTemplates(path.Join(pwd, "templates"))
+	err := parseTemplates()
 	if err != nil {
 		panic(err)
 	}