Separate test installation directories and environments

activeweb/bash_profile.sh

@@ -0,0 +1,2 @@
+export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-amd64
+export RESIN_HOME=${IROOT}/resin-4.0.36

config/benchmark_profile

@@ -1,23 +1,33 @@
-# Start Benchmark profile
+# This is auto-loaded from toolset/run-tests.py
+
+# Always reference ROOT for other vars
+if [ -z "${FWROOT}" ]; then
+  # Default value
+  FWROOT=~/FrameworkBenchmarks
+else
+  # Re-declare so it can be used in this script
+  FWROOT=$(echo $FWROOT)
+fi
+IROOT=${FWROOT}/installs
 
 export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-amd64
-export RESIN_HOME=~/FrameworkBenchmarks/installs/resin-4.0.36
-export GRAILS_HOME=~/FrameworkBenchmarks/installs/grails-2.4.2
-export VERTX_HOME=~/FrameworkBenchmarks/installs/vert.x-2.1.1
-export GOROOT=~/FrameworkBenchmarks/installs/go
-export GOPATH=~/FrameworkBenchmarks/go:~/FrameworkBenchmarks/webgo:~/FrameworkBenchmarks/revel
-export TOMCAT_HOME=~/FrameworkBenchmarks/installs/apache-tomcat-7.0.35
-export NODE_HOME=~/FrameworkBenchmarks/installs/node-v0.10.8-linux-x64
-export PLAY_HOME=~/FrameworkBenchmarks/installs/play-2.2.0
-export PLAY1_HOME=~/FrameworkBenchmarks/installs/play-1.2.5
-export MAVEN_HOME=~/FrameworkBenchmarks/installs/apache-maven-3.0.5
-export PERL_HOME=~/FrameworkBenchmarks/installs/perl-5.18
-export DART_HOME=~/FrameworkBenchmarks/installs/dart-sdk
-export PYTHON_HOME=~/FrameworkBenchmarks/installs/python-2.7.5
-export RACKET_HOME=~/FrameworkBenchmarks/installs/racket-5.3.6
-export NIMROD_HOME=~/FrameworkBenchmarks/installs/nimrod
+export RESIN_HOME=${IROOT}/resin-4.0.36
+export GRAILS_HOME=${IROOT}/grails-2.4.2
+export VERTX_HOME=${IROOT}/vert.x-2.1.1
+export GOROOT=${IROOT}/go
+export GOPATH=${FWROOT}/go:${FWROOT}/webgo:${FWROOT}/revel
+export TOMCAT_HOME=${IROOT}/apache-tomcat-7.0.35
+export NODE_HOME=${IROOT}/node-v0.10.8-linux-x64
+export PLAY_HOME=${IROOT}/play-2.2.0
+export PLAY1_HOME=${IROOT}/play-1.2.5
+export MAVEN_HOME=${IROOT}/apache-maven-3.0.5
+export PERL_HOME=${IROOT}/perl-5.18
+export DART_HOME=${IROOT}/dart-sdk
+export PYTHON_HOME=${IROOT}/python-2.7.5
+export RACKET_HOME=${IROOT}/racket-5.3.6
+export NIMROD_HOME=${IROOT}/nimrod
 export NGINX_HOME=/usr/local/nginx
-export ELIXIR_HOME=~/FrameworkBenchmarks/installs/elixir-0.13.3
+export ELIXIR_HOME=${IROOT}/elixir-0.13.3
 
 export PATH="$PYTHON_HOME/bin:$JAVA_HOME/bin:$GRAILS_HOME/bin:$PLAY_HOME:$PLAY1_HOME:$VERTX_HOME/bin:$GOROOT/bin:$NODE_HOME/bin:$HOME/FrameworkBenchmarks/installs/bin:$MAVEN_HOME/bin:$PERL_HOME/bin:$DART_HOME/bin:$RACKET_HOME/bin:$NIMROD_HOME/bin:$NGINX_HOME/sbin:$ELIXIR_HOME/bin:$PATH"
 

go/bash_profile.sh

@@ -0,0 +1,7 @@
+# Set the root of our go installation
+export GOROOT=${IROOT}/go
+
+# Where to find the go executable
+export PATH="$GOROOT/bin:$PATH"
+
+export GOPATH=${FWROOT}/go

go/setup.py

@@ -12,7 +12,7 @@ def start(args, logfile, errfile):
     subprocess.call("set GOPATH=C:\\FrameworkBenchmarks\\go&& go get ./...", shell=True, cwd="go", stderr=errfile, stdout=logfile)
     subprocess.Popen("setup.bat", shell=True, cwd="go", stderr=errfile, stdout=logfile) 
     return 0
-  os.environ["GOPATH"] = os.path.expanduser('~/FrameworkBenchmarks/go')
+  # os.environ["GOPATH"] = os.path.expanduser('~/FrameworkBenchmarks/go')
   subprocess.call("go get ./...", shell=True, cwd="go", stderr=errfile, stdout=logfile) 
   subprocess.Popen("go run src/hello/hello.go".rsplit(" "), cwd="go", stderr=errfile, stdout=logfile)
   return 0

toolset/benchmark/benchmarker.py

@@ -1,4 +1,6 @@
 from setup.linux.installer import Installer
+from setup.linux import setup_util
+
 from benchmark import framework_test
 
 import os
@@ -861,6 +863,9 @@ def __init__(self, args):
     if self.database_host == None: self.database_host = self.client_host
     if self.database_identity_file == None: self.database_identity_file = self.client_identity_file
 
+    # Remember root directory
+    self.fwroot = setup_util.get_fwroot()
+
     # setup results and latest_results directories 
     self.result_directory = os.path.join("results", self.name)
     self.latest_results_directory = self.latest_results_directory()
@@ -968,8 +973,8 @@ def __init__(self, args):
     if self.client_identity_file != None:
       self.client_ssh_string = self.client_ssh_string + " -i " + self.client_identity_file
 
-    if self.install_software:
-      install = Installer(self)
+    if self.install is not None:
+      install = Installer(self, self.install_strategy)
       install.install_software()
 
   ############################################################

toolset/benchmark/framework_test.py

@@ -1,4 +1,5 @@
 from benchmark.fortune_html_parser import FortuneHTMLParser
+from setup.linux import setup_util
 
 import importlib
 import os
@@ -10,6 +11,7 @@
 import traceback
 import json
 import textwrap
+import logging
 
 class FrameworkTest:
   ##########################################################################################
@@ -285,6 +287,15 @@ def validatePlaintext(self, jsonString, out, err):
   # Start the test using it's setup file
   ############################################################
   def start(self, out, err):
+    # Load profile for this installation
+    profile="%s/bash_profile.sh" % self.directory
+    if not os.path.exists(profile):
+      logging.warning("Framework %s does not have a bash_profile" % self.name)
+      profile="$FWROOT/config/benchmark_profile"
+
+    set_iroot="export IROOT=%s" % self.install_root
+    setup_util.replace_environ(config=profile, command=set_iroot)
+
     return self.setup_module.start(self.benchmarker, out, err)
   ############################################################
   # End start
@@ -900,6 +911,15 @@ def __init__(self, name, directory, benchmarker, runTests, args):
     self.directory = directory
     self.benchmarker = benchmarker
     self.runTests = runTests
+    self.fwroot = benchmarker.fwroot
+
+    # setup logging
+    logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+
+    self.install_root="%s/%s" % (self.fwroot, "installs")
+    if benchmarker.install_strategy is 'pertest':
+      self.install_root="%s/pertest/%s" % (self.install_root, name)
+
     self.__dict__.update(args)
 
     # ensure directory has __init__.py file so that we can use it as a Python package

toolset/run-tests.py

@@ -4,9 +4,11 @@
 import sys
 import os
 import multiprocessing
+import subprocess
 from pprint import pprint 
 from benchmark.benchmarker import Benchmarker
 from setup.linux.unbuffered import Unbuffered
+from setup.linux import setup_util
 
 ###################################################################################################
 # Main
@@ -19,12 +21,19 @@ def main(argv=None):
     # Enable unbuffered output so messages will appear in the proper order with subprocess output.
     sys.stdout=Unbuffered(sys.stdout)
 
-    # Ensure the current directory (which should be the benchmark home directory) is in the path so that the tests can be imported.
+    # Update python environment
+    # 1) Ensure the current directory (which should be the benchmark home directory) is in the path so that the tests can be imported.
     sys.path.append('.')
-
-    # Ensure toolset/setup/linux is in the path so that the tests can "import setup_util".
+    # 2) Ensure toolset/setup/linux is in the path so that the tests can "import setup_util".
     sys.path.append('toolset/setup/linux')
 
+    # Update environment for shell scripts
+    fwroot = setup_util.get_fwroot()
+    if not fwroot: 
+        fwroot = os.getcwd()
+    setup_util.replace_environ(config='config/benchmark_profile', root=fwroot)
+    print "FWROOT is %s"%setup_util.get_fwroot()
+
     conf_parser = argparse.ArgumentParser(
         description=__doc__,
         formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -78,12 +87,15 @@ def main(argv=None):
     parser.add_argument('--database-identity-file', default=dbIdenFile, dest='database_identity_file',
                         help='The key to use for SSH to the database instance.  If not provided, defaults to the value of --client-identity-file.')
     parser.add_argument('-p', dest='password_prompt', action='store_true', help='Prompt for password')
-    parser.add_argument('--install-software', action='store_true', help='runs the installation script before running the rest of the commands')
+
 
     # Install options
     parser.add_argument('--install', choices=['client', 'database', 'server', 'all'], default=None,
                         help='Runs installation script(s) before continuing on to execute the tests.')
     parser.add_argument('--install-error-action', choices=['abort', 'continue'], default='continue', help='action to take in case of error during installation')
+    parser.add_argument('--install-strategy', choices=['unified', 'pertest'], default='pertest', 
+        help='''Affects `--install server`: With unified, all server software is installed into a single directory. 
+        With pertest each test gets its own installs directory, but installation takes longer''')
 
     # Test options
     parser.add_argument('--test', nargs='+', help='names of tests to run')
@@ -128,6 +140,7 @@ def main(argv=None):
     if args.verbose:
         print 'Configuration options: '
         pprint(args)
+
     benchmarker = Benchmarker(vars(args))
 
     # Run the benchmarker in the specified mode

toolset/setup/linux/bash_functions.sh

@@ -34,8 +34,11 @@ fw_traperror () {
   IFS=':' read -a linestack <<< "$7" # Stack (line numbers)
   FW_dep_error=1
   FW_any_errors=1
+
+  wd=$(pwd)
+  relative_wd=\$FWROOT${wd#$FWROOT}
 
-  echo "ERROR: $(echo ${bashstack[1]#$FWROOT}): Command '$command' exited with status $err (dependency=$depend)"
+  echo "ERROR: $(echo ${bashstack[1]#$FWROOT}): Command '$command' exited with status $err (dependency=$depend) (cwd=$relative_wd)"
   #echo "  Function stack    : ${funcstack[@]}"
   #echo "  Bash source stack : ${bashstack[@]}"
   #echo "  Bash line stack   : ${linestack[@]}"
@@ -56,18 +59,22 @@ fw_depends() {
     echo Searching for $depend
     trap 'fw_traperror $depend $? $LINENO "$BASH_COMMAND" $(printf ":%s" ${FUNCNAME[@]}) $(printf ":%s" ${BASH_SOURCE[@]}) $(printf ":%s" ${BASH_LINENO[@]})'  ERR
     retcode=0
-    if [ -f ../toolset/setup/linux/systools/${depend}.sh ]; then
-      echo Installing system tool: $depend 
-      . ../toolset/setup/linux/systools/${depend}.sh
-    elif [ -f ../toolset/setup/linux/languages/${depend}.sh ]; then
-      echo Installing language: $depend 
-      . ../toolset/setup/linux/languages/${depend}.sh
-    elif [ -f ../toolset/setup/linux/webservers/${depend}.sh ]; then
-      echo Installing webserver: $depend 
-      . ../toolset/setup/linux/webservers/${depend}.sh
-    elif [ -f ../toolset/setup/linux/frameworks/${depend}.sh ]; then
-      echo Installing framework: $depend
-      . ../toolset/setup/linux/frameworks/${depend}.sh
+
+    wd=$(pwd)
+    relative_wd=\$FWROOT${wd#$FWROOT}
+
+    if [ -f $FWROOT/toolset/setup/linux/systools/${depend}.sh ]; then
+      echo Installing system tool: $depend in $relative_wd
+      . $FWROOT/toolset/setup/linux/systools/${depend}.sh
+    elif [ -f $FWROOT/toolset/setup/linux/languages/${depend}.sh ]; then
+      echo Installing language: $depend in $relative_wd
+      . $FWROOT/toolset/setup/linux/languages/${depend}.sh
+    elif [ -f $FWROOT/toolset/setup/linux/webservers/${depend}.sh ]; then
+      echo Installing webserver: $depend in $relative_wd
+      . $FWROOT/toolset/setup/linux/webservers/${depend}.sh
+    elif [ -f $FWROOT/toolset/setup/linux/frameworks/${depend}.sh ]; then
+      echo Installing framework: $depend in $relative_wd
+      . $FWROOT/toolset/setup/linux/frameworks/${depend}.sh
     else
       echo WARN: No installer found for $depend
       continue

toolset/setup/linux/installer.py

@@ -6,6 +6,7 @@
 import sys
 import glob
 import logging
+import setup_util
 
 class Installer:
 
@@ -30,10 +31,11 @@ def install_software(self):
   # __install_server_software
   ############################################################
   def __install_server_software(self):
-    print("\nINSTALL: Installing server software\n")
+    print("\nINSTALL: Installing server software (strategy=%s)\n"%self.strategy)
 
-    bash_functions_path='../toolset/setup/linux/bash_functions.sh'
-    prereq_path='../toolset/setup/linux/prerequisites.sh'
+    # Install global prerequisites
+    bash_functions_path='$FWROOT/toolset/setup/linux/bash_functions.sh'
+    prereq_path='$FWROOT/toolset/setup/linux/prerequisites.sh'
     self.__run_command(". %s && . %s" % (bash_functions_path, prereq_path))
 
     # Pull in benchmarker include and exclude list
@@ -42,25 +44,58 @@ def __install_server_software(self):
     if exclude == None:
         exclude = []
 
-    # Assume we are running from FrameworkBenchmarks
-    install_files = glob.glob('*/install.sh')
+    # Locate all known tests
+    install_files = glob.glob("%s/*/install.sh" % self.fwroot)
 
-    for install_file in install_files:
-        test = os.path.dirname(install_file)
-
-        if test in exclude:
-            logging.debug("%s has been excluded", test)
+    # Run install for selected tests
+    for test_install_file in install_files:
+        test_dir = os.path.dirname(test_install_file)
+        test_name = os.path.basename(test_dir)
+        test_rel_dir = setup_util.path_relative_to_root(test_dir)
+
+        if test_name in exclude:
+            logging.debug("%s has been excluded", test_name)
             continue
-        elif include is not None and test not in include:
-            logging.debug("%s not in include list", test)
+        elif include is not None and test_name not in include:
+            logging.debug("%s not in include list", test_name)
             continue
         else:
-            logging.debug("Running installer for %s", test)
-            bash_functions_path="../toolset/setup/linux/bash_functions.sh"
-            self.__run_command(". %s && . ../%s" % (bash_functions_path, install_file))
-
-    self.__run_command("sudo apt-get -y autoremove");
-
+            logging.info("Running installation for %s"%test_name)
+
+            # Find installation directory 
+            # e.g. FWROOT/installs or FWROOT/installs/pertest/<test-name>
+            test_install_dir="%s/%s" % (self.fwroot, self.install_dir)
+            if self.strategy is 'pertest':
+              test_install_dir="%s/pertest/%s" % (test_install_dir, test_name)
+            test_rel_install_dir=setup_util.path_relative_to_root(test_install_dir)
+            if not os.path.exists(test_install_dir):
+              os.makedirs(test_install_dir)
+
+            # Load profile for this installation
+            profile="%s/bash_profile.sh" % test_dir
+            if not os.path.exists(profile):
+              logging.warning("Framework %s does not have a bash_profile"%test_name)
+              profile="$FWROOT/config/benchmark_profile"
+            setup_util.replace_environ(config=profile)
+
+            # Find relative installation file
+            test_rel_install_file = "$FWROOT%s" % setup_util.path_relative_to_root(test_install_file)
+
+            # Then run test installer file
+            # Give all installers a number of variables
+            # FWROOT - Path of the FwBm root
+            # IROOT  - Path of this test's install directory
+            # TROOT  - Path to this test's directory 
+            self.__run_command('''
+              export TROOT=$FWROOT%s && 
+              export IROOT=$FWROOT%s && 
+              . %s && 
+              . %s''' % 
+              (test_rel_dir, test_rel_install_dir, 
+                bash_functions_path, test_rel_install_file),
+                cwd=test_install_dir)
+
+    self.__run_command("sudo apt-get -y autoremove");    
 
     print("\nINSTALL: Finished installing server software\n")
   ############################################################
@@ -247,10 +282,8 @@ def __path_exists(self, path, cwd=None):
   # __run_command
   ############################################################
   def __run_command(self, command, send_yes=False, cwd=None, retry=False):
-    try:
-      cwd = os.path.join(self.install_dir, cwd)
-    except AttributeError:
-      cwd = self.install_dir
+    if cwd is None: 
+        cwd = self.install_dir
 
     if retry:
       max_attempts = 5
@@ -261,12 +294,13 @@ def __run_command(self, command, send_yes=False, cwd=None, retry=False):
     if send_yes:
       command = "yes yes | " + command
 
-
-    print("\nINSTALL: %s (cwd=%s)" % (command, cwd))
+    rel_cwd = setup_util.path_relative_to_root(cwd)
+    print("INSTALL: %s (cwd=$FWROOT/%s)" % (command, rel_cwd))
 
     while attempt <= max_attempts:
       error_message = ""
       try:
+
         # Execute command.
         subprocess.check_call(command, shell=True, cwd=cwd, executable='/bin/bash')
         break  # Exit loop if successful.
@@ -323,9 +357,11 @@ def __download(self, uri, filename=""):
   ############################################################
   # __init__(benchmarker)
   ############################################################
-  def __init__(self, benchmarker):
+  def __init__(self, benchmarker, install_strategy):
     self.benchmarker = benchmarker
     self.install_dir = "installs"
+    self.fwroot = benchmarker.fwroot
+    self.strategy = install_strategy
 
     # setup logging
     logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)