3.1.0rc1 release

destijl · destijl · commit 46b0226ef4fa · 2016-04-06T11:22:18.000-07:00
- pypi setup.py files
- minor bugfixes
diff --git a/grr/client/client_build.py b/grr/client/client_build.py
@@ -81,6 +81,9 @@
 parser_build = subparsers.add_parser(
     "build", help="Build a client from source.")
 
+parser_build.add_argument("--output", default=None,
+                          help="The path to write the output template.")
+
 parser_repack = subparsers.add_parser(
     "repack", help="Repack a zip file into an installer (Only useful when "
     "signing).")
@@ -457,8 +460,13 @@ def main(_):
     signer = GetSigner(context)
 
   if args.subparser_name == "build":
+    template_path = None
+    if flags.FLAGS.output:
+      template_path = os.path.join(flags.FLAGS.output, config_lib.CONFIG.Get(
+          "PyInstaller.template_filename", context=context))
+
     builder_obj = GetBuilder(context)
-    builder_obj.MakeExecutableTemplate()
+    builder_obj.MakeExecutableTemplate(output_file=template_path)
   elif args.subparser_name == "repack":
     Repack(context, signer=signer)
   elif args.subparser_name == "deploy":
diff --git a/grr/config/grr-response-client/setup.py b/grr/config/grr-response-client/setup.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+"""This is the setup.py file for the GRR client.
+
+This is just a meta-package which pulls in the minimal requirements to create a
+client.
+"""
+from setuptools import setup
+
+setup(
+    name="grr-response-client",
+    version="3.1.0",
+    description="The GRR Rapid Response client.",
+    license="Apache License, Version 2.0",
+    url="https://github.com/google/grr",
+    entry_points={
+        "console_scripts": [
+            "grr_client = grr.lib.distro_entry:Client",
+            "grr_client_build = grr.lib.distro_entry:ClientBuild",
+        ]
+    },
+    install_requires=[
+        "grr-response-core",
+    ],
+)
diff --git a/grr/config/grr-response-server/setup.py b/grr/config/grr-response-server/setup.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+"""This is the setup.py file for the GRR client.
+
+This is just a meta-package which pulls in the minimal requirements to create a
+full grr server.
+"""
+from setuptools import setup
+
+setup(
+    name="grr-response-server",
+    version="3.1.0",
+    description="The GRR Rapid Response Server.",
+    license="Apache License, Version 2.0",
+    url="https://github.com/google/grr",
+    entry_points={
+        "console_scripts": [
+            "grr_console = grr.lib.distro_entry:Console",
+            "grr_config_updater = grr.lib.distro_entry:ConfigUpdater",
+            "grr_front_end = grr.lib.distro_entry:GrrFrontEnd",
+            "grr_server = grr.lib.distro_entry:GrrServer",
+            "grr_end_to_end_tests = grr.lib.distro_entry:EndToEndTests",
+            "grr_export = grr.lib.distro_entry:Export",
+            "grr_worker = grr.lib.distro_entry:Worker",
+            "grr_admin_ui = grr.lib.distro_entry:AdminUI",
+            "grr_fuse = grr.lib.distro_entry:GRRFuse",
+        ]
+    },
+    install_requires=[
+        "grr-response-core[Server,test]",
+        "grr-response-client",
+    ],
+
+    extras_require={
+        # This is an optional component. Install to get MySQL data
+        # store support:
+        # pip install grr-response[MySQLDataStore]
+        "MySQLDataStore": [
+            "MySQL-python==1.2.5"
+        ],
+    }
+)
diff --git a/grr/defaults.py b/grr/defaults.py
diff --git a/grr/endtoend_tests/grep.py b/grr/endtoend_tests/grep.py
@@ -7,23 +7,6 @@
 from grr.lib.rdfvalues import client as rdf_client
 
 
-class TestSearchFiles(base.AutomatedTest):
-  """Test SearchFileContent."""
-  platforms = ["Linux"]
-  flow = "SearchFileContent"
-  test_output_path = "analysis/SearchFiles/testing"
-  args = {"output": test_output_path,
-          "paths": ["/bin/ls*"],
-          "also_download": True}
-
-  def CheckFlow(self):
-    results = aff4.FACTORY.Open(self.client_id.Add(self.test_output_path),
-                                token=self.token)
-    self.assertGreater(len(results), 1)
-    for result in results:
-      self.assertTrue(result.pathspec.path.startswith("/bin/ls"))
-
-
 class TestSearchFilesGrep(base.AutomatedTest):
   """Test SearchFileContent with grep."""
   platforms = ["Linux"]
@@ -37,7 +20,7 @@ class TestSearchFilesGrep(base.AutomatedTest):
   def CheckFlow(self):
     results = aff4.FACTORY.Open(self.client_id.Add(self.test_output_path),
                                 token=self.token)
-    self.assertGreater(len(results), 1)
+    self.assertGreaterEqual(len(results), 1)
     for result in results:
       self.assertTrue("ELF" in result.data)
       self.assertTrue("ls" in result.pathspec.path)
diff --git a/grr/gui/api_plugins/user.py b/grr/gui/api_plugins/user.py
@@ -349,7 +349,7 @@ class ApiGetPendingUserNotificationsResult(rdf_structs.RDFProtoStruct):
 
 class ApiGetPendingUserNotificationsHandler(
     api_call_handler_base.ApiCallHandler):
-  """Returns the of pending notifications for the current user."""
+  """Returns pending notifications for the current user."""
 
   category = CATEGORY
   args_type = ApiGetPendingUserNotificationsArgs
diff --git a/grr/gui/plugins/fileview_test.py b/grr/gui/plugins/fileview_test.py
@@ -266,7 +266,7 @@ def testRefreshDirectoryStartsFlow(self):
     # Go to the flow management screen.
     self.Click("css=a:contains('Manage launched flows')")
 
-    self.Click("css=grr-flows-list tr:visible:nth(1)")
+    self.Click("css=grr-flows-list tr:contains('RecursiveListDirectory')")
     self.WaitUntilContains("RecursiveListDirectory", self.GetText,
                            "css=#main_bottomPane")
     self.WaitUntilContains(
diff --git a/grr/lib/distro_entry.py b/grr/lib/distro_entry.py
@@ -30,8 +30,8 @@ def GetDistroDefaults():
       return DISTRO_DEFAULTS["debian"]
     if distribution in ["red hat enterprise linux server", "centos linux"]:
       return DISTRO_DEFAULTS["redhat"]
-  else:
-    return {"flag_defaults": {}, "config_opts": {}}
+
+  return {"flag_defaults": {}, "config_opts": {}}
 
 
 def SetConfigOptions():
diff --git a/grr/lib/hunts/results.py b/grr/lib/hunts/results.py
@@ -83,6 +83,10 @@ def DeleteNotifications(cls, record_ids, token=None):
     """Delete hunt notifications."""
     with aff4.FACTORY.OpenWithLock(RESULT_NOTIFICATION_QUEUE,
                                    aff4_type="HuntResultQueue",
+                                   lease_time=200,
+                                   blocking=True,
+                                   blocking_sleep_interval=15,
+                                   blocking_lock_timeout=1200,
                                    token=token) as queue:
       queue.DeleteRecords(record_ids)
 
diff --git a/setup.py b/setup.py
@@ -60,27 +60,12 @@ def run(self):
 
 
 setup_args = dict(
-    name="grr",
-    version="0.3.1",
+    name="grr-response-core",
+    version="3.1.0pre1",
     description="GRR Rapid Response",
     license="Apache License, Version 2.0",
     url="https://github.com/google/grr",
     packages=find_packages(),
-    entry_points={
-        "console_scripts": [
-            "grr_console = grr.lib.distro_entry:Console",
-            "grr_config_updater = grr.lib.distro_entry:ConfigUpdater",
-            "grr_front_end = grr.lib.distro_entry:GrrFrontEnd",
-            "grr_server = grr.lib.distro_entry:GrrServer",
-            "grr_end_to_end_tests = grr.lib.distro_entry:EndToEndTests",
-            "grr_export = grr.lib.distro_entry:Export",
-            "grr_client = grr.lib.distro_entry:Client",
-            "grr_client_build = grr.lib.distro_entry:ClientBuild",
-            "grr_worker = grr.lib.distro_entry:Worker",
-            "grr_admin_ui = grr.lib.distro_entry:AdminUI",
-            "grr_fuse = grr.lib.distro_entry:GRRFuse",
-        ]
-    },
     zip_safe=False,
     include_package_data=True,
     ext_modules=[
@@ -94,17 +79,11 @@ def run(self):
         "install": Install,
     },
     install_requires=[
-        "Django==1.8.3",
         "GRR-M2Crypto==0.22.6.post2",
         "PyYAML==3.11",
-        "Werkzeug==0.11.3",
         "binplist==0.1.4",
-        "google-api-python-client==1.4.2",
         "ipaddr==2.1.11",
         "ipython==4.1.1",
-        "matplotlib==1.5.1",
-        "mock==1.3.0",
-        "mox==0.5.3",
         "pexpect==4.0.1",
         "portpicker==1.1.1",
         "psutil==4.0.0",
@@ -115,21 +94,32 @@ def run(self):
         "python-dateutil==2.4.2",
         "pytsk3==20160226",
         "pytz==2015.7",
-        "selenium==2.50.1",
         "urllib3==1.14",
-        "wsgiref==0.1.2",
-
         "protobuf==2.6.1",
-        "rekall-core>=1.5.0.post4",
     ],
     extras_require={
         # This is an optional component. Install to get MySQL data
         # store support:
-        # pip install grr[MySQLDataStore]
+        # pip install grr-response[MySQLDataStore]
         "MySQLDataStore": [
             "MySQL-python==1.2.5"
         ],
 
+        "test": [
+            "mock==1.3.0",
+            "mox==0.5.3",
+        ],
+
+        "Server": [
+            "wsgiref==0.1.2",
+            "Werkzeug==0.11.3",
+            "Django==1.8.3",
+            "matplotlib==1.5.1",
+            "rekall-core>=1.5.0.post4",
+            "selenium==2.50.1",
+            "google-api-python-client==1.4.2",
+        ],
+
         # The following requirements are needed in Windows.
         ':sys_platform=="win32"': [
             "WMI==1.4.9",
diff --git a/travis/before_install.sh b/travis/before_install.sh
@@ -20,7 +20,7 @@ sudo apt-get install -y \
   python-pip \
   python-protobuf \
   python-support \
-  pytsk3 \
+  python-pytsk3 \
   sleuthkit \
   swig
 
diff --git a/vagrant/build_templates.sh b/vagrant/build_templates.sh
@@ -11,12 +11,6 @@ if [ $# -ne 1 ]; then
   usage
 fi
 
-export PROTO_SRC_INSTALLED="/grrbuild/"
-if [ "$1" == "OS_X_10.8.5" ]; then
-  # Protobuf library installed by homebrew on OS X
-  export PROTO_SRC_INSTALLED="/usr/local/Cellar/protobuf/2.6.1/include/"
-fi
-
 export SSH_AUTH_SOCK=""
 if [ "$1" == "windows_7_64" ]; then
   # Build the templates on windows by running --provision.
@@ -26,10 +20,8 @@ if [ "$1" == "windows_7_64" ]; then
   fi
 else
   vagrant up "$1"
-  # We need to compile the protos inside the build environment because the host
-  # may not have the correct proto compiler.
-  vagrant ssh -c "cd /grr/ && PROTO_SRC_ROOT=$PROTO_SRC_INSTALLED python makefile.py && cd / && source ~/PYTHON_ENV/bin/activate && PYTHONPATH=. python grr/client/client_build.py --config=grr/config/grr-server.yaml build" "$1"
-  vagrant ssh -c "cd / && source ~/PYTHON_ENV/bin/activate && PYTHONPATH=. python grr/client/client_build.py --config=grr/config/grr-server.yaml build_component /grr/client/components/chipsec_support/setup.py /grr/" "$1"
+  vagrant ssh -c "source ~/PYTHON_ENV/bin/activate && grr_client_build --config=/grr/grr/config/grr-server.yaml build --output /grr/grr/executables/" "$1"
+  vagrant ssh -c "source ~/PYTHON_ENV/bin/activate && grr_client_build --config=/grr/grr/config/grr-server.yaml build_components --output /grr/grr/executables/components" "$1"
 
   if [ $? -eq 0 ]; then
     vagrant halt "$1"
diff --git a/vagrant/install_linux.sh b/vagrant/install_linux.sh
@@ -100,9 +100,9 @@ function install_python_from_source() {
     yum install -y zlib-devel bzip2-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel
   fi
 
-  ${WGET} https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tgz
-  tar zxvf Python-2.7.9.tgz
-  cd Python-2.7.9
+  ${WGET} https://www.python.org/ftp/python/2.7.11/Python-2.7.11.tgz
+  tar zxvf Python-2.7.11.tgz
+  cd Python-2.7.11
 
   # --enabled-shared for better performance, discussed in some detail here:
   # https://code.google.com/p/modwsgi/wiki/InstallationIssues
@@ -120,34 +120,6 @@ function install_python_from_source() {
   python2.7 -c "import ssl"
 }
 
-# Get a newer protobuf library than what lucid has. Just installing the python
-# package isn't enough because we need the compiler and associated libraries.
-# This version needs to stay in sync with the requirements.txt python version.
-function install_protobuf_libs() {
-  ${WGET} https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.gz
-  tar zxvf protobuf-2.6.1.tar.gz
-  cd protobuf-2.6.1
-
-  if [ -e /grrbuild ]; then
-    rm -r /grrbuild
-  fi;
-  mkdir /grrbuild
-  ./configure --includedir=/grrbuild
-  make -j4
-
-  # 32bit CentOS fails 'make check' due to it having an older version of gcc
-  # with this issue:
-  # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=13358
-  if [ $DISTRO == "CentOS" ] && [ $(uname -m) != 'x86_64' ]; then
-    make install
-  else
-    make check -j4
-    make install
-  fi
-  ldconfig
-  cd -
-}
-
 # Install our python dependencies into a virtualenv that uses the new python
 # version
 function install_python_deps() {
@@ -164,12 +136,13 @@ function install_python_deps() {
   /usr/local/bin/virtualenv -p /usr/local/bin/python2.7 PYTHON_ENV
   source PYTHON_ENV/bin/activate
 
-  pip2.7 install .
+  # We need grr-reponse-client to get the grr_client_build entrypoint
+  pip2.7 install /grr/ /grr/grr/config/grr-response-client/
 
   # pyinstaller fails to include protobuf because there is no __init__.py:
   # https://github.com/google/protobuf/issues/713
   touch PYTHON_ENV/lib/python2.7/site-packages/google/__init__.py
-
+  chown -R vagrant PYTHON_ENV
 }
 
 # Lucid debhelper is too old to build debs that handle both upstart, init.d,
@@ -222,7 +195,6 @@ system_update
 install_openssl
 install_wget
 install_python_from_source
-install_protobuf_libs
 install_python_deps
 install_packagetools
 echo "Build environment provisioning complete."
