Pass contents of OPTIONS to memcached client (#14)

* Remove BMemcached._cache property

This property overrides the underlying
django.core.cache.backends.memcached.BaseMemcachedCache._cache property,
but it's not clear that it's doing much different, other than cherry-
picking certain options (username, password) and then ignoring the rest,
when creating the uncerlying Client object. This means that the OPTIONS
dict in the Django settings is being ignored, which in turn prevents
us from setting properties like the pickle_protocol.

* Add test for custom config

* Update README

* Convert tests to tox and update support

* Add tox.ini

* Fix tox.ini (remove py34-django21)

* Add InvalidCacheOptions error on initialisation

* Update InvalidCacheOptions error message

* Simplify invalid options checking using sets

* Fix Python 2 error

* Update readme

Author: hugorodgerbrown

.travis-runs-tests.sh

@@ -1,5 +1,4 @@
 #!/bin/bash
 sudo service memcached start
-nosetests --version
-DJANGO_SETTINGS_MODULE=tests.settings nosetests --with-coverage --cover-package=django_bmemcached
+tox
 exit $?

.travis.yml

@@ -6,10 +6,7 @@ python:
   - "3.5"
   - "3.6"
   - "3.7"
-env:
-  - DJANGO_VERSION=">=1.10,<1.11"
-  - DJANGO_VERSION=">=1.9,<1.10"
-  - DJANGO_VERSION=">=1.8,<1.9"
+  - "3.8"
 
-install: pip install -r requirements_test.txt Django$DJANGO_VERSION python-binary-memcached
+install: pip install -r requirements_test.txt tox-travis
 script: ./.travis-runs-tests.sh

README.md

@@ -1,16 +1,33 @@
 [![Build Status](https://secure.travis-ci.org/jaysonsantos/django-bmemcached.png?branch=master)](http://travis-ci.org/jaysonsantos/django-bmemcached)
+
 # Django-BMemcached
-A django cache backend to use [bmemcached] (https://github.com/jaysonsantos/python-binary-memcached) module which supports memcached binary protocol with authentication.
+
+A django cache backend to use [bmemcached](https://github.com/jaysonsantos/python-binary-memcached)
+module which supports memcached binary protocol with authentication.
 
 ## Installing
+
 Use pip:
 
 ```bash
 pip install django-bmemcached
 ```
 
 ## Using
-In your settings.py add bmemcached as backend.
+
+In your settings.py add bmemcached as backend:
+
+```python
+CACHES = {
+    'default': {
+        'BACKEND': 'django_bmemcached.memcached.BMemcached',
+        'LOCATION': 'your_server:port',
+    }
+}
+```
+
+If you are using Django 1.11 or above, you can also add `OPTIONS` which will be
+passed through to the client. The options available for `BMemcached` are as follows:
 
 ```python
 CACHES = {
@@ -19,13 +36,34 @@ CACHES = {
         'LOCATION': 'your_server:port',
         'OPTIONS': {
             'username': 'user',
-            'password': 'password'
+            'password': 'password',
+            'compression': None,
+            'socket_timeout': bmemcached.client.constants.SOCKET_TIMEOUT,
+            'pickler': pickle.Pickler,
+            'unpickler': pickle.Unpickler,
+            'pickle_protocol': 0
         }
     }
 }
 ```
 
+NB If you have options in your configuration that are not supported (see the sample
+above, or `django_bmemcached.memcached.VALID_CACHE_OPTIONS`)
+the cache will raise an `InvalidCacheOptions` error the first time it is accessed:
+
+```python
+>>> from django_bmemcached import BMemcached
+>>> client = BMemcached(None, {'OPTIONS': {'bad_option': True}})
+>>> client.get("foo")
+Traceback (most recent call last):
+...
+django_bmemcached.memcached.InvalidCacheOptions: Error initialising BMemcached - invalid options detected: {'bad_option'}
+Please check your CACHES config contains only valid OPTIONS: {'username', 'unpickler', 'compression', 'pickle_protocol', 'password', 'pickler', 'socket_timeout'}
+>>>
+```
+
 ### Using in Heroku
+
 Just add bmemcached as backend. It will work automagically if you have added memcached as Heroku addon.
 
 ```python
@@ -36,9 +74,6 @@ CACHES = {
 }
 ```
 
-
-
-
 ## Testing
 
 ```bash

django_bmemcached/memcached.py

@@ -1,5 +1,36 @@
 import os
+
+import bmemcached
 from django.core.cache.backends import memcached
+from django.core.exceptions import ImproperlyConfigured
+
+# keys that are acceptable in the Django
+# settings.CACHES[...]["OPTIONS"] dictionary.
+VALID_CACHE_OPTIONS = {
+    "username",
+    "password",
+    "compression",
+    "socket_timeout",
+    "pickler",
+    "unpickler",
+    "pickle_protocol",
+}
+
+
+class InvalidCacheOptions(ImproperlyConfigured):
+    """Custom error raised when Client initialisation fails."""
+
+    def __init__(self, options):
+        # in Python 3 you can compare a list and set, but in Python 2
+        # you cannot, so need to cast the options.keys() explicitly to a set.
+        # TODO: remove set() once Python 2 support is dropped.
+        invalid_options = set(options.keys()) - VALID_CACHE_OPTIONS
+        msg = (
+            "Error initialising BMemcached - invalid options detected: %s\n"
+            "Please check your CACHES config contains only valid OPTIONS: %s"
+            % (invalid_options, VALID_CACHE_OPTIONS)
+        )
+        super(InvalidCacheOptions, self).__init__(msg)
 
 
 class BMemcached(memcached.BaseMemcachedCache):
@@ -8,7 +39,6 @@ class BMemcached(memcached.BaseMemcachedCache):
     A.K.A BMemcached.
     """
     def __init__(self, server, params):
-        import bmemcached
         params.setdefault('OPTIONS', {})
 
         username = params['OPTIONS'].get('username', params.get('USERNAME', os.environ.get('MEMCACHE_USERNAME')))
@@ -38,12 +68,12 @@ def _cache(self):
             return client
 
         if self._options:
-            client = self._lib.Client(
-                self._servers, self._options.get('username', None),
-                self._options.get('password', None)
-            )
+            try:
+                client = self._lib.Client(self._servers, **self._options)
+            except TypeError:
+                raise InvalidCacheOptions(self._options)
         else:
-            client = self._lib.Client(self._servers,)
+            client = self._lib.Client(self._servers)
 
         self._client = client
 

requirements_test.txt

@@ -1,2 +1,4 @@
 nose
 coverage
+python-binary-memcached
+tox

tests/__init__.py

@@ -1,12 +1,20 @@
+import bz2
 import os
+import pickle
 import unittest
 
+import django.conf
+import django.core.cache
 import django_bmemcached
+from django.test import override_settings
+from django_bmemcached import BMemcached
+from django_bmemcached.memcached import InvalidCacheOptions
+from pickle import Pickler
 
 
 class TestWithExplicitAuth(unittest.TestCase):
     def setUp(self):
-        self.client = django_bmemcached.BMemcached(
+        self.client = BMemcached(
             ('127.0.0.1:11211',),
             {'OPTIONS': {'username': 'user', 'password': 'password'}}
         )
@@ -31,7 +39,7 @@ def testPropertyCacheRetunsAwaysSameServer(self):
 
 class TestWithTopLevelAuthParams(TestWithExplicitAuth):
     def setUp(self):
-        self.client = django_bmemcached.BMemcached(
+        self.client = BMemcached(
             ('127.0.0.1:11211',),
             {'USERNAME': 'user', 'PASSWORD': 'password'}
         )
@@ -42,7 +50,7 @@ def setUp(self):
         os.environ['MEMCACHE_SERVERS'] = '127.0.0.1'
         os.environ['MEMCACHE_USERNAME'] = 'user'
         os.environ['MEMCACHE_PASSWORD'] = 'password'
-        self.client = django_bmemcached.BMemcached(None, {})
+        self.client = BMemcached(None, {})
 
     def tearDown(self):
         del os.environ['MEMCACHE_SERVERS']
@@ -52,4 +60,49 @@ def tearDown(self):
 
 class TestWithoutAuth(TestWithExplicitAuth):
     def setUp(self):
-        self.client = django_bmemcached.BMemcached(('127.0.0.1:11211',), {})
+        self.client = BMemcached(('127.0.0.1:11211',), {})
+
+
+class TestPickler(Pickler):
+    pass
+
+class TestUnpickler(pickle.Unpickler):
+    pass
+
+
+class TestOptions(unittest.TestCase):
+
+    # these options are all non-default - so that we can confirm that
+    # they are passed through to the client.
+    TEST_OPTIONS = {
+        "compression": bz2,
+        "pickle_protocol": 2,
+        "socket_timeout": 1.0,
+        "pickler": TestPickler,
+        "unpickler": TestUnpickler
+    }
+    TEST_CACHE_CONFIG = {
+        "default": {
+            "BACKEND": "django_bmemcached.memcached.BMemcached",
+            "BINARY": True,
+            "OPTIONS": TEST_OPTIONS
+        }
+    }
+
+    @unittest.skipIf(django.get_version() < '1.11', "OPTIONS did not exist pre-1.11")
+    @override_settings(CACHES=TEST_CACHE_CONFIG)
+    def testWithOptions(self):
+        client = django.core.cache.caches["default"]
+        options = self.TEST_OPTIONS
+        self.assertIsInstance(client, BMemcached)
+        self.assertEqual(client._cache.pickle_protocol, options["pickle_protocol"])
+        self.assertEqual(client._cache.compression, options["compression"])
+        self.assertEqual(client._cache.socket_timeout, options["socket_timeout"])
+        self.assertEqual(client._cache.pickler, options["pickler"])
+        self.assertEqual(client._cache.unpickler, options["unpickler"])
+
+    def testInvalidOptions(self):
+        """Check that InvalidCacheOptions is raised."""
+        client = BMemcached(None, {'OPTIONS': {'bad_option': True}})
+        with self.assertRaises(InvalidCacheOptions):
+            client._cache

tox.ini

@@ -0,0 +1,22 @@
+[tox]
+skip_install = true
+envlist =
+    py{27}-django{19,110,111}
+    py{34,35}-django{19,110,111,20}
+    py{36,37}-django{111,20,21,22,30}
+    py{38}-django{30}
+
+[testenv]
+deps = -rrequirements_test.txt
+    django18: Django>=1.8,<1.9
+    django19: Django>=1.9,<1.10
+    django110: Django>=1.10,<1.11
+    django111: Django>=1.11,<2.0
+    django20: Django>=2.0,<2.1
+    django21: Django>=2.1,<2.2
+    django22: Django>=2.2,<2.3
+    django30: Django>=3.0,<3.1
+setenv =
+    DJANGO_SETTINGS_MODULE = tests.settings
+
+commands = nosetests --with-coverage --cover-package=django_bmemcached