feat: Add Environment & Services support (#213)

Tracing scenario for environment-aware teams:

* Dataset no longer propagates across services for new environment key.
* Add field `service.name` in addition to already added `service_name`
* `ServiceName` "required"†. If unset, will warn and set `service_name` AND `service.name` to `unknown_service:<process>` or `unknown_service:language`.
* Service name is used to populate dataset name; if `unknown_service.*` then truncate to`unknown_service` for dataset name.
* Trim whitespace from dataset name
* `WriteKey` required. If unset, will warn.
* `Dataset` **ignored**. If unset, no warning (because no longer used). If set, will warn to clarify that it will be ignored in favor of service name.

Tracing scenario for classic (non-environment-aware) teams:

* Add field `service.name` in addition to already added `service_name`
* `ServiceName` "required"†. If unset, will warn.
* `WriteKey` required. If unset, will warn.
* `Dataset` required. If unset, will warn and default to `beeline-<language>`.

† While not technically required, service name is highly encouraged to avoid `unknown_service` and to instead properly describe the data being sent to Honeycomb (and the data being viewed in Honeycomb UI).

Author: JamieDanielson

beeline/__init__.py

@@ -10,6 +10,7 @@
 from beeline.version import VERSION
 from beeline import internal
 import beeline.propagation.default
+import beeline.propagation
 import sys
 # pyflakes
 assert internal
@@ -74,15 +75,68 @@ def __init__(self,
         if debug:
             self._init_logger()
 
+        def IsClassicKey(writekey):
+            return len(writekey) == 32
+
         # allow setting some values from the environment
         if not writekey:
             writekey = os.environ.get('HONEYCOMB_WRITEKEY', '')
+        # also check API_KEY just in case
+        if not writekey:
+            writekey = os.environ.get('HONEYCOMB_API_KEY', '')
+        if not writekey:
+            logging.error(
+                'writekey not set! set the writekey if you want to send data to honeycomb'
+            )
+
+        if IsClassicKey(writekey):
+            if not dataset:
+                dataset = os.environ.get('HONEYCOMB_DATASET', '')
+            if not dataset:
+                logging.error(
+                    'dataset not set! set a value for dataset if you want to send data to honeycomb'
+                )
+        else:
+            if dataset:
+                logging.error(
+                    'dataset will be ignored in favor of service name'
+                )
 
-        if not dataset:
-            dataset = os.environ.get('HONEYCOMB_DATASET', '')
-
+        default_service_name = "unknown_service"
+        process_name = os.environ.get('PROCESS_EXECUTABLE_NAME', '')
         if not service_name:
-            service_name = os.environ.get('HONEYCOMB_SERVICE', dataset)
+            service_name = os.environ.get('HONEYCOMB_SERVICE')
+        # also check SERVICE_NAME just in case
+        if not service_name:
+            service_name = os.environ.get('SERVICE_NAME', '')
+        # no service name, set default
+        if not service_name:
+            service_name = default_service_name
+            if process_name:
+                service_name += ":" + process_name
+            else:
+                service_name += ":python"
+            logging.error(
+                'service name not set! service name will be ' + service_name
+            )
+            if not IsClassicKey(writekey):
+                logging.error(
+                    'data will be sent to unknown_service'
+                )
+
+        if not IsClassicKey(writekey):
+            # set dataset based on service name
+            dataset = service_name
+            if dataset.strip() != dataset:
+                # whitespace detected. trim whitespace, warn on diff
+                logging.error(
+                    'service name has unexpected spaces'
+                )
+                dataset = service_name.strip()
+
+            # set default, truncate to unknown_service if needed
+            if dataset == "" or dataset.startswith("unknown_service"):
+                dataset = "unknown_service"
 
         self.client = Client(
             writekey=writekey, dataset=dataset, sample_rate=sample_rate,
@@ -94,16 +148,16 @@ def __init__(self,
             debug=debug,
         )
 
+        if IsClassicKey(writekey):
+            beeline.propagation.propagate_dataset = True
+        else:
+            beeline.propagation.propagate_dataset = False
+
         self.log('initialized honeycomb client: writekey=%s dataset=%s service_name=%s',
                  writekey, dataset, service_name)
-        if not writekey:
-            self.log(
-                'writekey not set! set the writekey if you want to send data to honeycomb')
-        if not dataset:
-            self.log(
-                'dataset not set! set a value for dataset if you want to send data to honeycomb')
 
         self.client.add_field('service_name', service_name)
+        self.client.add_field('service.name', service_name)
         self.client.add_field('meta.beeline_version', VERSION)
         self.client.add_field('meta.local_hostname', socket.gethostname())
 

beeline/propagation/__init__.py

@@ -3,6 +3,10 @@
 import beeline
 
 
+def propagate_dataset():
+    return bool
+
+
 class PropagationContext(object):
     '''
     PropagationContext represents information that can either be read from, or

beeline/propagation/honeycomb.py

@@ -51,7 +51,7 @@ def marshal_propagation_context(propagation_context):
                   "parent_id={}".format(propagation_context.parent_id),
                   "context={}".format(trace_fields)]
 
-    if propagation_context.dataset:
+    if beeline.propagation.propagate_dataset and propagation_context.dataset:
         components.insert(0, "dataset={}".format(quote(propagation_context.dataset)))
 
     trace_header = "{};{}".format(version, ",".join(components))
@@ -94,7 +94,7 @@ def unmarshal_propagation_context_with_dataset(trace_header):
             parent_id = v
         elif k == 'context':
             context = json.loads(base64.b64decode(v.encode()).decode())
-        elif k == 'dataset':
+        elif k == 'dataset' and beeline.propagation.propagate_dataset:
             dataset = unquote(v)
 
     # context should be a dict

beeline/propagation/test_honeycomb.py

@@ -1,4 +1,5 @@
 import unittest
+import beeline.propagation
 from beeline.propagation import DictRequest, PropagationContext
 import beeline.propagation.honeycomb as hc
 
@@ -21,6 +22,7 @@ def test_roundtrip(self):
 
     def test_roundtrip_with_dataset(self):
         '''Verify that we can successfully roundtrip (marshal and unmarshal)'''
+        beeline.propagation.propagate_dataset = True
         dataset = "blorp blorp"
         trace_id = "bloop"
         parent_id = "scoop"
@@ -34,6 +36,22 @@ def test_roundtrip_with_dataset(self):
         self.assertEqual(parent_id, new_parent_id)
         self.assertEqual(trace_fields, new_trace_fields)
 
+    def test_roundtrip_with_dataset_propagation_disabled(self):
+        '''Verify that we can successfully roundtrip (marshal and unmarshal) without dataset propagation'''
+        beeline.propagation.propagate_dataset = False
+        dataset = "blorp blorp"
+        trace_id = "bloop"
+        parent_id = "scoop"
+        trace_fields = {"key": "value"}
+        pc = PropagationContext(trace_id, parent_id, trace_fields, dataset)
+        header = hc.marshal_propagation_context(pc)
+        new_trace_id, new_parent_id, new_trace_fields, new_dataset = hc.unmarshal_propagation_context_with_dataset(
+            header)
+        self.assertIsNone(new_dataset)
+        self.assertEqual(trace_id, new_trace_id)
+        self.assertEqual(parent_id, new_parent_id)
+        self.assertEqual(trace_fields, new_trace_fields)
+
 
 class TestHoneycombHTTPTraceParserHook(unittest.TestCase):
     def test_has_header(self):
@@ -55,6 +73,7 @@ def test_no_header(self):
 
 class TestHoneycombHTTPTracePropagationHook(unittest.TestCase):
     def test_generates_correct_header(self):
+        beeline.propagation.propagate_dataset = True
         dataset = "blorp blorp"
         trace_id = "bloop"
         parent_id = "scoop"
@@ -65,3 +84,16 @@ def test_generates_correct_header(self):
         self.assertIn('X-Honeycomb-Trace', headers)
         self.assertEqual(headers['X-Honeycomb-Trace'],
                          "1;dataset=blorp%20blorp,trace_id=bloop,parent_id=scoop,context=eyJrZXkiOiAidmFsdWUifQ==")
+
+    def test_generates_correct_header_with_dataset_propagation_disabled(self):
+        beeline.propagation.propagate_dataset = False
+        dataset = "blorp blorp"
+        trace_id = "bloop"
+        parent_id = "scoop"
+        trace_fields = {"key": "value"}
+        pc = PropagationContext(
+            trace_id, parent_id, trace_fields, dataset)
+        headers = hc.http_trace_propagation_hook(pc)
+        self.assertIn('X-Honeycomb-Trace', headers)
+        self.assertEqual(headers['X-Honeycomb-Trace'],
+                         "1;trace_id=bloop,parent_id=scoop,context=eyJrZXkiOiAidmFsdWUifQ==")

beeline/test_trace.py

@@ -7,6 +7,7 @@
 import re
 import os
 import binascii
+import beeline.propagation
 
 from libhoney import Event
 
@@ -624,6 +625,7 @@ def test_span_context(self):
 
 class TestPropagationHooks(unittest.TestCase):
     def test_propagate_and_start_trace_uses_honeycomb_header(self):
+        beeline.propagation.propagate_dataset = True
         # FIXME: Test basics, including error handling and custom hooks
 
         # implicitly tests finish_span
@@ -653,6 +655,40 @@ def test_propagate_and_start_trace_uses_honeycomb_header(self):
         # ensure that there is no current trace
         self.assertIsNone(tracer._trace)
 
+    def test_propagate_and_start_trace_uses_honeycomb_header_with_dataset_propagation_disabled(self):
+        beeline.propagation.propagate_dataset = False
+        # FIXME: Test basics, including error handling and custom hooks
+
+        # implicitly tests finish_span
+        m_client = Mock()
+        # these values are used before sending
+        m_client.new_event.return_value.start_time = datetime.datetime.now()
+        m_client.new_event.return_value.sample_rate = 1
+        tracer = SynchronousTracer(m_client)
+
+        header_value = '1;dataset=flibble,trace_id=bloop,parent_id=scoop,context=e30K'
+        req = DictRequest({
+            # case shouldn't matter
+            'X-HoNEyComb-TrACE': header_value,
+        })
+
+        span = tracer.propagate_and_start_trace(
+            context={'big': 'important_stuff'}, request=req)
+        self.assertEqual(tracer._trace.stack[0], span)
+        self.assertEqual(span.trace_id, "bloop")
+        self.assertEqual(span.parent_id, "scoop")
+
+        tracer.finish_trace(span)
+
+        # testing the absence of dataset
+        self.assertNotEqual(getattr(span.event, 'dataset'), "flibble")
+        self.assertNotIsInstance(span.event.dataset, str)
+
+        # ensure the event is sent
+        span.event.send_presampled.assert_called_once_with()
+        # ensure that there is no current trace
+        self.assertIsNone(tracer._trace)
+
     def test_propagate_and_start_trace_uses_w3c_header_as_fallback(self):
         # implicitly tests finish_span
         m_client = Mock()

examples/hello-world/README.md

@@ -0,0 +1,9 @@
+# hello world example
+
+## Setup and Run
+
+1. Set environment variable for `HONEYCOMB_API_KEY`
+1. In top-level directory of repo, run `poetry build` to create a wheel package in the `dist` directory
+1. In hello-world directory, ensure version of beeline in `pyproject.toml` matches the wheel package
+1. In hello-world directory, run `poetry install`
+1. In hello-world directory, run `poetry run python3 app.py`

examples/hello-world/app.py

@@ -0,0 +1,21 @@
+import beeline
+import os
+
+beeline.init(
+    # Get this via https://ui.honeycomb.io/account after signing up for Honeycomb
+    writekey=os.environ.get('HONEYCOMB_API_KEY'),
+    api_host=os.environ.get('HONEYCOMB_API_ENDPOINT', 'http://api.honeycomb.io:443'),
+    # The name of your app is a good choice to start with
+    # dataset='my-python-beeline-app', # only needed for classic
+    service_name=os.environ.get('SERVICE_NAME', 'my-python-app'),
+    debug=True, # enable to see telemetry in console
+)
+
+@beeline.traced(name='hello_world')
+def hello_world():
+    print('hello world')
+    beeline.add_context_field('message', 'hello world')
+
+hello_world()
+
+beeline.close()

examples/hello-world/pyproject.toml

@@ -0,0 +1,14 @@
+[tool.poetry]
+name = "hello-world"
+version = "0.1.0"
+description = "print hello world"
+authors = ["honeycombio"]
+
+[tool.poetry.dependencies]
+python = "^3.9"
+honeycomb-beeline = {path = "../../dist/honeycomb_beeline-3.42.99-py3-none-any.whl", develop = false}
+# honeycomb-beeline = "^3.2.0"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"