Adding functionality to add custom attributes in nginx otel-webserver-module. (#394)

* Nginx attributes feature  implementation

* corrected branch

Author: aryanishan1001

instrumentation/otel-webserver-module/Dockerfile

@@ -24,7 +24,7 @@ ARG LOG4CXX_VERSION="0.11.0"
 ARG GTEST_VERSION="1.10.0"
 ARG AUTOMAKE_VERSION="1.16.3"
 ARG PCRE_VERSION="8.44"
-ARG NGINX_VERSION="1.18.0"
+ARG NGINX_VERSION="1.24.0"
 
 
 # create default non-root user
@@ -273,7 +273,7 @@ RUN cd /otel-webserver-module/build \
     && cd /
 
 RUN cp /otel-webserver-module/conf/nginx/opentelemetry_module.conf /opt/ \
-    && sed -i '8i load_module /opt/opentelemetry-webserver-sdk/WebServerModule/Nginx/ngx_http_opentelemetry_module.so;' /etc/nginx/nginx.conf \
+    && sed -i '8i load_module /opt/opentelemetry-webserver-sdk/WebServerModule/Nginx/1.24.0/ngx_http_opentelemetry_module.so;' /etc/nginx/nginx.conf \
     && sed -i '33i include /opt/opentelemetry_module.conf;' /etc/nginx/nginx.conf \ 
     && cd /
 

instrumentation/otel-webserver-module/codeql-env.sh

@@ -36,7 +36,7 @@ APRUTIL_VERSION="1.6.1"
 LOG4CXX_VERSION="0.11.0"
 GTEST_VERSION="1.10.0"
 PCRE_VERSION="8.44"
-NGINX_VERSION="1.18.0"
+NGINX_VERSION="1.24.0"
 
 # Install GRPC
 git clone --shallow-submodules --depth 1 --recurse-submodules -b v${GRPC_VERSION} \

instrumentation/otel-webserver-module/include/core/api/Payload.h

@@ -87,11 +87,13 @@ class RequestPayload
     std::unordered_map<std::string, std::string>& get_request_headers() {
     	return request_headers;
     }
+
 };
 
 struct ResponsePayload
 {
 	std::unordered_map<std::string, std::string> response_headers;
+	std::unordered_map<std::string, std::string> otel_attributes;
 	unsigned int status_code{sdkwrapper::kStatusCodeInit};
 };
 

instrumentation/otel-webserver-module/include/core/api/opentelemetry_ngx_api.h

@@ -45,6 +45,7 @@ typedef struct {
     const char* http_post_param;
     const char* request_method;
     const char* client_ip;
+
     http_headers* propagation_headers;
     http_headers* request_headers;
 
@@ -55,6 +56,8 @@ typedef struct {
 typedef struct {
     http_headers* response_headers;
     int response_headers_count;
+    http_headers* otel_attributes;
+    int otel_attributes_count;
 
     unsigned int status_code;
 }response_payload;

instrumentation/otel-webserver-module/include/core/sdkwrapper/SdkConstants.h

@@ -39,6 +39,7 @@ const std::string kAttrNETHostPort            = "net.host.port";
 const std::string kAttrRequestProtocol        = "request_protocol";
 const std::string kHTTPFlavor1_0                 = "1.0";
 const std::string kHTTPFlavor1_1                = "1.1";
+const std::string kOtelAttributes             = "otel.attribute.";
 
 
 constexpr int HTTP_ERROR_1XX = 100;

instrumentation/otel-webserver-module/src/core/api/RequestProcessingEngine.cpp

@@ -82,6 +82,7 @@ OTEL_SDK_STATUS_CODE RequestProcessingEngine::startRequest(
                 std::string(itr->first);
         keyValueMap[key] = itr->second;
     }
+
     auto span = m_sdkWrapper->CreateSpan(spanName, sdkwrapper::SpanKind::SERVER, keyValueMap, payload->get_http_headers());
 
     LOG4CXX_TRACE(mLogger, "Span started for context: [" << wscontext
@@ -157,6 +158,11 @@ OTEL_SDK_STATUS_CODE RequestProcessingEngine::endRequest(
             rootSpan->AddAttribute(key, itr->second);
         }
 
+        for (auto itr = payload->otel_attributes.begin(); itr != payload->otel_attributes.end(); itr++) {
+            std::string key = std::string(kOtelAttributes) + std::string(itr->first);
+            rootSpan->AddAttribute(key, itr->second);
+        }
+
         rootSpan->AddAttribute(kAttrHTTPStatusCode, payload->status_code);
     }
 

instrumentation/otel-webserver-module/src/core/api/opentelemetry_ngx_api.cpp

@@ -119,13 +119,17 @@ OTEL_SDK_STATUS_CODE endRequest(OTEL_SDK_HANDLE_REQ req_handle_key, const char*
     if (payload != NULL) {
         for (int i = 0; i < payload->response_headers_count; i++) {
             std::string key(payload->response_headers[i].name);
-            if (responseHeadersToCapture.find(key)
-            != responseHeadersToCapture.end()) {
+            if (responseHeadersToCapture.find(key) != responseHeadersToCapture.end()) {
                 responsePayload->response_headers[key]
                     = payload->response_headers[i].value;
             }
         }
         responsePayload->status_code = payload->status_code;
+
+        for(int i=0; i < payload->otel_attributes_count; i++){
+            std::string key(payload->otel_attributes[i].name);
+            responsePayload->otel_attributes[key] = payload->otel_attributes[i].value;
+        }
     }
 
     res = wsAgent.endRequest(req_handle_key, errMsg, responsePayload.get());

instrumentation/otel-webserver-module/src/nginx/ngx_http_opentelemetry_module.c

@@ -383,10 +383,19 @@ static ngx_command_t ngx_http_opentelemetry_commands[] = {
       NGX_HTTP_LOC_CONF_OFFSET,
       offsetof(ngx_http_opentelemetry_loc_conf_t, nginxTrustIncomingSpans),
       NULL},
-
-    ngx_null_command	/* command termination */
+    	
+    { ngx_string("NginxAttributes"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_1MORE,
+      ngx_otel_attributes_set,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      0,
+      NULL},
+    
+    /* command termination */
+    ngx_null_command
 };
 
+
 /* The module context. */
 static ngx_http_module_t ngx_http_opentelemetry_module_ctx = {
     NULL,						/* preconfiguration */
@@ -454,6 +463,7 @@ static char* ngx_http_opentelemetry_merge_loc_conf(ngx_conf_t *cf, void *parent,
     ngx_http_opentelemetry_loc_conf_t *prev = parent;
     ngx_http_opentelemetry_loc_conf_t *conf = child;
     ngx_otel_set_global_context(prev);
+    ngx_otel_set_attributes(prev, conf);
 
     ngx_conf_merge_value(conf->nginxModuleEnabled, prev->nginxModuleEnabled, 1);
     ngx_conf_merge_value(conf->nginxModuleReportAllInstrumentedModules, prev->nginxModuleReportAllInstrumentedModules, 0);
@@ -673,6 +683,34 @@ static void ngx_http_opentelemetry_exit_worker(ngx_cycle_t *cycle)
         ngx_log_error(NGX_LOG_ERR, cycle->log, 0, "mod_opentelemetry: ngx_http_opentelemetry_exit_worker: Exiting Nginx Worker for process with PID: %s**********", worker_conf->pid);
     }
 }
+static char* ngx_otel_attributes_set(ngx_conf_t* cf, ngx_command_t* cmd, void* conf) {
+    ngx_http_opentelemetry_loc_conf_t * my_conf=(ngx_http_opentelemetry_loc_conf_t *)conf;
+
+    ngx_str_t *value = cf->args->elts;
+
+    ngx_array_t *arr;
+    ngx_str_t   *elt;
+    ngx_int_t arg_count = cf->args->nelts; 
+
+    arr = ngx_array_create(cf->pool, arg_count, sizeof(ngx_str_t));
+
+    if (arr == NULL) {
+        return NGX_CONF_ERROR;
+    }
+
+    // Add elements to the array
+    for (ngx_int_t i = 1; i < arg_count; i++) {
+        elt = ngx_array_push(arr);
+        if (elt == NULL) {
+            return NGX_CONF_ERROR;
+        }
+        ngx_str_set(elt, value[i].data);
+        elt->len = ngx_strlen(value[i].data);
+    }
+    my_conf->nginxAttributes = arr;
+    return NGX_CONF_OK;
+
+}
 
 static char* ngx_otel_context_set(ngx_conf_t *cf, ngx_command_t *cmd, void *conf){
     ngx_str_t* value;
@@ -704,6 +742,20 @@ static void ngx_otel_set_global_context(ngx_http_opentelemetry_loc_conf_t * prev
     }
 }
 
+static void ngx_otel_set_attributes(ngx_http_opentelemetry_loc_conf_t * prev, ngx_http_opentelemetry_loc_conf_t * conf)
+{
+    if (conf->nginxAttributes && (conf->nginxAttributes->nelts) > 0) {
+        return;
+    }
+    if (prev->nginxAttributes && (prev->nginxAttributes->nelts) > 0) {
+        if((conf->nginxAttributes) == NULL)
+        {
+            conf->nginxAttributes = prev->nginxAttributes;
+        }
+    }
+    return;
+}
+
 /*
     Begin a new interaction
 */
@@ -737,6 +789,7 @@ static OTEL_SDK_STATUS_CODE otel_startInteraction(ngx_http_request_t* r, const c
         int ix = 0;
         res = startModuleInteraction((void*)ctx->otel_req_handle_key, module_name, "", resolveBackends, propagationHeaders, &ix);
 
+
         if (OTEL_ISSUCCESS(res))
         {
             removeUnwantedHeader(r);
@@ -892,6 +945,39 @@ static ngx_uint_t otel_getErrorCode(ngx_http_request_t* r)
     else return 0;
 }
 
+static void resolve_attributes_variables(ngx_http_request_t* r)
+{
+    ngx_http_opentelemetry_loc_conf_t *conf = ngx_http_get_module_loc_conf(r, ngx_http_opentelemetry_module);
+
+    for (ngx_uint_t j = 0; j < conf->nginxAttributes->nelts; j++) {
+        
+        void *element = conf->nginxAttributes->elts + (j * conf->nginxAttributes->size);
+        ngx_str_t var_name = (((ngx_str_t *)(conf->nginxAttributes->elts))[j]);
+        ngx_uint_t           key; // The variable's hashed key.
+        ngx_http_variable_value_t  *value; // Pointer to the value object.
+
+        if(var_name.data[0] == '$'){
+            // Get the hashed key.
+            ngx_str_t new_var_name = var_name;
+            new_var_name.data++;
+            new_var_name.len--;
+            key = ngx_hash_key(new_var_name.data, new_var_name.len);
+
+            // Get the variable.
+            value = ngx_http_get_variable(r, &new_var_name, key);
+
+            if (value == NULL || value->not_found) {
+                // Variable was not found.
+            } else {
+                ngx_str_t * ngx_str = (ngx_str_t *)(element);
+                ngx_str->data = value->data;
+                ngx_str->len = ngx_strlen(value);
+                // Variable was found, `value` now contains the value.
+            }
+        }
+    }
+}
+
 static ngx_flag_t ngx_initialize_opentelemetry(ngx_http_request_t *r)
 {
     // check to see if we have already been initialized
@@ -1274,7 +1360,7 @@ static ngx_int_t ngx_http_otel_realip_handler(ngx_http_request_t *r){
 
     otel_startInteraction(r, "ngx_http_realip_module");
     ngx_int_t rvalue = h[NGX_HTTP_REALIP_MODULE_INDEX](r);
-    otel_stopInteraction(r, "ngx_http_realip_module", OTEL_SDK_NO_HANDLE);
+    otel_stopInteraction(r, "ngx_http_realip_module", OTEL_SDK_NO_HANDLE);    
 
     return rvalue;
 }
@@ -1555,6 +1641,7 @@ static void removeUnwantedHeader(ngx_http_request_t* r)
   }
 }
 
+
 static void fillRequestPayload(request_payload* req_payload, ngx_http_request_t* r){
     ngx_list_part_t  *part;
     ngx_table_elt_t  *header;
@@ -1636,6 +1723,7 @@ static void fillRequestPayload(request_payload* req_payload, ngx_http_request_t*
 
     ngx_http_opentelemetry_loc_conf_t *conf =
       ngx_http_get_module_loc_conf(r, ngx_http_opentelemetry_module);
+
     part = &r->headers_in.headers.part;
     header = (ngx_table_elt_t*)part->elts;
     nelts = part->nelts;
@@ -1669,6 +1757,7 @@ static void fillRequestPayload(request_payload* req_payload, ngx_http_request_t*
     }
     req_payload->propagation_count = propagation_headers_idx;
     req_payload->request_headers_count = request_headers_idx;
+
 }
 
 static void fillResponsePayload(response_payload* res_payload, ngx_http_request_t* r)
@@ -1689,6 +1778,36 @@ static void fillResponsePayload(response_payload* res_payload, ngx_http_request_
     res_payload->response_headers = ngx_pcalloc(r->pool, nelts * sizeof(http_headers));
     ngx_uint_t headers_count = 0;
 
+     ngx_http_opentelemetry_loc_conf_t *conf = ngx_http_get_module_loc_conf(r, ngx_http_opentelemetry_module);
+    
+    if (conf->nginxAttributes && (conf->nginxAttributes->nelts) > 0) {
+
+        res_payload->otel_attributes = ngx_pcalloc(r->pool, ((nelts+1)/3) * sizeof(http_headers));
+        int otel_attributes_idx=0;
+
+        resolve_attributes_variables(r);
+        for (ngx_uint_t j = 0, isKey = 1, isValue = 0; j < conf->nginxAttributes->nelts; j++) {
+            const char* data = (const char*)(((ngx_str_t *)(conf->nginxAttributes->elts))[j]).data;
+            if(strcmp(data, ",") == 0){
+                otel_attributes_idx++;
+                continue;
+            }
+            else if(isKey){
+                res_payload->otel_attributes[otel_attributes_idx].name = data; 
+            }
+            else{
+                res_payload->otel_attributes[otel_attributes_idx].value = data;
+            } 
+            isKey=!isKey;
+            isValue=!isValue;
+        }
+        res_payload->otel_attributes_count = otel_attributes_idx+1;
+    }else {
+        res_payload->otel_attributes_count = 0;
+    }
+
+
+
     for (ngx_uint_t j = 0; j < nelts; j++) {
         h = &header[j];
 

instrumentation/otel-webserver-module/src/nginx/ngx_http_opentelemetry_module.h

@@ -104,6 +104,7 @@ typedef struct {
     ngx_str_t   nginxModuleResponseHeaders;
     ngx_str_t   nginxModuleOtelExporterOtlpHeaders;
     ngx_flag_t  nginxTrustIncomingSpans;
+    ngx_array_t  *nginxAttributes;
 
 } ngx_http_opentelemetry_loc_conf_t;
 
@@ -154,7 +155,9 @@ static void otel_payload_decorator(ngx_http_request_t* r, OTEL_SDK_ENV_RECORD* p
 static ngx_flag_t otel_requestHasErrors(ngx_http_request_t* r);
 static ngx_uint_t otel_getErrorCode(ngx_http_request_t* r);
 static char* ngx_otel_context_set(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
+static char* ngx_otel_attributes_set(ngx_conf_t* cf, ngx_command_t*, void* conf);
 static void ngx_otel_set_global_context(ngx_http_opentelemetry_loc_conf_t * prev);
+static void ngx_otel_set_attributes(ngx_http_opentelemetry_loc_conf_t * prev, ngx_http_opentelemetry_loc_conf_t * conf);
 static void removeUnwantedHeader(ngx_http_request_t* r);
 /*
     Module specific handler