Update deb/rpm and installer script for fluentd (#52)

Author: jchengsfx

.circleci/config.yml

@@ -380,7 +380,7 @@ jobs:
           command: |
             sudo apt-get update
             sudo apt-get install -y ruby ruby-dev rubygems build-essential rpm
-            gem install --no-document fpm -v 1.11.0
+            sudo gem install --no-document fpm -v 1.11.0
       - run:
           name: Build << parameters.package_type >> amd64 package
           command: ./internal/buildscripts/packaging/fpm/<< parameters.package_type >>/build.sh "${CIRCLE_TAG:-}" "amd64" "./dist/"

README.md

@@ -90,6 +90,9 @@ For non-containerized Linux environments, a convenience script is available for
 installing the Collector package and [TD Agent
 (Fluentd)](https://www.fluentd.org/).
 
+You can view the [source](internal/buildscripts/packaging/installer/install.sh)
+for more details and available options.
+
 Run the following command on your host. Replace `SPLUNK_REALM`,
 `SPLUNK_BALLAST_SIZE`, and `SPLUNK_ACCESS_TOKEN` for your
 environment:
@@ -100,8 +103,22 @@ sudo sh /tmp/splunk-otel-collector.sh --realm SPLUNK_REALM --ballast SPLUNK_BALL
     -- SPLUNK_ACCESS_TOKEN
 ```
 
-You can view the [source](internal/buildscripts/packaging/installer/install.sh)
-for more details and other options.
+By default, the fluentd service will be installed and configured to forward
+log events with the `@SPLUNK` label to the collector (see the note below for
+how to add fluentd log sources), and the collector will send these events to
+the HEC ingest endpoint determined by the `--realm SPLUNK_REALM` option, e.g.
+`https://ingest.SPLUNK_REALM.signalfx.com/v1/log`.  To configure the collector
+to send log events to a custom HEC endpoint URL, specify the `--hec-url URL`
+and `--hec-token TOKEN` options to the command above.
+
+**Note**: The installer script does not include any fluentd log sources. Custom
+fluentd source config files can be added to the
+`/etc/otel/collector/fluentd/conf.d` directory after installation. Config files
+added to this directory should have a `.conf` extension, and the `td-agent`
+service will need to be restarted to include/enable the new files, i.e.
+`sudo systemctl restart td-agent`.  A sample config and instructions for
+collecting journald log events is available at
+`/etc/otel/collector/fluentd/conf.d/journald.conf.example`.
 
 Currently, only the following Linux distributions and versions are supported:
 

internal/buildscripts/packaging/fpm/Dockerfile

@@ -1,7 +1,7 @@
 FROM debian:9
 
 RUN apt-get update && \
-    apt-get install -y ruby ruby-dev rubygems build-essential git rpm
+    apt-get install -y ruby ruby-dev rubygems build-essential git rpm sudo
 
 RUN gem install --no-document fpm -v 1.11.0
 

internal/buildscripts/packaging/fpm/common.sh

@@ -14,16 +14,11 @@ SERVICE_NAME="splunk-otel-collector"
 SERVICE_USER="splunk-otel-collector"
 SERVICE_GROUP="splunk-otel-collector"
 
-SERVICE_REPO_PATH="$FPM_DIR/$SERVICE_NAME.service"
-SERVICE_INSTALL_PATH="/usr/lib/systemd/system/$SERVICE_NAME.service"
-
 OTELCOL_INSTALL_PATH="/usr/bin/otelcol"
-SPLUNK_CONFIG_REPO_PATH="$REPO_DIR/cmd/otelcol/config/collector/agent_config_linux.yaml"
-SPLUNK_CONFIG_INSTALL_PATH="/etc/otel/collector/splunk_config_linux.yaml"
-OTLP_CONFIG_REPO_PATH="$REPO_DIR/cmd/otelcol/config/collector/otlp_config_linux.yaml"
-OTLP_CONFIG_INSTALL_PATH="/etc/otel/collector/otlp_config_linux.yaml"
-SPLUNK_ENV_REPO_PATH="$FPM_DIR/splunk_env.example"
-SPLUNK_ENV_INSTALL_PATH="/etc/otel/collector/splunk_env.example"
+CONFIG_REPO_PATH="$FPM_DIR/etc/otel/collector/splunk_config_linux.yaml"
+CONFIG_INSTALL_PATH="/etc/otel/collector/splunk_config_linux.yaml"
+SERVICE_REPO_PATH="$FPM_DIR/$SERVICE_NAME.service"
+SERVICE_INSTALL_PATH="/lib/systemd/system/$SERVICE_NAME.service"
 
 PREINSTALL_PATH="$FPM_DIR/preinstall.sh"
 POSTINSTALL_PATH="$FPM_DIR/postinstall.sh"
@@ -42,3 +37,31 @@ get_version() {
         echo "$commit_tag"
     fi
 }
+
+create_user_group() {
+    sudo getent passwd $SERVICE_USER >/dev/null || \
+        sudo useradd --system --user-group --no-create-home --shell /sbin/nologin $SERVICE_USER
+}
+
+setup_files_and_permissions() {
+    local otelcol="$1"
+    local buildroot="$2"
+
+    create_user_group
+
+    mkdir -p "$buildroot/$(dirname $OTELCOL_INSTALL_PATH)"
+    cp -f "$otelcol" "$buildroot/$OTELCOL_INSTALL_PATH"
+    sudo chown root:root "$buildroot/$OTELCOL_INSTALL_PATH"
+    sudo chmod 755 "$buildroot/$OTELCOL_INSTALL_PATH"
+
+    cp -r "$FPM_DIR/etc" "$buildroot/etc"
+    cp -f "$CONFIG_REPO_PATH" "$buildroot/$CONFIG_INSTALL_PATH"
+    sudo chown -R $SERVICE_USER:$SERVICE_GROUP "$buildroot/etc/otel"
+    sudo chmod -R 755 "$buildroot/etc/otel"
+    sudo chmod 600 "$buildroot/etc/otel/collector/splunk_env.example"
+
+    mkdir -p "$buildroot/$(dirname $SERVICE_INSTALL_PATH)"
+    cp -f "$SERVICE_REPO_PATH" "$buildroot/$SERVICE_INSTALL_PATH"
+    sudo chown root:root "$buildroot/$SERVICE_INSTALL_PATH"
+    sudo chmod 644 "$buildroot/$SERVICE_INSTALL_PATH"
+}

internal/buildscripts/packaging/fpm/deb/build.sh

@@ -6,39 +6,36 @@ SCRIPT_DIR="$( cd "$( dirname ${BASH_SOURCE[0]} )" && pwd )"
 . $SCRIPT_DIR/../common.sh
 
 VERSION="${1:-}"
-ARCH="${2:-"amd64"}"
-OUTPUT_DIR="${3:-"$REPO_DIR/dist/"}"
-OTELCOL_REPO_PATH="$REPO_DIR/bin/otelcol_linux_${ARCH}"
-
-
-if [[ ! -f "$OTELCOL_REPO_PATH" ]]; then
-    echo "$OTELCOL_REPO_PATH not found!"
-    exit 1
-fi
+ARCH="${2:-amd64}"
+OUTPUT_DIR="${3:-$REPO_DIR/dist}"
 
 if [[ -z "$VERSION" ]]; then
     VERSION="$( get_version )"
 fi
+VERSION="${VERSION#v}"
+
+otelcol_path="$REPO_DIR/bin/otelcol_linux_${ARCH}"
+buildroot="$(mktemp -d)"
+
+setup_files_and_permissions "$otelcol_path" "$buildroot"
 
 mkdir -p "$OUTPUT_DIR"
 
-fpm -s dir -t deb -n $PKG_NAME -v ${VERSION#v} -f -p "$OUTPUT_DIR" \
+sudo fpm -s dir -t deb -n "$PKG_NAME" -v "$VERSION" -f -p "$OUTPUT_DIR" \
     --vendor "$PKG_VENDOR" \
     --maintainer "$PKG_MAINTAINER" \
     --description "$PKG_DESCRIPTION" \
     --license "$PKG_LICENSE" \
     --url "$PKG_URL" \
     --architecture "$ARCH" \
     --deb-dist "stable" \
-    --deb-user "$SERVICE_USER" \
-    --deb-group "$SERVICE_GROUP" \
+    --deb-use-file-permissions \
     --before-install "$PREINSTALL_PATH" \
     --after-install "$POSTINSTALL_PATH" \
     --before-remove "$PREUNINSTALL_PATH" \
-    --config-files $SPLUNK_CONFIG_INSTALL_PATH \
-    --config-files $OTLP_CONFIG_INSTALL_PATH \
-    $SPLUNK_CONFIG_REPO_PATH=$SPLUNK_CONFIG_INSTALL_PATH \
-    $OTLP_CONFIG_REPO_PATH=$OTLP_CONFIG_INSTALL_PATH \
-    $SPLUNK_ENV_REPO_PATH=$SPLUNK_ENV_INSTALL_PATH \
-    $SERVICE_REPO_PATH=$SERVICE_INSTALL_PATH \
-    $OTELCOL_REPO_PATH=$OTELCOL_INSTALL_PATH
+    --deb-no-default-config-files \
+    --config-files /etc/otel/collector/splunk_config_linux.yaml \
+    --config-files /etc/otel/collector/fluentd \
+    "$buildroot/"=/
+
+dpkg -c "${OUTPUT_DIR}/${PKG_NAME}_${VERSION}_${ARCH}.deb"

internal/buildscripts/packaging/fpm/etc/otel/collector/fluentd/README

@@ -0,0 +1,46 @@
+Custom TD Agent Fluentd Configuration for the Splunk OpenTelemetry Collector
+
+This directory contains a custom fluentd configuration to forward log events
+to the Splunk OpenTelemetry Collector.  By default, the collector will listen
+on 127.0.0.1:8006 for log events forwarded from fluentd.  See the
+"fluentforward" receiver in the default collector config at
+/etc/otel/collector/splunk_config_linux.yaml for details or to make any changes
+to the collector.
+
+Directory contents:
+
+- splunk-otel-collector.conf: Drop-in file for the fluentd service.  As an
+  alternative to overwriting the default fluentd config file
+  (/etc/td-agent/td-agent.conf), copy this file to
+  /etc/systemd/system/td-agent.service.d/splunk-otel-collector.conf to
+  override the default fluentd config path in favor of the custom
+  fluentd config file in this directory (see fluent.conf below), and run the
+  following commands to apply the changes:
+
+    systemctl daemon-reload
+    systemctl restart td-agent
+
+- fluent.conf: The main fluentd configuration file to forward events to the
+  collector.  By default, this file will configure fluentd to include custom
+  fluentd sources from the conf.d sub-directory (see conf.d below) and forward
+  all log events with the @SPLUNK label to the collector.  If changes are made
+  to this file, run the following command to apply the changes:
+
+    systemctl restart td-agent
+
+- conf.d: Directory for custom fluentd configuration files.  The main fluentd
+  configuration (see fluent.conf above) will automatically include all files
+  ending in .conf from the conf.d directory.  New fluentd sources should
+  include the @SPLUNK label for all log events intended to be forwarded to the
+  collector (see the sample file in conf.d for details).  After adding new
+  config files to the conf.d directory, run the following command to apply the
+  changes:
+
+    systemctl restart td-agent
+
+  *Important*: By default, the fluentd service runs as the "td-agent" user.
+  When adding new fluentd source configurations, ensure that the "td-agent"
+  user has permissions to access the paths defined in these sources.
+
+See https://docs.fluentd.org/configuration for general fluentd configuration
+details.

internal/buildscripts/packaging/fpm/etc/otel/collector/fluentd/conf.d/journald.conf.example

@@ -0,0 +1,55 @@
+# Sample fluentd source config to read from journald and forward all log events
+# to the Splunk OpenTelemetry Collector.
+#
+# To enable this source, perform the following steps:
+#
+# 1. Ensure the systemd fluentd plugin is installed by running:
+#
+#      td-agent-gem list fluent-plugin-systemd
+#
+#    If necessary, install the plugin by running:
+#
+#      td-agent-gem install fluent-plugin-systemd
+#
+#    Note: Additional dependencies may be required to be installed/upgraded in
+#    order to successfully install the plugin. See
+#    https://github.com/fluent-plugin-systemd/fluent-plugin-systemd for
+#    installation and configuration details.
+#
+# 2. Ensure that the fluentd service is configured to forward events to the
+#    collector by checking if the drop-in file
+#    "/etc/systemd/system/td-agent.service.d/splunk-otel-collector.conf"
+#    exists.  If not, copy "/etc/otel/collector/splunk-otel-collector.conf" to
+#    "/etc/systemd/system/td-agent.service.d/splunk-otel-collector.conf" and
+#    run:
+#
+#      systemctl daemon-reload
+#
+# 3. Make any necessary changes to the configuration below and copy/rename this
+#    file to "/etc/otel/collector/fluentd/conf.d/journald.conf".  See
+#    https://github.com/fluent-plugin-systemd/fluent-plugin-systemd for
+#    details and other options.
+#
+# 4. Since the fluentd service runs as the "td-agent" user, ensure that this
+#    user has permissions to access the path configured below.  For example, if
+#    the "/run/log/journal" path is only readable by the "systemd-journal"
+#    group, it may be necessary to add the "td-agent" user to the group by
+#    running the command:
+#
+#      usermod -a -G systemd-journal td-agent
+#
+# 5. Restart the fluentd service to apply the changes by running:
+#
+#      systemctl restart td-agent
+#
+
+<source>
+  @type systemd
+  @label @SPLUNK
+  tag "journald"
+  path "/run/log/journal"
+  <entry>
+    fields_strip_underscores true
+    fields_lowercase true
+  </entry>
+</source>

internal/buildscripts/packaging/fpm/etc/otel/collector/fluentd/fluent.conf

@@ -0,0 +1,26 @@
+@include conf.d/*.conf
+
+<label @SPLUNK>
+  <match **>
+    @type forward
+    heartbeat_type udp
+    <server>
+      host 127.0.0.1
+      port 8006
+    </server>
+    <buffer>
+      @type memory
+      total_limit_size 600m
+      chunk_limit_size 1m
+      chunk_limit_records 100000
+      flush_interval 5s
+      flush_thread_count 1
+      overflow_action block
+      retry_max_times 3
+    </buffer>
+  </match>
+</label>
+
+<system>
+  log_level info
+</system>

internal/buildscripts/packaging/fpm/etc/otel/collector/fluentd/splunk-otel-collector.conf

@@ -0,0 +1,11 @@
+# Drop-in file for the fluentd service
+
+# Copy this file to /etc/systemd/system/td-agent.service.d/ in order to override
+# the default fluentd config path in favor of the custom config defined below.
+
+# Then run the following commands to apply the changes:
+#   systemctl daemon-reload
+#   systemctl restart td-agent
+
+[Service]
+Environment=FLUENT_CONF=/etc/otel/collector/fluentd/fluent.conf