RFC-0030: Add support for file-based service bindings

Author: klapkov

jobs/rep/spec

@@ -22,7 +22,8 @@ templates:
   tls.key.erb: config/certs/tls.key
   tls_ca.crt.erb: config/certs/tls_ca.crt
   indicators.yml.erb: config/indicators.yml
-
+  volume_mounted_files.erb: bin/volume_mounted_files
+  
 packages:
   - pid_utils
   - rep

jobs/rep/templates/bpm-pre-start.erb

@@ -5,3 +5,5 @@ bin_dir=/var/vcap/jobs/rep/bin
 $bin_dir/set-rep-kernel-params
 
 $bin_dir/setup_mounted_data_dirs
+
+$bin_dir/volume_mounted_files

jobs/rep/templates/rep.json.erb

@@ -4,6 +4,7 @@
   trusted_certs_dir = "/var/vcap/data/rep/shared/garden/trusted_certs"
   instance_identity_dir = "/var/vcap/data/rep/shared/garden/instance_identity"
   download_cache_dir= "/var/vcap/data/rep/shared/garden/download_cache"
+  volume_mounted_files = "/var/vcap/data/rep/shared/garden/volume_mounted_files"
 
   zone = spec.az
   if_p("diego.rep.zone") do |value|
@@ -104,6 +105,7 @@
     report_interval: "1m",
     max_data_string_length: p("logging.max_data_string_length"),
     max_log_lines_per_second: p("diego.executor.max_log_lines_per_second"),
+    volume_mounted_files: "#{volume_mounted_files}",
   }
 
   if p("containers.graceful_shutdown_interval_in_seconds") < 10

jobs/rep/templates/rep_ctl.erb

@@ -38,6 +38,7 @@ case $1 in
 
     $bin_dir/set-rep-kernel-params
     $bin_dir/setup_mounted_data_dirs
+    $bin_dir/volume_mounted_files
 
     # Allowed number of open file descriptors
     ulimit -n 100000

jobs/rep/templates/volume_mounted_files.erb

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+<%
+  _max_containers = 250
+  if_p("diego.rep.max_containers") do |value|
+    _max_containers = value
+  end
+  if _max_containers <= 0
+    raise "The max_containers prop should be a positive integer"
+  end
+%>
+max_containers=<%= _max_containers %>
+
+# Define the service binding root directory
+volume_mounted_files="/var/vcap/data/rep/shared/garden/volume_mounted_files"
+
+# Calculate the size for the tmpfs (1MB per container)
+root_tmpfs_size=$((max_containers * 1))M
+
+# Ensure the root directory is safely removed and recreated
+if [ -d "$volume_mounted_files" ]; then
+  # Ensure the root directory is unmounted before removing it
+  if mountpoint -q "$volume_mounted_files"; then
+    fuser -k "$volume_mounted_files"
+    umount -l "$volume_mounted_files"
+  fi
+
+  sleep 10
+
+  rm -rf "$volume_mounted_files"
+fi
+
+mkdir -p "$volume_mounted_files"
+
+# Mount the root tmpfs
+mount -t tmpfs -o size="$root_tmpfs_size" tmpfs "$volume_mounted_files" || exit 1
+
+# Set permissions and ownership for the root directory and all subdirectories
+chmod 0700 "$volume_mounted_files"
+chown vcap:vcap "$volume_mounted_files"

jobs/rep_windows/templates/pre-start.ps1.erb

@@ -37,11 +37,12 @@ if (-Not (Get-NetFirewallRule | Where-Object { $_.DisplayName -eq "SecureRepPort
 }
 
 # Remove old data dirs so that disk space isn't wasted. This cleanup can be removed in v4
-Remove-Item -Path "/var/vcap/data/rep/download_cache" -Force -Recurse -ErrorAction "Continue"
-Remove-Item -Path "/var/vcap/data/rep/instance_identity" -Force -Recurse -ErrorAction "Continue"
-Remove-Item -Path "/var/vcap/data/rep/trusted_certs" -Force -Recurse -ErrorAction "Continue"
-Remove-Item -Path "/var/vcap/data/rep/proxy_config" -Force -Recurse -ErrorAction "Continue"
+Remove-Item -Path "/var/vcap/data/rep/shared/garden/download_cache" -Force -Recurse -ErrorAction "Continue"
+Remove-Item -Path "/var/vcap/data/rep/shared/garden/instance_identity" -Force -Recurse -ErrorAction "Continue"
+Remove-Item -Path "/var/vcap/data/rep/shared/garden/trusted_certs" -Force -Recurse -ErrorAction "Continue"
+Remove-Item -Path "/var/vcap/data/rep/shared/garden/proxy_config" -Force -Recurse -ErrorAction "Continue"
 Remove-Item -Path "/var/vcap/data/rep/shared_data" -Force -Recurse -ErrorAction "Continue"
+Remove-Item -Path "/var/vcap/data/rep/shared/garden/volume_mounted_files" -Force -Recurse -ErrorAction "Continue"
 
 $bindmountDirs = @()
 
@@ -53,6 +54,10 @@ $instance_identity_dir = "/var/vcap/data/rep/shared/garden/instance_identity"
 New-Item -Path $instance_identity_dir -ItemType "directory" -Force
 $bindmountDirs += $instance_identity_dir
 
+$service_bindings_dir = "/var/vcap/data/rep/shared/garden/volume_mounted_files"
+New-Item -Path $service_bindings_dir -ItemType "directory" -Force
+$bindmountDirs += $service_bindings_dir
+
 <% if_p("containers.trusted_ca_certificates") do |value| %>
   $conf_dir = "C:/var/vcap/jobs/rep_windows/config"
   $trusted_certs_dir = "C:/var/vcap/data/rep/shared/garden/trusted_certs"

jobs/rep_windows/templates/rep.json.erb

@@ -4,6 +4,7 @@
   trusted_certs_dir = "/var/vcap/data/rep/shared/garden/trusted_certs"
   instance_identity_dir = "/var/vcap/data/rep/shared/garden/instance_identity"
   download_cache_dir= "/var/vcap/data/rep/shared/garden/download_cache"
+  volume_mounted_files = "/var/vcap/data/rep/shared/garden/volume_mounted_files"
 
   zone = spec.az
   if_p("diego.rep.zone") do |value|
@@ -104,6 +105,7 @@
     report_interval: "1m",
     max_data_string_length: p("logging.max_data_string_length"),
     max_log_lines_per_second: p("diego.executor.max_log_lines_per_second"),
+    volume_mounted_files: "#{volume_mounted_files}",
   }
 
   if p("containers.graceful_shutdown_interval_in_seconds") < 10

spec/rep_template_spec.rb

@@ -106,5 +106,18 @@
         end.to raise_error(/The max_containers prop should be a positive integer/)
       end
     end
-  end  
-end
+  end
+
+  describe 'volume_mounted_files.erb' do
+    let(:template) { job.template('bin/volume_mounted_files') }
+
+    context 'checks the max_containers value' do
+      it 'raises an error if max_containers is <= 0' do
+        deployment_manifest_fragment['diego']['rep']['max_containers'] = -10
+        expect do
+          rendered_template
+        end.to raise_error(/The max_containers prop should be a positive integer/)
+      end
+    end
+  end
+end