Skip to content

Commit 63057a7

Browse files
vince-wekavince-weka
authored
Merge pull request #70 from weka/vince/loki-2025-07-11
enhanced loki support
2 parents e50933b + bbb1770 commit 63057a7

File tree

5 files changed

+386
-55
lines changed

5 files changed

+386
-55
lines changed

‎docker_build

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,5 @@
22
./build.sh
33
VERSION=$(./tarball/export/export --version | awk '{print $3}')
44
docker build --tag wekasolutions/export:latest --tag wekasolutions/export:$VERSION .
5+
#docker build --tag wekasolutions/export:experimental --tag wekasolutions/export:$VERSION .
56
#docker build --tag wekasolutions/export:$VERSION .

‎events.py

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -30,29 +30,40 @@ def __init__(self, registry):
3030
self.registry = registry
3131

3232

33-
def configure_loki(self, lokihost, lokiport):
33+
def configure_loki(self, loki):
3434
self.loki = True
35-
self.host = lokihost
36-
self.port = lokiport
35+
self.config = loki
36+
self.url = f"{self.config['protocol']}://{self.config['host']}:{self.config['port']}{self.config['path']}"
37+
38+
# set user & password, if needed
39+
if self.config['user'] is not None and self.config['password'] is not None:
40+
self.auth = (self.config['user'], self.config['password'])
41+
log.debug(f"Using password authentication")
42+
else:
43+
self.auth = None
44+
45+
# set the headers
46+
self.headers = {
47+
'Content-type': 'application/json'
48+
}
49+
50+
# ord_id is for multi-tenancy in Loki
51+
if self.config['org_id'] is not None:
52+
self.headers['X-Scope-OrgID'] = self.config['org_id']
53+
log.debug(f"Setting OrgID to {self.config['org_id']}")
54+
3755
# save some trouble, and make sure names are resolvable
3856
try:
39-
socket.gethostbyname(lokihost)
57+
socket.gethostbyname(self.config['host'])
4058
except socket.gaierror as exc:
41-
log.critical(f"Loki Server name '{lokihost}' is not resolvable - is it in /etc/hosts or DNS?")
59+
log.critical(f"Loki Server name '{self.config['host']}' is not resolvable - is it in /etc/hosts or DNS?")
4260
raise
4361
except Exception as exc:
4462
log.critical(exc)
4563
raise
4664

4765
# push msg log into grafana-loki
4866
def loki_logevent(self, timestamp, event, **labels):
49-
url = 'http://' + self.host + ':' + str(self.port) + '/loki/api/v1/push' # set the URL
50-
51-
# set the headers
52-
headers = {
53-
'Content-type': 'application/json'
54-
}
55-
5667
log.debug(f"{labels}")
5768
# set the payload
5869
payload = {
@@ -72,9 +83,15 @@ def loki_logevent(self, timestamp, event, **labels):
7283

7384
# this is where we actually send it
7485
try:
75-
answer = requests.post(url, data=payload_str, headers=headers)
86+
answer = requests.post(self.url,
87+
data=payload_str,
88+
headers=self.headers,
89+
auth=self.auth,
90+
verify=self.config['verify_cert'],
91+
cert=self.config['client_cert']
92+
)
7693
except requests.exceptions.ConnectionError as exc:
77-
log.critical(f"Unable to send Events to Loki: unable to establish connection: FATAL")
94+
log.critical(f"Unable to send Event to Loki: unable to establish connection: FATAL: {exc}")
7895
return False
7996
except Exception as exc:
8097
log.critical(f"Unable to send Events to Loki: {exc}")

‎export-old.yml

Lines changed: 271 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,271 @@
1+
#
2+
# config file for wekasolutions/export
3+
#
4+
5+
# exporter section - info about how we're going to run
6+
exporter:
7+
listen_port: 8150
8+
loki_host: wms
9+
loki_port: 3100
10+
events_only: False
11+
events_to_loki: True
12+
events_to_syslog: True
13+
timeout: 10.0
14+
max_procs: 8
15+
max_threads_per_proc: 100
16+
backends_only: True
17+
datapoints_per_collect: 5
18+
certfile: null
19+
keyfile: null
20+
21+
# cluster section - info about the weka cluster we want to export data from:
22+
cluster:
23+
auth_token_file: auth-token.json
24+
hosts:
25+
- 172.29.0.64
26+
force_https: False # only 3.10+ clusters support https
27+
verify_cert: False # default cert cannot be verified
28+
mgmt_port: 14000
29+
30+
# auth_token_file must be a filename.
31+
# If just a filename it will be searched for in ".", "~/.weka", and "./.weka"
32+
# This file can be generated with the 'weka user login' command and copied to where we are running
33+
# hosts is a list of hostnames or ip addresses. Minimum of 1 required. You do not need to list all hosts in the cluster
34+
35+
# This file comes pre-set to populate the Grafana Panels that we've provided
36+
#
37+
# File format:
38+
#
39+
#stats:
40+
# category:
41+
# statistic: unit_of_measurement
42+
# statistic: unit_of_measurement
43+
# statistic: unit_of_measurement
44+
#
45+
# if you are familiar with "weka stats", these are "--category <category> --stat <statistic>"
46+
#
47+
# For more info refer to: https://docs.weka.io/usage/statistics/list-of-statistics
48+
#
49+
# To monitor additional statistics, just uncomment the ones you want to be gathered/reported
50+
# If uncommenting things in a commented out section/Category (ie: 'object_storage:), don't forget to uncomment
51+
# the Category itself (eg: "# 'object_storage:") as well as one or more of the metrics under it
52+
# These lines are marked "# Category"
53+
#
54+
# It does not like Categories with no metrics - fix forthcoming
55+
# meanwhile, don't uncomment a Category without uncommenting at least one statistic under it.
56+
#
57+
#
58+
59+
stats:
60+
cpu: # Category
61+
CPU_UTILIZATION: "%" # metric
62+
# object_storage: # Category
63+
# FAILED_OBJECT_DELETES: count
64+
# FAILED_OBJECT_DOWNLOADS: count
65+
# FAILED_OBJECT_OPERATIONS: count
66+
# FAILED_OBJECT_UPLOADS: count
67+
# OBJECT_DELETES: count
68+
# OBJECT_DELETE_DURATION: microsecs
69+
# OBJECT_DELETE_LATENCY: microsecs
70+
# OBJECT_DOWNLOADS: count
71+
# OBJECT_DOWNLOADS_BG: count
72+
# OBJECT_DOWNLOADS_FG: count
73+
# OBJECT_DOWNLOAD_BYTES_BG: bytespersec
74+
# OBJECT_DOWNLOAD_BYTES_FG: bytespersec
75+
# OBJECT_DOWNLOAD_DURATION: microsecs
76+
# OBJECT_DOWNLOAD_LATENCY: microsecs
77+
# OBJECT_DOWNLOAD_SIZE: count
78+
# OBJECT_OPERATIONS: count
79+
# OBJECT_UPLOADS: count
80+
# OBJECT_UPLOADS_BACKPRESSURE: count
81+
# OBJECT_UPLOADS_MANUAL: count
82+
# OBJECT_UPLOADS_MIGRATE: count
83+
# OBJECT_UPLOADS_POLICY: count
84+
# OBJECT_UPLOADS_RECLAMATION_REUPLOAD: count
85+
# OBJECT_UPLOADS_STOW: count
86+
# OBJECT_UPLOAD_BYTES_BACKPRESSURE: bytespersec
87+
# OBJECT_UPLOAD_BYTES_MANUAL: bytespersec
88+
# OBJECT_UPLOAD_BYTES_MIGRATE: bytespersec
89+
# OBJECT_UPLOAD_BYTES_POLICY: bytespersec
90+
# OBJECT_UPLOAD_BYTES_RECLAMATION_REUPLOAD: bytespersec
91+
# OBJECT_UPLOAD_BYTES_STOW: bytespersec
92+
# OBJECT_UPLOAD_DURATION: microsecs
93+
# OBJECT_UPLOAD_LATENCY: microsecs
94+
# OBJECT_UPLOAD_SIZE: bytes
95+
# OBS_READ_BYTES: bytespersec
96+
# OBS_WRITE_BYTES: bytespersec
97+
# ONGOING_DOWNLOADS: count
98+
# ONGOING_REMOVES: count
99+
# ONGOING_UPLOADS: count
100+
# READ_BYTES: bytespersec
101+
# WRITE_BYTES: bytespersec
102+
ops:
103+
# ACCESS_LATENCY: microsecs
104+
ACCESS_OPS: ops
105+
# COMMIT_LATENCY: microsecs
106+
COMMIT_OPS: ops
107+
# CREATE_LATENCY: microsecs
108+
CREATE_OPS: ops
109+
# FILEATOMICOPEN_LATENCY: microsecs
110+
# FILEATOMICOPEN_OPS: ops
111+
# FILECLOSE_LATENCY: microsecs
112+
FILECLOSE_OPS: ops
113+
# FILEOPEN_LATENCY: microsecs
114+
FILEOPEN_OPS: ops
115+
# FLOCK_LATENCY: microsecs
116+
FLOCK_OPS: ops
117+
# FSINFO_LATENCY: microsecs
118+
FSINFO_OPS: ops
119+
# GETATTR_LATENCY: microsecs
120+
GETATTR_OPS: ops
121+
# LINK_LATENCY: microsecs
122+
LINK_OPS: ops
123+
# LOOKUP_LATENCY: microsecs
124+
# LOOKUP_OPS: ops
125+
# MKDIR_LATENCY: microsecs
126+
MKDIR_OPS: ops
127+
# MKNOD_LATENCY: microsecs
128+
MKNOD_OPS: ops
129+
OPS: ops
130+
# PATHCONF_LATENCY: microsecs
131+
# PATHCONF_OPS: ops
132+
# READDIR_LATENCY: microsecs
133+
READDIR_OPS: ops
134+
# READLINK_LATENCY: microsecs
135+
# READLINK_OPS: ops
136+
READS: iops
137+
READ_BYTES: bytespersec
138+
READ_DURATION: microsecs
139+
READ_LATENCY: microsecs
140+
# REMOVE_LATENCY: microsecs
141+
REMOVE_OPS: ops
142+
# RENAME_LATENCY: microsecs
143+
RENAME_OPS: ops
144+
# RMDIR_LATENCY: microsecs
145+
RMDIR_OPS: ops
146+
# SETATTR_LATENCY: microsecs
147+
# SETATTR_OPS: ops
148+
# STATFS_LATENCY: microsecs
149+
# STATFS_OPS: ops
150+
# SYMLINK_LATENCY: microsecs
151+
# SYMLINK_OPS: ops
152+
THROUGHPUT: bytespersec
153+
# UNLINK_LATENCY: microsecs
154+
UNLINK_OPS: ops
155+
WRITES: iops
156+
WRITE_BYTES: bytespersec
157+
# WRITE_DURATION: microsecs
158+
WRITE_LATENCY: microsecs
159+
ops_driver: # Category
160+
DIRECT_READ_SIZES: sizes
161+
DIRECT_WRITE_SIZES: sizes
162+
# FILEATOMICOPEN_LATENCY: microsecs
163+
# FILEATOMICOPEN_OPS: ops
164+
# FILECLOSE_LATENCY: microsecs
165+
# FILECLOSE_OPS: ops
166+
# FILEOPEN_LATENCY: microsecs
167+
# FILEOPEN_OPS: ops
168+
# FLOCK_LATENCY: microsecs
169+
# FLOCK_OPS: ops
170+
# GETATTR_LATENCY: microsecs
171+
# GETATTR_OPS: ops
172+
# IOCTL_OBS_PREFETCH_LATENCY: microsecs
173+
# IOCTL_OBS_PREFETCH_OPS: ops
174+
# LINK_LATENCY: microsecs
175+
# LINK_OPS: ops
176+
# LOOKUP_LATENCY: microsecs
177+
# LOOKUP_OPS: ops
178+
# MKNOD_LATENCY: microsecs
179+
# MKNOD_OPS: ops
180+
# OPS: ops
181+
# READDIR_LATENCY: microsecs
182+
# READDIR_OPS: ops
183+
# READLINK_LATENCY: microsecs
184+
# READLINK_OPS: ops
185+
# READS: iops
186+
# READ_BYTES: bytespersec
187+
# READ_DURATION: microsecs
188+
# READ_LATENCY: microsecs
189+
READ_SIZES: sizes
190+
# RENAME_LATENCY: microsecs
191+
# RENAME_OPS: ops
192+
# RMDIR_LATENCY: microsecs
193+
# RMDIR_OPS: ops
194+
# SETATTR_LATENCY: microsecs
195+
# SETATTR_OPS: ops
196+
# STATFS_LATENCY: microsecs
197+
# STATFS_OPS: ops
198+
# SYMLINK_LATENCY: microsecs
199+
# SYMLINK_OPS: ops
200+
# THROUGHPUT: bytespersec
201+
# UNLINK_LATENCY: microsecs
202+
# UNLINK_OPS: ops
203+
# WRITES: iops
204+
# WRITE_BYTES: bytespersec
205+
# WRITE_DURATION: microsecs
206+
# WRITE_LATENCY: microsecs
207+
WRITE_SIZES: sizes
208+
ops_nfs: # Category
209+
# ACCESS_LATENCY: microsecs
210+
# ACCESS_OPS: ops
211+
# COMMIT_LATENCY: microsecs
212+
# COMMIT_OPS: ops
213+
# CREATE_LATENCY: microsecs
214+
# CREATE_OPS: ops
215+
# FSINFO_LATENCY: microsecs
216+
# FSINFO_OPS: ops
217+
# GETATTR_LATENCY: microsecs
218+
# GETATTR_OPS: ops
219+
# LINK_LATENCY: microsecs
220+
# LINK_OPS: ops
221+
# LOOKUP_LATENCY: microsecs
222+
# LOOKUP_OPS: ops
223+
# MKDIR_LATENCY: microsecs
224+
# MKDIR_OPS: ops
225+
# MKNOD_LATENCY: microsecs
226+
# MKNOD_OPS: ops
227+
# OPS: ops
228+
# PATHCONF_LATENCY: microsecs
229+
# PATHCONF_OPS: ops
230+
# READDIR_LATENCY: microsecs
231+
# READDIR_OPS: ops
232+
# READLINK_LATENCY: microsecs
233+
# READLINK_OPS: ops
234+
# READS: iops
235+
# READ_BYTES: bytespersec
236+
# READ_DURATION: microsecs
237+
# READ_LATENCY: microsecs
238+
# READ_SIZES: sizes
239+
# REMOVE_LATENCY: microsecs
240+
# REMOVE_OPS: ops
241+
# RENAME_LATENCY: microsecs
242+
# RENAME_OPS: ops
243+
# SETATTR_LATENCY: microsecs
244+
# SETATTR_OPS: ops
245+
# STATFS_LATENCY: microsecs
246+
# STATFS_OPS: ops
247+
# SYMLINK_LATENCY: microsecs
248+
# SYMLINK_OPS: ops
249+
# THROUGHPUT: bytespersec
250+
# WRITES: iops
251+
# WRITE_BYTES: bytespersec
252+
# WRITE_DURATION: microsecs
253+
# WRITE_LATENCY: microsecs
254+
# WRITE_SIZES: sizes
255+
ssd: # Category
256+
# DRIVE_READ_LATENCY: microsecs
257+
# DRIVE_READ_OPS: ops
258+
# DRIVE_WRITE_LATENCY: microsecs
259+
# DRIVE_WRITE_OPS: ops
260+
# SSD_BLOCKS_READ: count
261+
# SSD_BLOCKS_WRITTEN: count
262+
# SSD_MEDIA_ERRORS: count
263+
# SSD_NON_MEDIA_ERRORS: count
264+
# SSD_READ_ERRORS: count
265+
# SSD_READ_LATENCY: microsecs
266+
# SSD_READ_REQS: iops
267+
# SSD_WRITES: iops
268+
# SSD_WRITE_ERRORS: count
269+
# SSD_WRITE_LATENCY: microsecs
270+
# network:
271+
# PUMPS_TXQ_FULL: times

0 commit comments

Comments
 (0)