🎨 Use dedicated socket for telemetry logging

can't use syslog as systemd/journal intercepts

Author: akrherz

CHANGELOG.md

@@ -11,7 +11,8 @@ All notable changes to this library are documented in this file.
 - Disable IEMAccess write of wind information from the DSM due to inprecision
   of reported units in MPH.
 - Return signature of `str2multipolygon` changed to include a list of errors.
-- `webutil.write_telemetry` now writes to syslog (local1.info) with
+- `webutil.write_telemetry` now writes to dedicated socket
+  `/run/rsyslog/telemetry.sock` with
   `webutil.TELEMETRY_PREFIX` and a JSON payload string.
 
 ### New Features

src/pyiem/webutil.py

@@ -7,9 +7,9 @@
 import os
 import random
 import re
+import socket
 import string
 import sys
-import syslog
 import traceback
 import warnings
 from collections import namedtuple
@@ -77,6 +77,7 @@
     ],
 )
 TELEMETRY_PREFIX = "Telemetry "
+TELEMETRY_SOCKET = "/run/rsyslog/telemetry.sock"
 XSS_SENTINEL = "XSS"
 MEMCACHED_HIT = "_mhit"
 
@@ -205,19 +206,21 @@ def model_to_rst(model: BaseModel) -> str:
 
 
 def write_telemetry(data: TELEMETRY) -> bool:
-    """Write telemetry to syslog."""
+    """Best-effort telemetry write."""
+    # 141 is local1.notice and is critical to make this work.
+    # The TELEMETRY_PREFIX becomes the syslog tag
+    payload = (
+        "<141>"
+        + TELEMETRY_PREFIX
+        + json.dumps(data._asdict(), separators=(",", ":"), sort_keys=True)
+    ).encode("utf-8")
+    # We need to avoid syslog as systemd/journal will intercept this and
+    # fill logs quickly.
     try:
-        syslog.syslog(
-            syslog.LOG_LOCAL1 | syslog.LOG_INFO,
-            TELEMETRY_PREFIX
-            + json.dumps(
-                data._asdict(),
-                separators=(",", ":"),
-                sort_keys=True,
-            ),
-        )
-    except Exception as exp:
-        LOG.info("write_telemetry failed: %s", exp, exc_info=True)
+        with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
+            sock.setblocking(False)
+            sock.sendto(payload, TELEMETRY_SOCKET)
+    except (BlockingIOError, OSError):
         return False
     return True
 

tests/test_webutil.py

@@ -20,6 +20,7 @@
 from pyiem.reference import ISO8601
 from pyiem.webutil import (
     TELEMETRY,
+    TELEMETRY_SOCKET,
     CGIModel,
     ListOrCSVType,
     _is_xss_payload,
@@ -552,23 +553,39 @@ def test_disable_parse_times():
 
 
 def test_add_telemetry():
-    """Test adding something to the queue."""
-    assert write_telemetry(
-        TELEMETRY(
-            timing=1,
-            status_code=200,
-            client_addr=None,
-            app="test",
-            request_uri="",
-            vhost="",
-            valid=datetime.now().strftime(ISO8601),
+    """Test adding something to the telemetry socket."""
+    data = TELEMETRY(
+        timing=1,
+        status_code=200,
+        client_addr=None,
+        app="test",
+        request_uri="",
+        vhost="",
+        valid=datetime.now().strftime(ISO8601),
+    )
+    socket_mock = mock.MagicMock()
+    cm_mock = mock.MagicMock()
+    cm_mock.__enter__.return_value = socket_mock
+    cm_mock.__exit__.return_value = False
+    with mock.patch("pyiem.webutil.socket.socket", return_value=cm_mock):
+        assert write_telemetry(data)
+
+    socket_mock.setblocking.assert_called_once_with(False)
+    socket_mock.sendto.assert_called_once_with(
+        b"<141>Telemetry "
+        + (
+            b'{"app":"test","client_addr":null,"request_uri":"",'
+            b'"status_code":200,"timing":1,"valid":"'
+            + data.valid.encode("utf-8")
+            + b'","vhost":""}'
         ),
+        TELEMETRY_SOCKET,
     )
 
 
 def test_add_telemetry_failure_is_swallowed():
     """Test telemetry failures stay contained inside write_telemetry."""
-    with mock.patch("pyiem.webutil.syslog.syslog", side_effect=RuntimeError()):
+    with mock.patch("pyiem.webutil.socket.socket", side_effect=OSError()):
         assert not write_telemetry(
             TELEMETRY(
                 timing=1,