Make Redis connection resilient and add lifecycle logging

A transient Redis outage made ioredis throw MaxRetriesPerRequestError
(default 20 retries) and crash the process, restart-looping the
container. Provide a bull createClient factory with maxRetriesPerRequest
null and a capped reconnect backoff so commands wait for reconnection
instead of failing, and log each client's connection lifecycle (target,
connect, ready, error, reconnect) for visibility in the docker network.
Add process-level uncaughtException/unhandledRejection guards as a
safety net, and declare ioredis as a direct dependency.

Author: oskarrisberg

index.js

@@ -1,6 +1,7 @@
 require( 'dotenv' ).config();
 
 const Arena = require( 'bull-arena' );
+const Redis = require( 'ioredis' );
 const express = require( 'express' );
 const basicAuth = require( 'express-basic-auth' );
 
@@ -9,6 +10,80 @@ const router = express.Router();
 
 const DEFAULT_LISTEN_PORT =  4000;
 
+// A transient Redis outage (Heroku failover, network blip) makes ioredis/bull
+// emit errors that, left unhandled, crash the process and put the container in
+// a restart loop. This is a read-only monitoring UI, so log and keep running —
+// ioredis will reconnect on its own (see retryStrategy below).
+process.on( 'uncaughtException', ( error ) => {
+    console.error( 'Uncaught exception (continuing):', error );
+} );
+process.on( 'unhandledRejection', ( error ) => {
+    console.error( 'Unhandled rejection (continuing):', error );
+} );
+
+// Build ioredis connection options from a redis:// URL, adding resilience
+// settings. We parse the URL rather than passing the string through so we can
+// inject these options; the derived fields match what ioredis would parse from
+// the same URL, so AUTH/TLS behavior is unchanged.
+const buildRedisOptions = function buildRedisOptions ( redisUrl ) {
+    const parsed = new URL( redisUrl );
+
+    const options = {
+        host: parsed.hostname,
+        port: Number( parsed.port ) || 6379,
+        // Keep retrying commands across reconnects instead of throwing
+        // MaxRetriesPerRequestError after ioredis' default of 20 retries.
+        maxRetriesPerRequest: null,
+        // Reconnect with a capped backoff after the connection drops.
+        retryStrategy: ( times ) => Math.min( times * 200, 5000 ),
+    };
+
+    if ( parsed.username ) {
+        options.username = parsed.username;
+    }
+
+    if ( parsed.password ) {
+        options.password = parsed.password;
+    }
+
+    if ( parsed.protocol === 'rediss:' ) {
+        options.tls = {};
+    }
+
+    return options;
+};
+
+// Returns a bull `createClient(type)` factory so we own the ioredis clients and
+// can log their connection lifecycle. Bull creates a few client types per queue
+// ('client', 'subscriber', and 'bclient' for blocking ops); we label logs by
+// type so it's clear which connection is doing what.
+const createRedisClient = function createRedisClient ( redisUrl ) {
+    const baseOptions = buildRedisOptions( redisUrl );
+
+    console.log( `[redis] target ${ baseOptions.host }:${ baseOptions.port }${ baseOptions.tls ? ' (tls)' : '' }` );
+
+    return ( type ) => {
+        const options = Object.assign( {}, baseOptions );
+
+        // bull requires these for its blocking and subscriber clients.
+        if ( type === 'bclient' || type === 'subscriber' ) {
+            options.enableReadyCheck = false;
+            options.maxRetriesPerRequest = null;
+        }
+
+        const client = new Redis( options );
+
+        client.on( 'connect', () => console.log( `[redis:${ type }] connecting` ) );
+        client.on( 'ready', () => console.log( `[redis:${ type }] ready` ) );
+        client.on( 'error', ( error ) => console.error( `[redis:${ type }] error: ${ error.message }` ) );
+        client.on( 'close', () => console.warn( `[redis:${ type }] connection closed` ) );
+        client.on( 'reconnecting', ( delay ) => console.warn( `[redis:${ type }] reconnecting in ${ delay }ms` ) );
+        client.on( 'end', () => console.warn( `[redis:${ type }] connection ended; no further reconnects` ) );
+
+        return client;
+    };
+};
+
 if ( !process.env.QUEUES ) {
     throw new Error( 'Unable to load queues' );
 }
@@ -34,9 +109,12 @@ const getUnauthorizedResponse = function getUnauthorizedResponse ( request ) {
     return 'No credentials provided';
 };
 
-if ( process.env.REDIS_URL ) {
-    for ( const queue of queues ) {
-        queue.url = process.env.REDIS_URL;
+for ( const queue of queues ) {
+    const redisUrl = process.env.REDIS_URL || queue.url;
+
+    if ( redisUrl ) {
+        queue.createClient = createRedisClient( redisUrl );
+        delete queue.url;
     }
 }
 

package-lock.json

@@ -15,6 +15,7 @@
     "bull-arena": "^2.6.4",
     "dotenv": "^16.0.1",
     "express": "^4.17.1",
-    "express-basic-auth": "^1.2.0"
+    "express-basic-auth": "^1.2.0",
+    "ioredis": "^4.28.5"
   }
 }