0

We're currently working on implementing a high-availability solution for our UDP tool, Logstash, by using Nginx as a UDP load balancer. However, I'm facing one problem, In our configuration file, we're trying to route syslog traffic to the backend Logstash servers. However, when one of the Logstash servers goes down, Nginx continues to forward logs to the unavailable server instead of routing all traffic to the available one. We've configured it to use round-robin load balancing, but it doesn't seem to be working as expected.

user  nse;
worker_processes  auto;

#error_log logs/error.log; #error_log logs/error.log notice; #error_log logs/error.log info;

pid /home/nse/etc/nginx/nginx.pid;

events { worker_connections 1024; }

http { include mime.types; default_type application/octet-stream;

#log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
#                  '$status $body_bytes_sent "$http_referer" '
#                  '"$http_user_agent" "$http_x_forwarded_for"';
error_log /home/nse/var/log/nginx/error.log info;
access_log /home/nse/var/log/nginx/access.log;
#access_log  logs/access.log  main;
#proxy_timeout 1s;
#proxy_responses 10;
#proxy_connect_timeout 180s;
#proxy_send_timeout 180s;
#proxy_read_timeout 180s;
sendfile        on;
#tcp_nopush     on;

#keepalive_timeout  0;
keepalive_timeout  65;

#gzip  on;

}

stream { upstream logstash { zone dns_zone 64k; server 10.139.168.12:10514 fail_timeout=30s; server 10.139.168.11:10544 fail_timeout=30s; }

server {

    listen 5144 udp;
    proxy_pass logstash;
    health_check interval=5 passes=2 fails=2 udp;
    #proxy_timeout 1s;
    #proxy_responses 1;
    proxy_buffer_size 16k;
    proxy_responses 60;
}

}

  • 2
    syslog/UDP is a fire and forget protocol. There is no way for a server to signal a problem and there is no way to detect a failed server - maybe except if the system is returning with an ICMP unreachable which might then be filtered away by wrongly configured firewalls. So you need to make sure that "system goes down" can actually be noticed by nginx. – Steffen Ullrich Mar 22 '24 at 08:22
  • In order for nginx UDP health check to fail it needs to receive ICMP Destination Unreachable. – AlexD Mar 22 '24 at 12:34

0 Answers0