Uwsgi - Nginx - Django - 499 and 502 errors fix

On rare occasions, we have been receiving 499 and 502 response codes from our higher environment server. Before that, we have added

max-requests = 500

in UWSGI configuration to reduce the high consumption of physical memory. So we are suspecting add max requests would have caused the error. Loglines recorded in the uwsgi.log file.


10.115.21.68 - - [01/Jan/2020:04:09:50 +0000] "GET / HTTP/1.1" 502 157 "-" "check_http/v2.2.1 (nagios-plugins 2.2.1)" "-"
2020/01/01 04:09:50 [error] 30522#0: *22690 connect() to unix:///tmp/uwsgi.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: 10.115.21.68, server: , request: "GET / HTTP/1.1", upstream: "uwsgi://unix:///tmp/uwsgi.sock:"
10.115.13.75 - - [01/Jan/2020:04:09:48 +0000] "GET / HTTP/1.1" 502 157 "-" "check_http/v2.2.1 (nagios-plugins 2.2.1)" "-"
2020/01/01 04:09:48 [error] 30523#0: *22688 connect() to unix:///tmp/uwsgi.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: 10.115.13.75, server: , request: "GET / HTTP/1.1", upstream: "uwsgi://unix:///tmp/uwsgi.sock:"
10.115.21.68 - - [01/Jan/2020:04:09:06 +0000] "GET / HTTP/1.1" 502 157 "-" "check_http/v2.2.1 (nagios-plugins 2.2.1)" "-"
2020/01/01 04:09:06 [error] 30522#0: *22682 connect() to unix:///tmp/uwsgi.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: 10.115.21.68, server: , request: "GET / HTTP/1.1", upstream: "uwsgi://unix:///tmp/uwsgi.sock:"
10.115.13.75 - - [01/Jan/2020:04:08:49 +0000] "GET / HTTP/1.1" 502 157 "-" "check_http/v2.2.1 (nagios-plugins 2.2.1)" "-"
2020/01/01 04:08:49 [error] 30523#0: *22680 connect() to unix:///tmp/uwsgi.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: 10.115.13.75, server: , request: "GET / HTTP/1.1", upstream: "uwsgi://unix:///tmp/uwsgi.sock:"
10.115.21.68 - - [01/Jan/2020:04:08:23 +0000] "GET / HTTP/1.1" 502 157 "-" "check_http/v2.2.1 (nagios-plugins 2.2.1)" "-"
2020/01/01 04:08:23 [error] 30522#0: *22676 connect() to unix:///tmp/uwsgi.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: 10.115.21.68, server: , request: "GET / HTTP/1.1", upstream: "uwsgi://unix:///tmp/uwsgi.sock:"
10.115.13.75 - - [01/Jan/2020:04:07:51 +0000] "GET / HTTP/1.1" 502 157 "-" "check_http/v2.2.1 (nagios-plugins 2.2.1)" "-"
2020/01/01 04:07:51 [error] 30522#0: *22672 connect() to unix:///tmp/uwsgi.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: 10.115.13.75, server: , request: "GET / HTTP/1.1", upstream: "uwsgi://unix:///tmp/uwsgi.sock:"
10.115.21.68 - - [01/Jan/2020:04:07:49 +0000] "GET / HTTP/1.1" 502 157 "-" "check_http/v2.2.1 (nagios-plugins 2.2.1)" "-"
2020/01/01 04:07:49 [error] 30522#0: *22670 connect() to unix:///tmp/uwsgi.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: 10.115.21.68, server: , request: "GET / HTTP/1.1", upstream: "uwsgi://unix:///tmp/uwsgi.sock:"
10.115.13.75 - - [01/Jan/2020:04:07:07 +0000] "GET / HTTP/1.1" 502 157 "-" "check_http/v2.2.1 (nagios-plugins 2.2.1)" "-"
2020/01/01 04:07:07 [error] 30522#0: *22664 connect() to unix:///tmp/uwsgi.sock failed (11: Resource temporarily unavailable) while connecting to upstream, client: 10.115.13.75, server: , request: "GET / HTTP/1.1", upstream: "uwsgi://unix:///tmp/uwsgi.sock:"
10.115.21.68 - - [01/Jan/2020:02:59:47 +0000] "GET / HTTP/1.1" 499 0 "-" "check_http/v2.2.1 (nagios-plugins 2.2.1)" "-"
10.115.13.75 - - [01/Jan/2020:02:59:09 +0000] "GET / HTTP/1.1" 499 0 "-" "check_http/v2.2.1 (nagios-plugins 2.2.1)" "-"
10.115.21.68 - - [01/Jan/2020:02:59:08 +0000] "GET / HTTP/1.1" 499 0 "-" "check_http/v2.2.1 (nagios-plugins 2.2.1)" "-"

After some analyses on the internet, we came to a conclusion the issues have been raised due to different timeout periods in AWS Elastic Load Balancer (Idle 60 seconds by default), Nginx and UWSGI. Finally, we fixed error in two places 1. In UWSGI


[uwsgi]
socket = /tmp/uwsgi.sock
stats = /tmp/stats.sock
max-requests = 1000
enable-threads = true
single-interpreter = true
processes = 4
threads = 2
uid = uwsgi
gid = uwsgi
harikari = 120
master = true
harikari-verbose = true
buffer-size = 8192
logformat = ctime_start_request="%(ctime)", request_uri="%(uri)", request_method="%(method)", remote_user="%(user)", remote_addr="%(addr)", http_host="%(host)", server_protocol="%(proto)", http_user_agent="%(uagent)", http_referer="%(referer)", http_response_code="%(status)", http_response_microsecs="%(micros)", http_response_milisecs="%(msecs)", http_response_microsecs="%(secs)", timestamp_start_request="%(time)", time_unix_format="%(epoch)", response_size="%(size)", response_headers_size="%(hsize)", response_body_size="%(rsize)", request_content_body_size="%(cl)", pid="%(pid)", wid="%(wid)", core="%(core)", virtual_memory_usage_mb="%(vszM)", rss_memory_usage_mb="%(rssM)", internal_request_uwsgi_packet_size="%(pktsize)", num_read_error="%(rerr)", num_write_error="%(werr)", num_io_error="%(ioerr)", num_generated_response_headers="%(headers)"
daemonize = /var/log/uwsgi/uwsgi.log
die-on-term = true
for-readline = /srv/env_vars
  env = %(_)
endfor =

We have set harikari seconds and uwsgi_send/read_timeout is equal to avoid the timeout error between UWSGI and Nginx.

2. In Nginx, django.conf - Added

header_timeout = 75s

to be same as

keepalive_timeout

in nginx.conf


upstream django {
  server unix:///tmp/uwsgi.sock;
}

server {
  listen       *:443 ssl;
  ssl on;
  ssl_certificate           /srv/certs/main.crt;
  ssl_certificate_key       /srv/certs/main.key;
  ssl_session_cache         shared:SSL:50m;
  ssl_session_timeout       10m;
  ssl_prefer_server_ciphers on;

  # The domain name it will serve for
  charset     utf-8;

  #Max upload size
  client_max_body_size 75M;   # adjust to taste

  # Timeout should be equal to ngix.conf -> time keepalive_timeout 75;
  client_header_timeout 75s;

  # Django media
  location /media  {
        alias /var/www/media;      # your Django project's media files
  }

  location /static {
        alias /var/www/static;     # your Django project's static files
    }

  # Finally, send all non-media requests to the Django server.
  location / {
    uwsgi_pass  django;
   # when a client closes the connection then keep the channel to uwsgi open. Otherwise uwsgi throws an IOError
    uwsgi_ignore_client_abort on;
    uwsgi_read_timeout 120s;
    uwsgi_send_timeout 120s;
    client_max_body_size 200M;
    # uwsgi_buffering off;
    include     /etc/nginx/uwsgi_params; # or the uwsgi_params you installed manually
    }
}

The above changes in our config helped us to avoid 499 and 5xx errors. References
1. https://sigopt.com/blog/the-case-of-the-mysterious-aws-elb-504-errors/
2. https://medium.com/@gabriel.m.troy/aws-alb-docker-uwsgi-502-bad-gateway-16d0a36f6240
3. https://www.techatbloomberg.com/blog/configuring-uwsgi-production-deployment/

Developer Notes

Search This Blog

Uwsgi - Nginx - Django - 499 and 502 errors fix

Comments

Post a Comment