Failing to get console log: "stream 1 was reset with about ..." #295
Description
We have this issue with our OSS deployment: Azure Container App for web server and another separate Azure Container App for api server. Both in the same Azure Container App Environment.
For task that have "relatively" big console output (~200kb) the Web UI fail to load console log. The error underneath is something along the line of: upstream prematurely closed connection while reading upstream or with curl querying directely: HTTP/2 stream 1 was reset with about 56009 received on total of 87025
It's not an issue between api server and ES server because we can query directely the api server, via curl, without failure.
Clearml server v2.2.0
Slack thread: https://clearml.slack.com/archives/CTK20V944/p1753235440688829
Solution
We had to put proxy_set_header Connection ""; in the location /api section of the web server nginx configuration. Putting that outside of the location /api seems to not have any effect (still failing to load console log)
Example of our nginx -T:
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
# configuration file /etc/nginx/nginx.conf:
user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;
error_log stderr notice;
events {
worker_connections 768;
# multi_accept on;
}
http {
rewrite_log on;
log_format devlogging '$remote_addr - $remote_user [$time_local] "proxy_host" $proxy_host "proxy_add_x_forwarded_for" $proxy_add_x_forwarded_for "upstream_addr" $upstream_addr "proxy_protocol_addr" $proxy_protocol_addr "proxy_protocol_server_addr" $proxy_protocol_server_addr "request_uri" $request_uri "request" $request "http_x_forwarded_for" $http_x_forwarded_for "status" $status "args" $args';
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
types_hash_max_size 2048;
# server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# SSL Settings
##
ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
access_log /dev/stdout devlogging;
##
# Gzip Settings
##
gzip on;
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
# configuration file /etc/nginx/mime.types:
types {
text/html html htm shtml;
text/css css;
text/xml xml;
image/gif gif;
image/jpeg jpeg jpg;
application/javascript js;
application/atom+xml atom;
application/rss+xml rss;
text/mathml mml;
text/plain txt;
text/vnd.sun.j2me.app-descriptor jad;
text/vnd.wap.wml wml;
text/x-component htc;
image/avif avif;
image/png png;
image/svg+xml svg svgz;
image/tiff tif tiff;
image/vnd.wap.wbmp wbmp;
image/webp webp;
image/x-icon ico;
image/x-jng jng;
image/x-ms-bmp bmp;
font/woff woff;
font/woff2 woff2;
application/java-archive jar war ear;
application/json json;
application/mac-binhex40 hqx;
application/msword doc;
application/pdf pdf;
application/postscript ps eps ai;
application/rtf rtf;
application/vnd.apple.mpegurl m3u8;
application/vnd.google-earth.kml+xml kml;
application/vnd.google-earth.kmz kmz;
application/vnd.ms-excel xls;
application/vnd.ms-fontobject eot;
application/vnd.ms-powerpoint ppt;
application/vnd.oasis.opendocument.graphics odg;
application/vnd.oasis.opendocument.presentation odp;
application/vnd.oasis.opendocument.spreadsheet ods;
application/vnd.oasis.opendocument.text odt;
application/vnd.openxmlformats-officedocument.presentationml.presentation pptx;
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx;
application/vnd.openxmlformats-officedocument.wordprocessingml.document docx;
application/vnd.wap.wmlc wmlc;
application/wasm wasm;
application/x-7z-compressed 7z;
application/x-cocoa cco;
application/x-java-archive-diff jardiff;
application/x-java-jnlp-file jnlp;
application/x-makeself run;
application/x-perl pl pm;
application/x-pilot prc pdb;
application/x-rar-compressed rar;
application/x-redhat-package-manager rpm;
application/x-sea sea;
application/x-shockwave-flash swf;
application/x-stuffit sit;
application/x-tcl tcl tk;
application/x-x509-ca-cert der pem crt;
application/x-xpinstall xpi;
application/xhtml+xml xhtml;
application/xspf+xml xspf;
application/zip zip;
application/octet-stream bin exe dll;
application/octet-stream deb;
application/octet-stream dmg;
application/octet-stream iso img;
application/octet-stream msi msp msm;
audio/midi mid midi kar;
audio/mpeg mp3;
audio/ogg ogg;
audio/x-m4a m4a;
audio/x-realaudio ra;
video/3gpp 3gpp 3gp;
video/mp2t ts;
video/mp4 mp4;
video/mpeg mpeg mpg;
video/quicktime mov;
video/webm webm;
video/x-flv flv;
video/x-m4v m4v;
video/x-mng mng;
video/x-ms-asf asx asf;
video/x-ms-wmv wmv;
video/x-msvideo avi;
}
# configuration file /etc/nginx/sites-enabled/default:
server {
listen 80 default_server;
listen [::]:80 default_server;
server_name _;
root /usr/share/nginx/html;
proxy_http_version 1.1;
client_max_body_size 0;
# compression
gzip on;
gzip_comp_level 9;
gzip_http_version 1.0;
gzip_min_length 512;
gzip_proxied expired no-cache no-store private auth;
gzip_types text/plain
text/css
application/json
application/javascript
application/x-javascript
text/xml application/xml
application/xml+rss
text/javascript
application/x-font-ttf
font/woff2
image/svg+xml
image/x-icon;
# Load configuration files for the default server block.
include /etc/nginx/default.d/*.conf;
location / {
try_files $uri $uri/ /index.html;
}
location /version.json {
add_header Cache-Control 'no-cache';
}
location /api {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $proxy_host;
proxy_pass http://ti-dev-aca-03-12-clearml-api/;
rewrite /api/(.*) /$1 break;
proxy_set_header Connection ""; # <=================== Fix here ==============================
}
location /widgets {
alias /usr/share/nginx/widgets;
try_files $uri $uri/ /widgets/index.html;
add_header Content-Security-Policy "frame-ancestors *";
}
error_page 404 /404.html;
location = /40x.html {
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
}
}