Hi guys,
We are quite desperate, we have been running Moodle since 2004 without problems, upgrading all the way from 1.4 to 3.0, but about a month ago the site is running extremely low becoming useless.
The symptoms are:
When concurrent users grow over 200 (aprox) the CPU usage for php-fpm jumps to 100% with 60% system (according to NewRelic) which corresponds to Hardware Interrupts according to "top". The pages take minutes to respond.
All I've read about cpu in hi is that it should be a hardware problem, but giving that we are on top of a vmware cloud, it is pretty much unlikely.
Our hardware is:
Moodle webapp running nginx + php-fpm with 12 vCPUs and 8Gb ram. And 2Tb disk (moodledata is quite large).
MariaDB database on an 8vCPU with 32Gb ram 100Gb data disk.
Currently the database is no stressed at all and the webapp is the one suffering the load. Trying to speed up the webapp server we have tried:
- Sessions on a separate memcached server
- Sessions on a tempfs folder
- Caching dynamic content on nginx
- php-fpm sessions on a memcached server in the same server
Most of the attempts improve a little bit things but nothing more.
During a peak moment our stats are:
Google Analytics: 244 concurrent users.
846 connections to nginx (netstat -aln | grep ":80" | grep ESTABLISHED | wc -l)
59 connections to the Database server (same with 3306)
top:
Our configuration for nginx:
user www-data;
worker_processes auto;
pid /run/nginx.pid;
events {
worker_connections 1024;
multi_accept on;
}
http {
sendfile on;
tcp_nodelay on;
keepalive_timeout 600;
keepalive_requests 10000;
types_hash_max_size 2048;
client_max_body_size 256M;
open_file_cache max=1000 inactive=20s;
open_file_cache_valid 30s;
open_file_cache_min_uses 2;
open_file_cache_errors on;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
gzip on;
gzip_disable "msie6";
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
nginx site config:
fastcgi_cache_path /etc/nginx/cache levels=1:2 keys_zone=moodle:100m inactive=1d max_size=100m;
fastcgi_cache_key "$scheme$request_method$host$request_uri";
#limit_req_zone $cookie_MoodleSessionaplaplac zone=userlimit:128m rate=2r/s;
#limit_req_status 503;
#limit_req zone=userlimit burst=60 nodelay;
server {
listen 80 default_server;
listen [::]:80 default_server ipv6only=on;
listen 443 ssl;
root /data/moodle/moodle/;
index index.php;
# Make site accessible from http://localhost/
server_name webcursos.uai.cl;
ssl_certificate /data/etc/httpd/conf/webcursos/webcursos.uai.cl.crt;
ssl_certificate_key /data/etc/httpd/conf/webcursos/webcursos-nk.uai.cl.key;
ssl_session_timeout 5m;
ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers "HIGH:!aNULL:!MD5 or HIGH:!aNULL:!MD5:!3DES";
ssl_prefer_server_ciphers on;
location /stts {
stub_status;
}
location / {
try_files $uri $uri/index.php;
}
location /dataroot/ {
internal;
alias /data/data/moodledata/; # ensure the path ends with /
}
#location ~ ^/theme/.*\.(jpg|jpeg|gif|png|css|js|ico|xml)$ {
# access_log off;
# log_not_found off;
# expires 360d;
#}
# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
#
location ~ [^/]\.php(/|$) {
fastcgi_split_path_info ^(.+\.php)(/.+)$;
#location ~ ^/lib/javascript.php/.*\.js(/|$) {
#fastcgi_pass unix:/var/run/php5-fpm.sock;
#fastcgi_cache moodle;
#fastcgi_cache_valid 200 1d;
#expires 1d;
#}
#location ~ ^/.*/favicon\.ico(/|$) {
# fastcgi_pass unix:/var/run/php5-fpm.sock;
# fastcgi_cache moodle;
# fastcgi_cache_valid 200 1d;
# expires 1d;
#}
location ~ ^/theme/.*\.php(/|$) {
fastcgi_pass unix:/var/run/php5-fpm.sock;
fastcgi_cache moodle;
fastcgi_cache_valid 200 1d;
expires 1d;
}
location ~ ^/.*/theme_essential/.*\.css(/|$) {
fastcgi_pass unix:/var/run/php5-fpm.sock;
fastcgi_cache moodle;
fastcgi_cache_valid 200 1d;
expires 1d;
}
fastcgi_pass unix:/var/run/php5-fpm.sock;
fastcgi_buffers 8 16k;
fastcgi_buffer_size 32k;
fastcgi_connect_timeout 600;
fastcgi_send_timeout 600;
fastcgi_read_timeout 600;
fastcgi_index index.php;
include fastcgi_params;
fastcgi_param PATH_INFO $fastcgi_path_info;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
}
location ~ ^/(stts-fpm|ping)$ {
access_log off;
allow 127.0.0.1;
allow 10.50.200.37;
deny all;
include fastcgi_params;
fastcgi_pass unix:/var/run/php5-fpm.sock;
}
#location ~* \.(jpg|jpeg|gif|png|css|js|ico|xml)$ {
# access_log off;
# log_not_found off;
# expires 360d;
#}
}
Our php.ini:
[PHP]
engine = On
short_open_tag = Off
asp_tags = Off
precision = 14
output_buffering = 4096
zlib.output_compression = Off
implicit_flush = Off
unserialize_callback_func =
serialize_precision = 17
disable_functions = pcntl_alarm,pcntl_fork,pcntl_waitpid,pcntl_wait,pcntl_wifexited,pcntl_wifstopped,pcntl_wifsignaled,pcntl_wexitstatus,pcntl_wtermsig,pcntl_wstopsig,pcntl_signal,pcntl_signal_dispatch,pcntl_get_last_error,pcntl_strerror,pcntl_sigprocmask,pcntl_sigwaitinfo,pcntl_sigtimedwait,pcntl_exec,pcntl_getpriority,pcntl_setpriority,
disable_classes =
zend.enable_gc = On
expose_php = On
max_execution_time = 600
max_input_time = 60
memory_limit = 128M
error_reporting = E_ALL & ~E_DEPRECATED & ~E_STRICT
display_errors = Off
display_startup_errors = Off
log_errors = On
log_errors_max_len = 1024
ignore_repeated_errors = Off
ignore_repeated_source = Off
report_memleaks = On
track_errors = Off
html_errors = On
variables_order = "GPCS"
request_order = "GP"
register_argc_argv = Off
auto_globals_jit = On
post_max_size = 256M
auto_prepend_file =
auto_append_file =
default_mimetype = "text/html"
doc_root =
user_dir =
enable_dl = Off
cgi.fix_pathinfo=0
file_uploads = On
upload_max_filesize = 256M
max_file_uploads = 20
allow_url_fopen = On
allow_url_include = Off
default_socket_timeout = 600
[MySQLi]
mysqli.max_persistent = -1
mysqli.allow_persistent = On
mysqli.max_links = -1
mysqli.cache_size = 2000
mysqli.default_port = 3306
mysqli.default_socket =
mysqli.default_host =
mysqli.default_user =
mysqli.default_pw =
mysqli.reconnect = Off
[Session]
session.save_handler = memcached
session.save_path = "127.0.0.1:11211"
session.use_strict_mode = 0
session.use_cookies = 1
session.use_only_cookies = 1
session.name = PHPSESSID
session.auto_start = 0
session.cookie_lifetime = 0
session.cookie_path = /
session.cookie_domain =
session.cookie_httponly =
session.serialize_handler = php
session.gc_probability = 0
session.gc_divisor = 1000
session.gc_maxlifetime = 1440
session.bug_compat_42 = Off
session.bug_compat_warn = Off
session.referer_check =
session.cache_limiter = nocache
session.cache_expire = 180
session.use_trans_sid = 0
session.hash_function = 0
session.hash_bits_per_character = 5
url_rewriter.tags = "a=href,area=href,frame=src,input=src,form=fakeentry"
[opcache]
opcache.enable=1
opcache.enable_cli=1
opcache.memory_consumption=256
opcache.interned_strings_buffer=16
opcache.max_accelerated_files=8000
opcache.use_cwd=1
opcache.validate_timestamps=0
opcache.save_comments=1
opcache.load_comments=1
opcache.fast_shutdown=1
opcache.enable_file_override=0
php-fpm.conf
[global]
pid = /var/run/php5-fpm.pid
error_log = /var/log/php5-fpm.log
include=/etc/php5/fpm/pool.d/*.conf
pool.d/www.conf
[www]
user = www-data
group = www-data
listen = /var/run/php5-fpm.sock
listen.backlog = 65535
listen.owner = www-data
listen.group = www-data
pm = dynamic
pm.max_children = 50
pm.start_servers = 8
pm.min_spare_servers = 8
pm.max_spare_servers = 8
pm.status_path = /stts-fpm
request_terminate_timeout = 600
rlimit_files = 1024
chdir = /
Does anyone see anything that could help us?
