apache无法启动报错No space left on device
故障现象:apache无法启动ipcs信号量很多
# service httpd start
Starting httpd : [Failed]
当达到极限信号量：
# ipcs -s | wc -l
32004
而：
# cat /proc/sys/kernel/msgmni
32000
# cat /proc/sys/kernel/sem
250 24000 32 1024
原因
这些错误意味着有缺乏在系统内处理的通信资源，例如信号量或共享存储器段。
解析度
登录到通过服务器的SSH。
增加的限制sysctl 配置：
# mkdir -p /etc/sysctl.d/
# touch /etc/sysctl.d/99-zz_plesk_semaphores.conf
# chmod 755 /etc/sysctl.d/ /etc/sysctl.d/99-zz_plesk_semaphores.conf
# chown root:root /etc/sysctl.d/ /etc/sysctl.d/99-zz_plesk_semaphores.conf
这样的 /etc/sysctl.d/99-zz_plesk_semaphores.conf配置文件必须包含：
kernel.msgmni = 64000
kernel.sem = 250 256000 32 1024
负荷了新的sysctl设置 sysctl
# sysctl -p
检查哪些用户用尽信号灯：
# ipcs -s | awk '{print $3}' | uniq -c
4 root
33 httpd
234 somesoftware
阿帕奇保持清洁信号灯：
# for i in `ipcs -s | awk '/httpd/ {print $2}'`; do (ipcrm -s $i); done
在某些情况下，其他软件滥用严重限制信号，禁用该软件，清理作进一步调查其信号量和接触软件供应商：
# for i in `ipcs -s | awk '/httpd/ {print $2}'`; do (ipcrm -s $i); done
for i in `ipcs -s | awk '/httpd/{ next; } {print $2}'`; do (ipcrm -s $i); done
执行清理后问题依旧，反复执行也无效
[root@server:~]# ipcs -s
------ Semaphore Arrays --------
key semid owner perms nsems
0x00000000 0 root 600 1
0x00000000 65537 root 600 1
0x00000000 131074 apache 600 1
0x00000000 163843 apache 600 1
0x7a004a4e 196612 zabbix 600 13
0x00000000 229381 apache 600 1
0x00000000 262150 apache 600 1
0x00000000 294919 apache 600 1
0x00000000 327688 apache 600 1
[root@server:~]# ipcs -s | grep apache | perl -e 'while (<STDIN>) { @a=split(/\s+/); print `ipcrm sem $a[1]`}'
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
for i in `ipcs -s | awk '/httpd/{ next; } {print $2}'`; do (ipcrm -s $i); done
[root@server:~]# ipcs -s
------ Semaphore Arrays --------
key semid owner perms nsems
0x00000000 0 root 600 1
0x00000000 65537 root 600 1
0x7a004a4e 196612 zabbix 600 13
日志：
[root@server:~]# tail -f /var/log/messages
Dec 4 04:47:36 server nrpe[36326]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:47:36 server nrpe[36328]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:47:36 server nrpe[36330]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:47:36 server nrpe[36332]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:47:36 server nrpe[36334]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:48:07 server nrpe[36711]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:48:07 server nrpe[36713]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:48:07 server nrpe[36715]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:49:04 server nrpe[37324]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:49:04 server nrpe[37333]: Host 192.168.254.63 is not allowed to talk to us!
# 原因是因为切分、监控等程序造成的，全部kill就可以了
[root@server:~]# ps -ef|grep apache|grep -v grep
root 18475 18474 0 16:50 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 18477 18475 0 16:50 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 26621 26618 0 17:00 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 26623 26621 0 17:00 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 34354 34352 0 17:10 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 34355 34354 0 17:10 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 41177 41175 0 17:20 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 41178 41177 0 17:20 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 47964 47962 0 17:30 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 47965 47964 0 17:30 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 49905 49904 0 14:20 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 49907 49905 0 14:20 ? 00:00:03 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 54555 54554 0 17:40 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 54557 54555 0 17:40 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 56774 56773 0 14:30 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 56775 56774 0 14:30 ? 00:00:03 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 61962 61960 0 17:50 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 61963 61962 0 17:50 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 70159 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/error/error.log.%Y-%m-%d 86400
root 70160 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/error/i-job.chinasoft.com_error.log.%Y-%m-%d 86400
root 70161 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/error/support.chinasoft.com.old_error.log.%Y-%m-%d 86400
root 70162 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/error/dlcbs.chinasoft.com_error.log.%Y-%m-%d 86400
root 70163 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/access.log.%Y-%m-%d 86400
root 70164 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/i-job.chinasoft.com_access.log.%Y-%m-%d 86400
root 70165 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/dlcbs.chinasoft.com_access.log.%Y-%m-%d 86400
apache 70166 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70167 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70168 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70169 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70170 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70171 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70172 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70173 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70174 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70175 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
root 84551 84548 0 06:00 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 84553 84551 0 06:00 ? 00:00:23 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 97069 97068 0 15:30 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 97071 97069 0 15:30 ? 00:00:01 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 109607 109605 0 15:50 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 109608 109607 0 15:50 ? 00:00:01 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 115643 115642 0 16:00 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 115645 115643 0 16:00 ? 00:00:01 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
[root@server:~]# ps -ef|grep apache|grep -v grep|awk '{print $2}'|xargs kill -9
[root:~]# ps -ef|grep apache
root 70590 70566 0 17:58 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/access.log.%Y-%m-%d 86400
root 70591 70566 0 17:58 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/i-job.chinasoft.com_access.log.%Y-%m-%d 86400
root 70592 70566 0 17:58 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/dlcbs.chinasoft.com_access.log.%Y-%m-%d 86400
apache 70608 70566 0 17:58 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70609 70566 0 17:58 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70610 70566 0 17:58 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
root 70612 35669 0 17:58 pts/0 00:00:00 grep apache
[root:~]# killall httpd
[root:~]# ipcs -s
------ Semaphore Arrays --------
key semid owner perms nsems
# 删除信号
for i in `ipcs -s | awk '/httpd/{ next; } {print $2}'`; do (ipcrm -s $i); done