SEARU.ORG
当前位置:SEARU.ORG > Linux 新闻 > 正文

【原创】遇到一个 rabbitmqctl 无法退出的问题


—————————-
—————————-
—————————-
—————————-





重大发现:这两天再次深究问题根源,结果又有重大发现,详见文章最后新增部分!!!


—————————-

—————————-
—————————-
—————————-


      这两天又遇到了一个奇怪的问题:简单的说,就是在 vmware

通过 rabbitmq 的 init 脚本无法正常 stop 掉相应的进程(实际上调用的是
rabbitmqctl stop <pidfile>
)。

问题表现

确认 RabbitMQ 的运行状态和对应的 pid 文件内容

[[email protected] ~]# 
[[email protected] ~]# ps aux|grep rabbit
root     11813  0.0  0.0 106368  1004 pts/2    S    19:21   0:00 /bin/sh /etc/init.d/rabbitmq start
root     11815  0.0  0.0 106100  1200 pts/2    S    19:21   0:00 /bin/bash -c ulimit -S -c 0 >/dev/null 2>&1 ; /usr/local/rabbitmq/sbin/rabbitmq-server
root     11817 46.1  0.3 2806172 53252 pts/2   Sl   19:21   0:07 /usr/local/rabbitmq/erts-6.0/bin/beam.smp -W w -K true -A30 -P 1048576 -- -root /usr/local/rabbitmq -progname erl -- -home /root -- -boot /usr/local/rabbitmq/releases/3.3.5/start_clean -pa /usr/local/rabbitmq/sbin/../ebin -noshell -noinput -s rabbit boot -sname [email protected]_1 -boot start_sasl -config /usr/local/rabbitmq/sbin/../etc/rabbitmq/rabbitmq -kernel inet_default_connect_options [{nodelay,true}] -sasl errlog_type error -sasl sasl_error_logger false -rabbit error_logger {file,"/usr/local/rabbitmq/sbin/../var/log/rabbitmq/[email protected]_1.log"} -rabbit sasl_error_logger {file,"/usr/local/rabbitmq/sbin/../var/log/rabbitmq/[email protected]_1-sasl.log"} -rabbit enabled_plugins_file "/usr/local/rabbitmq/sbin/../etc/rabbitmq/enabled_plugins" -rabbit plugins_dir "/usr/local/rabbitmq/sbin/../plugins" -rabbit plugins_expand_dir "/usr/local/rabbitmq/sbin/../var/lib/rabbitmq/mnesia/[email protected]_1-plugins-expand" -os_mon start_cpu_sup false -os_mon start_disksup false -os_mon start_memsup false -mnesia dir "/usr/local/rabbitmq/sbin/../var/lib/rabbitmq/mnesia/[email protected]_1"
root     12534  0.0  0.0 103252   836 pts/2    S+   19:21   0:00 grep rabbit
[[email protected] ~]# 
[[email protected] ~]# 
[[email protected] ~]# cat /var/run/rabbitmq/pid 
11817
[[email protected] ~]#

通过 init 脚本 stop 服务

[[email protected] ~]# /etc/init.d/rabbitmq stop
Stopping rabbitmq-server: /var/run/rabbitmq/pid
Stopping and halting node [email protected]_1 ...

(卡住)

查看日志,发现 RabbitMQ 服务已经执行了停止动作。

[[email protected] ~]# tail -f /usr/local/rabbitmq/sbin/../var/log/rabbitmq/rabbitmq\@rmq_1.log
...
=INFO REPORT==== 14-Mar-2016::19:21:38 ===
Stopping RabbitMQ

=INFO REPORT==== 14-Mar-2016::19:21:38 ===
stopped TCP Listener on [::]:6672

=ERROR REPORT==== 14-Mar-2016::19:21:38 ===
AMQP connection <0.424.0> (running), channel 0 - error:
{amqp_error,connection_forced,
            "broker forced connection closure with reason 'shutdown'",none}

=INFO REPORT==== 14-Mar-2016::19:21:38 ===
Halting Erlang VM

对应代码在 rabbit.erl 中,如下所示





可以看到,最终调用了 init:stop() 进行了停止。


      然而,此时虽然 RabbitMQ 服务进程已经退出,但调用 rabbitmqctl stop <pidfile> 进程却退不出来!

[[email protected] ~]# 
[[email protected] ~]# ps ajxf|grep rabbit
27541 13082 13082 27541 pts/2    13082 S+       0   0:00  |   |   \_ /bin/sh /etc/init.d/rabbitmq stop
13082 13127 13082 27541 pts/2    13082 Sl+      0   0:00  |   |       \_ /usr/local/rabbitmq/erts-6.0/bin/beam.smp -- -root /usr/local/rabbitmq -progname erl -- -home /root -- -boot /usr/local/rabbitmq/releases/3.3.5/start_clean -pa /usr/local/rabbitmq/sbin/../ebin -noshell -noinput -hidden -sname rabbitmqctl13127 -boot start_clean -s rabbit_control_main -nodename [email protected]_1 -extra stop /var/run/rabbitmq/pid
12431 13687 13686 12431 pts/5    13686 S+       0   0:00  |       \_ grep rabbit
[[email protected] ~]#

通过 Ctrl+C 停止调用后,可以看到
rabbitmqctl stop <pidfile>
调用进程才停止。

[[email protected] ~]# /etc/init.d/rabbitmq stop
Stopping rabbitmq-server: /var/run/rabbitmq/pid
Stopping and halting node [email protected]_1 ...

(卡住)

^C
BREAK: (a)bort (c)ontinue (p)roc info (i)nfo (l)oaded
       (v)ersion (k)ill (D)b-tables (d)istribution
remove pid file and dir
rabbitmq-server.
[[email protected] ~]# ^C
[[email protected] ~]# 
[[email protected] ~]# 
[[email protected] ~]# ps ajxf|grep rabbit  
27541  1658  1657 27541 pts/2     1657 S+       0   0:00  |   |   \_ grep rabbit
[[email protected] ~]#  
[[email protected] ~]# 
[[email protected] ~]#  ll /var/run/rabbitmq/pid
ls: cannot access /var/run/rabbitmq/pid: No such file or directory
[[email protected] ~]#


问题分析

既然 RabbitMQ 服务能够正常停止,那么问题就只有一个:为什么
rabbitmqctl stop <pidfile>
会退不出来?让我们
查看下相关代码

在 rabbitmqctl 脚本中

...
exec ${ERL_DIR}erl \
    -pa "${RABBITMQ_HOME}/ebin" \
    -noinput \
    -hidden \
    ${RABBITMQ_CTL_ERL_ARGS} \
    -sname rabbitmqctl$$ \
    -boot "${CLEAN_BOOT_FILE}" \
    -s rabbit_control_main \        # 入口函数
    -nodename $RABBITMQ_NODENAME \
    -extra "[email protected]"

在 rabbit_control_main.erl 中

...
-define(EXTERNAL_CHECK_INTERVAL, 1000).
...
start() ->
    ...
    {Command, Opts, Args} =
        case parse_arguments(init:get_plain_arguments(), NodeStr) of
            {ok, Res}  -> Res;
            no_command -> print_error("could not recognise command", []),
                          usage()
        end,
...
    %% The reason we don't use a try/catch here is that rpc:call turns
    %% thrown errors into normal return values
    case catch action(Command, Node, Args, Opts, Inform) of
        ok ->
            case Quiet of
                true  -> ok;
                false -> io:format("...done.~n")
            end,
            rabbit_misc:quit(0);
        {ok, Info} ->
            case Quiet of
                true  -> ok;
                false -> io:format("...done (~p).~n", [Info])
            end,
            rabbit_misc:quit(0);
        ...
    end.

对应到实际情况,我们的 Command 为 stop ,所以会调用

...
action(stop, Node, Args, _Opts, Inform) ->
    Inform("Stopping and halting node ~p", [Node]),
    Res = call(Node, {rabbit, stop_and_halt, []}),
    case {Res, Args} of
        {ok, [PidFile]} -> wait_for_process_death(              %% 等待指定 pid 的进程退出
                             read_pid_file(PidFile, false));    %% 读取 pid 文件的内容
        {ok, [_, _| _]} -> exit({badarg, Args});
        _               -> ok
    end,
    Res;
...
wait_for_process_death(Pid) ->
    case process_up(Pid) of
        true  -> timer:sleep(?EXTERNAL_CHECK_INTERVAL),   %% 只要认为指定 pid 的进程未退出,则每隔 1s 检查一次
                 wait_for_process_death(Pid);
        false -> ok
    end.
...
% Test using some OS clunkiness since we shouldn't trust
% rpc:call(os, getpid, []) at this point
process_up(Pid) ->
    with_os([{unix, fun () ->
                            run_ps(Pid) =:= 0    %% Linux 中会执行这里。从上面的判定知道,只有不等于 0 时,才认为目标进程已经退出
                    end},
             {win32, fun () ->
                             Cmd = "tasklist /nh /fi \"pid eq " ++ Pid ++ "\" ",
                             Res = rabbit_misc:os_cmd(Cmd ++ "2>&1"),
                             case re:run(Res, "erl\\.exe", [{capture, none}]) of
                                 match -> true;
                                 _     -> false
                             end
                     end}]).

with_os(Handlers) ->
    {OsFamily, _} = os:type(),
    case proplists:get_value(OsFamily, Handlers) of
        undefined -> throw({unsupported_os, OsFamily});
        Handler   -> Handler()
    end.

run_ps(Pid) ->
    Port = erlang:open_port({spawn, "ps -p " ++ Pid},            %% 这里就是整个问题的关键了
                            [exit_status, {line, 16384},
                             use_stdio, stderr_to_stdout]),
    exit_loop(Port).

exit_loop(Port) ->
    receive
        {Port, {exit_status, Rc}} -> Rc;                 %% 对应上面,只有 Rc 不等于 0 时,才会认为目标进程已退出
        {Port, _}                 -> exit_loop(Port)
    end.

知道了代码执行流程,简单添加一些关键打印,重新测试如下

问题复现

存在问题的环境

[[email protected] ~]# 
[[email protected] ~]# ps aux|grep rabbit
root     11813  0.0  0.0 106368  1004 pts/2    S    19:21   0:00 /bin/sh /etc/init.d/rabbitmq start
root     11815  0.0  0.0 106100  1200 pts/2    S    19:21   0:00 /bin/bash -c ulimit -S -c 0 >/dev/null 2>&1 ; /usr/local/rabbitmq/sbin/rabbitmq-server
root     11817 46.1  0.3 2806172 53252 pts/2   Sl   19:21   0:07 /usr/local/rabbitmq/erts-6.0/bin/beam.smp -W w -K true -A30 -P 1048576 -- -root /usr/local/rabbitmq -progname erl -- -home /root -- -boot /usr/local/rabbitmq/releases/3.3.5/start_clean -pa /usr/local/rabbitmq/sbin/../ebin -noshell -noinput -s rabbit boot -sname [email protected]_1 -boot start_sasl -config /usr/local/rabbitmq/sbin/../etc/rabbitmq/rabbitmq -kernel inet_default_connect_options [{nodelay,true}] -sasl errlog_type error -sasl sasl_error_logger false -rabbit error_logger {file,"/usr/local/rabbitmq/sbin/../var/log/rabbitmq/[email protected]_1.log"} -rabbit sasl_error_logger {file,"/usr/local/rabbitmq/sbin/../var/log/rabbitmq/[email protected]_1-sasl.log"} -rabbit enabled_plugins_file "/usr/local/rabbitmq/sbin/../etc/rabbitmq/enabled_plugins" -rabbit plugins_dir "/usr/local/rabbitmq/sbin/../plugins" -rabbit plugins_expand_dir "/usr/local/rabbitmq/sbin/../var/lib/rabbitmq/mnesia/[email protected]_1-plugins-expand" -os_mon start_cpu_sup false -os_mon start_disksup false -os_mon start_memsup false -mnesia dir "/usr/local/rabbitmq/sbin/../var/lib/rabbitmq/mnesia/[email protected]_1"
root     12534  0.0  0.0 103252   836 pts/2    S+   19:21   0:00 grep rabbit
[[email protected] ~]# 
[[email protected] ~]# 
[[email protected] ~]# cat /var/run/rabbitmq/pid 
11817
[[email protected] ~]# 
[[email protected] ~]# 
[[email protected] ~]# /etc/init.d/rabbitmq stop
Stopping rabbitmq-server: /var/run/rabbitmq/pid
Stopping and halting node [email protected]_1 ...
Args = ["/var/run/rabbitmq/pid"]
 ...
action_stop => Res = ok   Args = ["/var/run/rabbitmq/pid"]
 ...
read_pid_file => PidFile : "/var/run/rabbitmq/pid"   Wait = false
wait_for_process_death Pid = "11817"
run_ps => erlang:open_port -> ps -p "11817"
exit_loop => Data = {data,{eol,"  PID TTY          TIME CMD"}}
exit_loop => Data = {data,{eol,"11817 pts/2    00:00:08 beam.smp"}}
exit_loop => Exit = {exit_status,0}
wait_for_process_death Pid = "11817"
run_ps => erlang:open_port -> ps -p "11817"
exit_loop => Data = {data,{eol,"  PID TTY          TIME CMD"}}
exit_loop => Data = {data,{eol,"11817 pts/2    00:00:08 beam.smp"}}
exit_loop => Exit = {exit_status,0}
wait_for_process_death Pid = "11817"
run_ps => erlang:open_port -> ps -p "11817"
exit_loop => Data = {data,{eol,"  PID TTY          TIME CMD"}}
exit_loop => Exit = {exit_status,0}
wait_for_process_death Pid = "11817"
run_ps => erlang:open_port -> ps -p "11817"
exit_loop => Data = {data,{eol,"  PID TTY          TIME CMD"}}
exit_loop => Exit = {exit_status,0}
wait_for_process_death Pid = "11817"
run_ps => erlang:open_port -> ps -p "11817"
exit_loop => Data = {data,{eol,"  PID TTY          TIME CMD"}}
exit_loop => Exit = {exit_status,0}
wait_for_process_death Pid = "11817"
run_ps => erlang:open_port -> ps -p "11817"
exit_loop => Data = {data,{eol,"  PID TTY          TIME CMD"}}
exit_loop => Exit = {exit_status,0}
...
(永远停不下来)
...
^C
BREAK: (a)bort (c)ontinue (p)roc info (i)nfo (l)oaded
       (v)ersion (k)ill (D)b-tables (d)istribution
^Cremove pid file and dir
rabbitmq-server.
[[email protected] ~]# ^C
[[email protected] ~]#

可以看到无论执行多少次循环,永远返回 {exit_status,0} ,所以导致 rabbitmqctl stop <pidfile> 无法停止。

正常环境中的输出

[[email protected] ~]# 
[[email protected] ~]# /etc/init.d/rabbitmq start
Starting rabbitmq-server: SUCCESS
rabbitmq-server.
[[email protected] ~]# 
[[email protected] ~]# 
[[email protected] ~]# ps aux|grep rabbit        
root      2431  0.0  0.0 106456  1004 pts/3    S    14:40   0:00 /bin/sh /etc/init.d/rabbitmq start
root      2434  0.0  0.0 106092  1172 pts/3    S    14:40   0:00 /bin/bash -c ulimit -S -c 0 >/dev/null 2>&1 ; /usr/sbin/rabbitmq-server
root      2436 42.8  1.4 2296732 56332 pts/3   Sl   14:40   0:02 /usr/local/lib/erlang/erts-6.0/bin/beam.smp -W w -K true -A30 -P 1048576 -- -root /usr/local/lib/erlang -progname erl -- -home /root -- -pa /usr/lib/rabbitmq/sbin/../ebin -noshell -noinput -s rabbit boot -sname rmq_betty -boot start_sasl -config /etc/rabbitmq/rabbitmq -kernel inet_default_connect_options [{nodelay,true}] -sasl errlog_type error -sasl sasl_error_logger false -rabbit error_logger {file,"/var/log/rabbitmq/rmq_betty.log"} -rabbit sasl_error_logger {file,"/var/log/rabbitmq/rmq_betty-sasl.log"} -rabbit enabled_plugins_file "/etc/rabbitmq/enabled_plugins" -rabbit plugins_dir "/usr/lib/rabbitmq/sbin/../plugins" -rabbit plugins_expand_dir "/var/lib/rabbitmq/mnesia/rmq_betty-plugins-expand" -os_mon start_cpu_sup false -os_mon start_disksup false -os_mon start_memsup false -mnesia dir "/var/lib/rabbitmq/mnesia/rmq_betty" -kernel inet_dist_listen_min 25672 -kernel inet_dist_listen_max 25672
root      2567  0.0  0.0 103256   856 pts/3    S+   14:40   0:00 grep rabbit
[[email protected] ~]# 
[[email protected] ~]# 
[[email protected] ~]# cat /var/run/rabbitmq/pid 
2436
[[email protected] ~]# 
[[email protected] ~]# 
[[email protected] ~]# ps -p 2436
  PID TTY          TIME CMD
 2436 pts/3    00:00:02 beam.smp
[[email protected] ~]# 
[[email protected] ~]# 
[[email protected] ~]# /etc/init.d/rabbitmq stop
Stopping rabbitmq-server: Stopping and halting node [email protected] ...
Args = ["/var/run/rabbitmq/pid"]
 ...
read_pid_file => PidFile : "/var/run/rabbitmq/pid"   Wait = false
wait_for_process_death Pid = "2436"
run_ps => erlang:open_port -> ps -p "2436"
exit_loop => Data = {data,{eol,"  PID TTY          TIME CMD"}}
exit_loop => Data = {data,{eol," 2436 pts/3    00:00:02 beam.smp"}}
exit_loop => Exit = {exit_status,0}
wait_for_process_death Pid = "2436"
run_ps => erlang:open_port -> ps -p "2436"
exit_loop => Data = {data,{eol,"  PID TTY          TIME CMD"}}
exit_loop => Data = {data,{eol," 2436 pts/3    00:00:02 beam.smp"}}
exit_loop => Exit = {exit_status,0}
wait_for_process_death Pid = "2436"
run_ps => erlang:open_port -> ps -p "2436"
exit_loop => Data = {data,{eol,"  PID TTY          TIME CMD"}}
exit_loop => Exit = {exit_status,1}
...done.
rabbitmq-server.
[[email protected] ~]#

可以看到,循环检测 3 次后就返回了 {exit_status,1} ,从而 rabbitmqctl stop <pidfile> 能够正常退出。

问题深入

深入研究一下 erlang:open_port/2 的使用

手册内容如下:

-=-=-=-=-=- 我是开始研究大数据问题的分隔线 -=-=-=-=-=-=-

open_port(PortName, PortSettings) -> port()


Returns a port identifier as the result of opening a new Erlang port. A port can be seen as an external Erlang process.

PortName is one of the following:

{spawn, Command}


Starts an external program.  Command is the name of the external program which will be run.  Command runs


outside the Erlang work space unless an Erlang driver with the name Command is found. If found, that driver will


be started. A driver runs in the Erlang workspace, which means that it is linked with the Erlang runtime system.


When starting external programs on Solaris, the system call vfork is used in preference to  fork for performance


reasons, although it has a history of being less robust. If there are problems with using  vfork, setting the


environment variable ERL_NO_VFORK to any value will cause fork to be used instead.


For external programs, the PATH is searched (or an equivalent method is used to find programs, depending on


operating system). This is done by invoking the shell on certain platforms. The first space separated token of


the command will be considered as the name of the executable (or driver). This (among other things) makes


this option unsuitable for running programs having spaces in file or directory names. Use {spawn_executable,


Command} instead if spaces in executable file names is desired.

{line, L}


Messages are delivered on a per line basis. Each line (delimited by the OS-dependent newline sequence) is


delivered in one single message. The message data format is {Flag, Line}, where  Flag is either  eol or


noeol and Line is the actual data delivered (without the newline sequence).


消息以行为基础单位进行投递;


每一行输出内容(由操作系统相关的换行序列确定)作为单独一条消息投递;


消息数据格式为 {Flag, Line} ,其中 Flag 是 eol 或者 noeol ;Line 为实际投递的数据内容(不包含换行序列)

L specifies the maximum line length in bytes. Lines longer than this will be delivered in more than one message,


with the  Flag set to  noeol for all but the last message. If end of file is encountered anywhere else than


immediately following a newline sequence, the last line will also be delivered with the Flag set to noeol. In


all other cases, lines are delivered with Flagset to eol.


L 指定了以字节为单位的最大行内容长度;


超过规定长度的行,将会以多条消息进行投递,除最后一条消息外,均设置 Flag 为 noeol ;


若遇到了文件的结束,但没有换行序列,则最后一行内容同样会被投递,且设置 Flag 为 noeol ;


在其他情况下,行数据会设置 Flag 为 eol 进行投递。

The {packet, N} and {line, L} settings are mutually exclusive.

exit_status


This is only valid for  {spawn, Command} where  Command refers to an external program, and for


{spawn_executable, Command}.

When the external process connected to the port exits, a message of the form  {Port,


{exit_status,Status}} is sent to the connected process, where Status is the exit status of the external


process. If the program aborts, on Unix the same convention is used as the shells do (i.e., 128+signal).


If the  eofoption has been given as well, the  eofmessage and the  exit_statusmessage appear in an


unspecified order.


当与 port 连接的外部进程退出时,形式为 {Port,{exit_status,Status}} 的消息会被发送到 erlang 进程,其中 Status 的内容为外部进程的退出状态;


如果外部程序主动 abort 了,在 Unix 系统中,将采用类似 shell 形式的退出码策略(即 128+signal)

If the port program closes its stdout without exiting, the exit_status option will not work.


如果 port 程序关闭了其 stdout 却没有退出,则 exit_status 选项不起作用。

use_stdio


This is only valid for {spawn, Command}and {spawn_executable, Command}. It allows the standard


input and output (file descriptors 0 and 1) of the spawned (UNIX) process for communication with Erlang.


该选项允许被创建 Unix 进程的 stdin 和 stdout 可以和 Erlang 进行通信。

stderr_to_stdout


Affects ports to external programs. The executed program gets its standard error file redirected to its standard


output file. stderr_to_stdout and nouse_stdio are mutually exclusive.


该选项对外部程序的 port 产生影响;


可执行程序的 stderr 会被重定向到其 stdout 上;

-=-=-=-=-=- 我是开始研究大数据问题的分隔线 -=-=-=-=-=-=-

从手册中可以知道,RabbitMQ 代码中是通过 Erlang Port 调用操作系统的命令,并根据返回结果的内容来判定对应的进程是否已停止的。

但在我们的环境中却遇到了外部进程退出状态错误的问题。

解决办法

1.修改 init 脚本,将调用 rabbitmqctl stop <pidfile> 的地方,改成 rabbitmqctl stop(需要相应调整针对 pidfile 的各种条件判定
2.不要使用 VMware 虚拟机,因为在 KVM 虚拟机上没有发现此问题(VMware 粉不要喷我~)

—————————-
—————————-
—————————-
—————————-





重大发现:根本原因是服务器被攻击了,ps 文件被替换掉了!



【被攻击机器】



确认问题 ps 的退出码

[[email protected] ~]# 
[[email protected] ~]# ps -p 123
  PID TTY          TIME CMD
  123 ?        00:00:00 kthrotld/1
[[email protected] ~]# 
[[email protected] ~]# echo $?
0
[[email protected] ~]# 
[[email protected] ~]# ps -p 1234
  PID TTY          TIME CMD
[[email protected] ~]# 
[[email protected] ~]# echo $?   
0                          -- 正常的 ps 程序,此处应该返回 1
[[email protected] ~]#

确认问题 ps 程序有哪些特征

[[email protected] ~]# uname -a
Linux gfsd1 2.6.32-431.el6.x86_64 #1 SMP Fri Nov 22 03:15:09 UTC 2013 x86_64 x86_64 x86_64 GNU/Linux
[[email protected] ~]#

ps 文件大小和正常机器上不同
[[email protected] ~]# ll /bin/ps 
-rwxr-xr-x 1 root root 1223123 Mar 18 10:42 /bin/ps
[[email protected] ~]# 
[[email protected] ~]# 

64bit 机器上出现了 32bit 的 ps 文件
[[email protected] ~]# file /bin/ps   
/bin/ps: ELF 32-bit LSB executable, Intel 80386, version 1 (SYSV), statically linked, for GNU/Linux 2.2.5, not stripped
[[email protected] ~]# 

正常机器上 ps 是动态链接的
[[email protected] ~]# ldd /bin/ps       
        not a dynamic executable
[[email protected] ~]# 

跟踪问题 ps 的执行

[[email protected] ~]# 
[[email protected] ~]# strace -f ps -p 1234
execve("/bin/ps", ["ps", "-p", "1234"], [/* 32 vars */]) = 0   -- 调用问题 ps 程序
[ Process PID=14740 runs in 32 bit mode. ]
...
set_thread_area(0xffc3db64)             = 0
set_tid_address(0xa12c878)              = 14740
...
getrlimit(RLIMIT_STACK, {rlim_cur=-4284481536, rlim_max=564187122106368*1024}) = 0
_sysctl({0x2081076ec, -3941488, (nil), (nil), (nil), 18429817727855099905}) = 0
...
close(3)                                = -1 EBADF (Bad file descriptor)
...
close(1023)                             = -1 EBADF (Bad file descriptor)
readlink("/proc/14740/exe", "/bin/ps", 1024) = 7
stat64("/bin/ps", {st_mode=S_IFREG|0755, st_size=1223123, ...}) = 0
getppid()                               = 14734
readlink("/proc/14734/exe", "/usr/bin/strace", 255) = 15
readlink("/proc/14740/exe", "/bin/ps", 1024) = 7
readlink("/proc/14740/exe", "/bin/ps", 1024) = 7
readlink("/proc/14740/exe", "/bin/ps", 1024) = 7
access("/usr/bin/dpkgd/ps", F_OK)       = 0
pipe([3, 4])                            = 0
clone(Process 14845 attached
child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0) = 14845    -- 创建了一个线程(正常 ps 没有该行为)
...
[pid 14845] dup2(4, 1 <unfinished ...>
[pid 14740] <... fstat64 resumed> {st_mode=S_IFIFO|0600, st_size=0, ...}) = 0
[pid 14845] <... dup2 resumed> )        = 1
[pid 14740] mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0 <unfinished ...>
[pid 14845] close(4)                    = 0
...
[pid 14845] execve("/bin/sh", ["sh", "-c", "/usr/bin/dpkgd/ps -p 1234"], [/* 40 vars */] <unfinished ...>   -- 在新建线程中调用的是 /usr/bin/dpkgd/ 目录下的 ps 文件!!!
[pid 14740] read(3,  <unfinished ...>
[pid 14845] <... execve resumed> )      = 0
[ Process PID=14845 runs in 64 bit mode. ]
...
[pid 14845] getdents(4, /* 281 entries */, 32768) = 7352
[pid 14845] stat("/proc/1", {st_mode=S_IFDIR|0555, st_size=0, ...}) = 0
[pid 14845] open("/proc/1/stat", O_RDONLY) = 5
[pid 14845] read(5, "1 (init) S 0 1 1 0 -1 4202752 30"..., 1023) = 260
[pid 14845] close(5)                    = 0
[pid 14845] open("/proc/1/status", O_RDONLY) = 5
[pid 14845] read(5, "Name:\tinit\nState:\tS (sleeping)\nT"..., 1023) = 870
[pid 14845] close(5)                    = 0
[pid 14845] stat("/proc/2", {st_mode=S_IFDIR|0555, st_size=0, ...}) = 0
[pid 14845] open("/proc/2/stat", O_RDONLY) = 5
[pid 14845] read(5, "2 (kthreadd) S 0 0 0 0 -1 214961"..., 1023) = 155
[pid 14845] close(5)                    = 0
[pid 14845] open("/proc/2/status", O_RDONLY) = 5
[pid 14845] read(5, "Name:\tkthreadd\nState:\tS (sleepin"..., 1023) = 656
[pid 14845] close(5)                    = 0
...
[pid 14845] stat("/proc/28465", {st_mode=S_IFDIR|0555, st_size=0, ...}) = 0
[pid 14845] open("/proc/28465/stat", O_RDONLY) = 5
[pid 14845] read(5, "28465 (inet_gethost) S 28451 284"..., 1023) = 237
[pid 14845] close(5)                    = 0
[pid 14845] open("/proc/28465/status", O_RDONLY) = 5
[pid 14845] read(5, "Name:\tinet_gethost\nState:\tS (sle"..., 1023) = 884
[pid 14845] close(5)                    = 0
[pid 14845] getdents(4, /* 0 entries */, 32768) = 0
[pid 14845] close(4)                    = 0
[pid 14845] fstat(1, {st_mode=S_IFIFO|0600, st_size=0, ...}) = 0
[pid 14845] mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4528922000
[pid 14845] write(1, "  PID TTY          TIME CMD\n", 28) = 28
[ Process PID=14740 runs in 32 bit mode. ]
[pid 14740] <... read resumed> "  PID TTY          TIME CMD\n", 4096) = 28
[ Process PID=14845 runs in 64 bit mode. ]
[pid 14845] exit_group(1)               = ?                        -- 线程中的 ps 退出码为 1
Process 14845 detached
[ Process PID=14740 runs in 32 bit mode. ]
fstat64(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 1), ...}) = 0
mmap2(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0xfffffffff778a000
--- SIGCHLD (Child exited) @ 0 (0) ---
write(1, "  PID TTY          TIME CMD\n", 28  PID TTY          TIME CMD
) = 28
read(3, "", 4096)                       = 0
close(3)                                = 0
waitpid(14845, [{WIFEXITED(s) && WEXITSTATUS(s) == 1}], 0) = 14845      -- 可以看到,问题 ps 成功拿到了正确 ps 的退出码
munmap(0xf778b000, 4096)                = 0
munmap(0xf778a000, 4096)                = 0
exit_group(0)                           = ?       -- 但问题 ps 最终以退出码 0 退出
[[email protected] ~]# 

由上面的信息,可以知道正确的 ps 程序被搬移到了下面的位置

[[email protected] ~]# ll /usr/bin/dpkgd/
total 436
-rwxr-xr-x 1 root root 145872 Jan 28 05:00 lsof
-rwxr-xr-x 1 root root 128192 Jan 28 05:00 netstat
-rwxr-xr-x 1 root root  87088 Jan 28 05:00 ps
-rwxr-xr-x 1 root root  74840 Jan 28 05:00 ss
[[email protected] ~]# 
[[email protected] ~]# file /usr/bin/dpkgd/*
/usr/bin/dpkgd/lsof:    ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked (uses shared libs), for GNU/Linux 2.6.18, stripped
/usr/bin/dpkgd/netstat: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked (uses shared libs), for GNU/Linux 2.6.18, stripped
/usr/bin/dpkgd/ps:      ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked (uses shared libs), for GNU/Linux 2.6.18, stripped
/usr/bin/dpkgd/ss:      ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked (uses shared libs), for GNU/Linux 2.6.18, stripped
[[email protected] ~]# 
[[email protected] ~]# 
[[email protected] ~]# ldd /usr/bin/dpkgd/* 
/usr/bin/dpkgd/lsof:
        linux-vdso.so.1 =>  (0x00007fff86bff000)
        libselinux.so.1 => /lib64/libselinux.so.1 (0x0000003522000000)
        libc.so.6 => /lib64/libc.so.6 (0x0000003ef8200000)
        libdl.so.2 => /lib64/libdl.so.2 (0x0000003ef7e00000)
        /lib64/ld-linux-x86-64.so.2 (0x0000003ef7a00000)
/usr/bin/dpkgd/netstat:
        linux-vdso.so.1 =>  (0x00007fff3ddc2000)
        libselinux.so.1 => /lib64/libselinux.so.1 (0x0000003522000000)
        libc.so.6 => /lib64/libc.so.6 (0x0000003ef8200000)
        libdl.so.2 => /lib64/libdl.so.2 (0x0000003ef7e00000)
        /lib64/ld-linux-x86-64.so.2 (0x0000003ef7a00000)
/usr/bin/dpkgd/ps:
        linux-vdso.so.1 =>  (0x00007fff961b6000)
        libselinux.so.1 => /lib64/libselinux.so.1 (0x0000003522000000)
        libproc-3.2.8.so => /lib64/libproc-3.2.8.so (0x0000003ef8600000)
        libc.so.6 => /lib64/libc.so.6 (0x0000003ef8200000)
        libdl.so.2 => /lib64/libdl.so.2 (0x0000003ef7e00000)
        /lib64/ld-linux-x86-64.so.2 (0x0000003ef7a00000)
/usr/bin/dpkgd/ss:
        linux-vdso.so.1 =>  (0x00007fffb51e1000)
        libresolv.so.2 => /lib64/libresolv.so.2 (0x0000003ef9e00000)
        libc.so.6 => /lib64/libc.so.6 (0x0000003ef8200000)
        /lib64/ld-linux-x86-64.so.2 (0x0000003ef7a00000)
[[email protected] ~]# 

可以看到,被攻击的可执行程序不止 ps 一个…



【正常机器】

正常 ps 程序的特征

[[email protected] ~]# uname -a
Linux YOYO 2.6.32-358.el6.x86_64 #1 SMP Fri Feb 22 00:31:26 UTC 2013 x86_64 x86_64 x86_64 GNU/Linux
[[email protected] ~]# 
[[email protected] ~]# ll /bin/ps 
-rwxr-xr-x. 1 root root 87088 11月 15 2012 /bin/ps
[[email protected] ~]# 
[[email protected] ~]# file /bin/ps 
/bin/ps: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked (uses shared libs), for GNU/Linux 2.6.18, stripped
[[email protected] ~]# 
[[email protected] ~]# ldd /bin/ps 
        linux-vdso.so.1 =>  (0x00007fffb6d24000)
        libselinux.so.1 => /lib64/libselinux.so.1 (0x00000034e7000000)
        libproc-3.2.8.so => /lib64/libproc-3.2.8.so (0x0000003f38400000)
        libc.so.6 => /lib64/libc.so.6 (0x0000003f38000000)
        libdl.so.2 => /lib64/libdl.so.2 (0x0000003f37c00000)
        /lib64/ld-linux-x86-64.so.2 (0x0000003f37800000)
[[email protected] ~]# 

跟踪正常 ps 程序的执行

[[email protected] ~]# strace -f ps -p 1234                
execve("/bin/ps", ["ps", "-p", "1234"], [/* 28 vars */]) = 0
...
getdents(4, /* 241 entries */, 32768)   = 6192
stat("/proc/1", {st_mode=S_IFDIR|0555, st_size=0, ...}) = 0
open("/proc/1/stat", O_RDONLY)          = 5
read(5, "1 (init) S 0 1 1 0 -1 4202752 29"..., 1023) = 256
close(5)                                = 0
open("/proc/1/status", O_RDONLY)        = 5
read(5, "Name:\tinit\nState:\tS (sleeping)\nT"..., 1023) = 866
close(5)                                = 0
stat("/proc/2", {st_mode=S_IFDIR|0555, st_size=0, ...}) = 0
open("/proc/2/stat", O_RDONLY)          = 5
read(5, "2 (kthreadd) S 0 0 0 0 -1 214961"..., 1023) = 156
close(5)                                = 0
open("/proc/2/status", O_RDONLY)        = 5
read(5, "Name:\tkthreadd\nState:\tS (sleepin"..., 1023) = 654
close(5)                                = 0
...
stat("/proc/25701", {st_mode=S_IFDIR|0555, st_size=0, ...}) = 0
open("/proc/25701/stat", O_RDONLY)      = 5
read(5, "25701 (bash) S 25697 25701 25701"..., 1023) = 257
close(5)                                = 0
open("/proc/25701/status", O_RDONLY)    = 5
read(5, "Name:\tbash\nState:\tS (sleeping)\nT"..., 1023) = 879
close(5)                                = 0
getdents(4, /* 0 entries */, 32768)     = 0
close(4)                                = 0
fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 2), ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4d2da3c000
write(1, "  PID TTY          TIME CMD\n", 28  PID TTY          TIME CMD
) = 28
exit_group(1)                           = ?
[[email protected] ~]# 

可以看到正常情况下 ps -p xxx 是以退出码 1 退出的。

—————————-

—————————-
—————————-
—————————-


与此木马程序相关的文章:这里 。
系统漏洞相关文档:《NTRG_ElasticBotnetReport_06102015》
漏洞号:CVE-2015-1427

问题 ps 中的符号片段

...
08048b30 T CAttackIe::UpdateCurVariant(unsigned int)
08048b36 T CTcpAttack::UpdateCurVariant(unsigned int)
08048be4 T CAttackIcmp::MakePacket(unsigned int)
08048c5a T CAttackSyn::MakePacket(unsigned int)
08048d02 T CAttackUdp::MakePacket(unsigned int)
08048dce T CPacketAttack::SendPacket()
08048eac T CPacketAttack::Create()
08048f7e T CAttackAmp::Create()
08049070 T CPacketAttack::UpdateCurVariant(unsigned int)
08049306 T CTcpAttack::Do(unsigned int)
08049546 T CAttackPrx::MakePacket(unsigned int)
080497d6 T CAttackAmp::MakePacket(unsigned int)
0804996c T CAttackDns::MakePacket(unsigned int)
08049bf8 T CAttackCc::Destroy()
08049cc8 T CAttackCc::Create()
08049e8c T CAttackTns::Destroy()
08049f5c T CAttackTns::Create()
0804a05a T CAttackPrx::Create()
0804a2a6 T CAttackDns::Create()
0804a4f2 T CAttackCompress::CreatePackets()
0804a6ec T CAttackCompress::MakePacket(unsigned int)
0804a8be T CAttackCompress::Create()
...


未经允许不得转载:SEARU.ORG » 【原创】遇到一个 rabbitmqctl 无法退出的问题

赞 (0)
分享到:更多 ()

评论 0