2012年1月6日金曜日

nginx debug #1

nginx -s reload をすると core を吐く現象が。
debugオプションを付けてrecompileしgdb で探る。

execution environment
$ uname -a
Linux hostname 2.6.26-2-amd64 #1 SMP Wed Sep 21 03:36:44 UTC 2011 x86_64 GNU/Linux

nginx version & compile options
$ /usr/local/nginx/sbin/nginx -V
nginx: nginx version: nginx/1.0.10
nginx: TLS SNI support enabled
nginx: configure arguments: \
--with-debug \
--prefix=/usr/local/nginx \
--conf-path=/usr/local/nginx/etc/nginx/nginx.conf \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--lock-path=/usr/local/nginx/var/lock/nginx.lock \
--pid-path=/usr/local/nginx/var/run/nginx.pid \
--with-http_addition_module \
--with-http_dav_module \
--with-http_flv_module \
--with-http_geoip_module \
--with-http_gzip_static_module \
--with-http_image_filter_module \
--with-http_random_index_module \
--with-http_realip_module \
--with-http_secure_link_module \
--with-http_stub_status_module \
--with-http_ssl_module \
--with-http_sub_module \
--with-http_xslt_module \
--with-ipv6 \
--with-sha1=/usr/include/openssl \
--with-md5=/usr/include/openssl \
--add-module=/var/tmp/debug-nginx-1.0.10/debian/modules/ngx_devel_kit \
--add-module=/var/tmp/debug-nginx-1.0.10/debian/modules/chunkin-nginx-module \
--add-module=/var/tmp/debug-nginx-1.0.10/debian/modules/echo-nginx-module \
--add-module=/var/tmp/debug-nginx-1.0.10/debian/modules/headers-more-nginx-module \
--add-module=/var/tmp/debug-nginx-1.0.10/debian/modules/healthcheck_nginx_upstreams \
--add-module=/var/tmp/debug-nginx-1.0.10/debian/modules/lua-nginx-module \
--add-module=/var/tmp/debug-nginx-1.0.10/debian/modules/nginx-upstream-fair \
--without-http_uwsgi_module \
--without-http_scgi_module \
--http-proxy-temp-path=/usr/local/nginx/cache/proxy_temp \
--http-fastcgi-temp-path=/usr/local/nginx/cache/fastcgi_temp \
--http-client-body-temp-path=/usr/local/nginx/cache/client_body_temp


nginx.conf
worker_processes 4;
pid /usr/local/nginx/var/run/nginx.pid;
worker_rlimit_core  500M;
working_directory   /var/tmp;

events {
    worker_connections 3000;
    use epoll;
}

http {
    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 0;
    server_tokens off;

    include /usr/local/nginx/etc/nginx/mime.types;
    default_type text/plain;
    include /usr/local/nginx/etc/nginx/proxy_params;

    log_format custom '$remote_addr - $remote_user $time_local "$request" $status $bytes_sent "$http_referer" "$http_user_agent" "$http_cookie" $request_time\
 $upstream_response_time';
    access_log /var/log/nginx/access.log custom;
    error_log  /var/log/nginx/error.log debug;

    include /usr/local/nginx/etc/nginx/conf.d/*.conf;
    include /usr/local/nginx/etc/nginx/sites-enabled/*;
}

dump a core
$ nginx
$ nginx -s reload
$ stat /var/tmp/core
  File: `/var/tmp/core'
  Size: 2617344         Blocks: 4608       IO Block: 4096   regular file
Device: 801h/2049d      Inode: 295019      Links: 1
Access: (0600/-rw-------)  Uid: (   33/www-data)   Gid: (   33/www-data)
Access: 2012-01-06 12:04:09.000000000 +0900
Modify: 2012-01-06 12:39:29.000000000 +0900
Change: 2012-01-06 12:39:29.000000000 +0900

syslog, error.log
==> /var/log/syslog <==
Jan  6 12:00:16 hostname kernel: [ 7968.550937] nginx[29042]: segfault at 18 ip 425609 sp 7fffd8f51d10 error 6 in nginx[400000+115000]

==> /var/log/nginx/error.log <==
2012/01/06 12:00:16 [alert] 29041#0: worker process 29042 exited on signal 11 (core dumped)
debug with gdb
$ sudo gdb /usr/local/nginx/sbin/nginx /var/tmp/core
GNU gdb 6.8-debian
Copyright (C) 2008 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later 
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu"...

warning: Can't read pathname for load map: Input/output error.
Reading symbols from /lib/libpthread.so.0...done.
Loaded symbols for /lib/libpthread.so.0
Reading symbols from /lib/libcrypt.so.1...done.
Loaded symbols for /lib/libcrypt.so.1
Reading symbols from /lib/libm.so.6...done.
Loaded symbols for /lib/libm.so.6
Reading symbols from /usr/lib/liblua5.1.so.0...done.
Loaded symbols for /usr/lib/liblua5.1.so.0
Reading symbols from /usr/lib/libpcre.so.3...done.
Loaded symbols for /usr/lib/libpcre.so.3
Reading symbols from /usr/lib/libssl.so.0.9.8...done.
Loaded symbols for /usr/lib/libssl.so.0.9.8
Reading symbols from /usr/lib/libcrypto.so.0.9.8...done.
Loaded symbols for /usr/lib/libcrypto.so.0.9.8
Reading symbols from /lib/libdl.so.2...done.
Loaded symbols for /lib/libdl.so.2
Reading symbols from /usr/lib/libz.so.1...done.
Loaded symbols for /usr/lib/libz.so.1
Reading symbols from /usr/lib/libxml2.so.2...done.
Loaded symbols for /usr/lib/libxml2.so.2
Reading symbols from /usr/lib/libxslt.so.1...done.
Loaded symbols for /usr/lib/libxslt.so.1
Reading symbols from /usr/lib/libexslt.so.0...done.
Loaded symbols for /usr/lib/libexslt.so.0
Reading symbols from /usr/lib/libgd.so.2...done.
Loaded symbols for /usr/lib/libgd.so.2
Reading symbols from /usr/lib/libGeoIP.so.1...done.
Loaded symbols for /usr/lib/libGeoIP.so.1
Reading symbols from /lib/libc.so.6...done.
Loaded symbols for /lib/libc.so.6
Reading symbols from /lib/ld-linux-x86-64.so.2...done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /usr/lib/libgcrypt.so.11...done.
Loaded symbols for /usr/lib/libgcrypt.so.11
Reading symbols from /usr/lib/libjpeg.so.62...done.
Loaded symbols for /usr/lib/libjpeg.so.62
Reading symbols from /usr/lib/libfreetype.so.6...done.
Loaded symbols for /usr/lib/libfreetype.so.6
Reading symbols from /usr/lib/libpng12.so.0...done.
Loaded symbols for /usr/lib/libpng12.so.0
Reading symbols from /usr/lib/libgpg-error.so.0...done.
Loaded symbols for /usr/lib/libgpg-error.so.0
Reading symbols from /lib/libnss_compat.so.2...done.
Loaded symbols for /lib/libnss_compat.so.2
Reading symbols from /lib/libnsl.so.1...done.
Loaded symbols for /lib/libnsl.so.1
Reading symbols from /lib/libnss_nis.so.2...done.
Loaded symbols for /lib/libnss_nis.so.2
Reading symbols from /lib/libnss_files.so.2...done.
Loaded symbols for /lib/libnss_files.so.2
Reading symbols from /lib/libnss_dns.so.2...done.
Loaded symbols for /lib/libnss_dns.so.2
Reading symbols from /lib/libresolv.so.2...done.
Loaded symbols for /lib/libresolv.so.2
Core was generated by `nginx: worker process is shu'.
Program terminated with signal 11, Segmentation fault.
[New process 5749]
#0  0x00000000004259b1 in ngx_rbtree_min (node=0x0, sentinel=0x728d00) at src/core/ngx_rbtree.h:75
75          while (node->left != sentinel) {
(gdb) backtrace full
#0  0x00000000004259b1 in ngx_rbtree_min (node=0x0, sentinel=0x728d00) at src/core/ngx_rbtree.h:75
No locals.
#1  0x00000000004254ff in ngx_rbtree_delete (tree=0x729710, node=0x9dc8c0) at src/core/ngx_rbtree.c:178
        red = 10170528
        root = (ngx_rbtree_node_t **) 0x729710
        sentinel = (ngx_rbtree_node_t *) 0x728d00
        subst = (ngx_rbtree_node_t *) 0xa07f08
        temp = (ngx_rbtree_node_t *) 0x2
        w = (ngx_rbtree_node_t *) 0x4f066cf1
#2  0x00000000004ccb5b in ngx_event_del_timer (ev=0x9dc898) at src/event/ngx_event_timer.h:43
No locals.
#3  0x00000000004cdb1b in ngx_http_healthcheck_clear_events (log=0x9c8e28)
    at /var/tmp/debug-nginx-1.0.10/debian/modules/healthcheck_nginx_upstreams/ngx_http_healthcheck_module.c:669
        i = 0
#4  0x00000000004cd91b in ngx_http_healthcheck_try_for_ownership (event=0x9dca30)
    at /var/tmp/debug-nginx-1.0.10/debian/modules/healthcheck_nginx_upstreams/ngx_http_healthcheck_module.c:627
        stat = (ngx_http_healthcheck_status_t *) 0x9dc998
        i_own_it = 10341864
#5  0x000000000043ab9a in ngx_event_expire_timers () at src/event/ngx_event_timer.c:149
        ev = (ngx_event_t *) 0x9dca30
        node = (ngx_rbtree_node_t *) 0x9dca58
        root = (ngx_rbtree_node_t *) 0x9dc928
        sentinel = (ngx_rbtree_node_t *) 0x728d00
#6  0x0000000000438988 in ngx_process_events_and_timers (cycle=0x9c8e10) at src/event/ngx_event.c:261
        flags = 1
        timer = 779
        delta = 779
#7  0x0000000000444ff3 in ngx_worker_process_cycle (cycle=0x9c8e10, data=0x0) at src/os/unix/ngx_process_cycle.c:801
        i = 3000
        c = (ngx_connection_t *) 0x7f2be4b34010
#8  0x0000000000441e92 in ngx_spawn_process (cycle=0x9c8e10, proc=0x444e49 , data=0x0, name=0x4f0a9e "worker process", respawn=-3)
    at src/os/unix/ngx_process.c:196
        on = 1
        pid = 0
        s = 3
#9  0x0000000000443ef0 in ngx_start_worker_processes (cycle=0x9c8e10, n=4, type=-3) at src/os/unix/ngx_process_cycle.c:360
        i = 3
        ch = {command = 1, pid = 5748, slot = 2, fd = 11}
#10 0x00000000004435e0 in ngx_master_process_cycle (cycle=0x9c8e10) at src/os/unix/ngx_process_cycle.c:136
        title = 0x9f7776 "master process /usr/local/nginx/sbin/nginx"
        p = (u_char *) 0x9f77a0 ""
        size = 43
        i = 1
        n = 0
        sigio = 10260008
        set = {__val = {0 }}
        itv = {it_interval = {tv_sec = 0, tv_usec = 7425080}, it_value = {tv_sec = 8, tv_usec = 34}}
        live = 0
        delay = 0
        ls = (ngx_listening_t *) 0x5
        ccf = (ngx_core_conf_t *) 0x9c9db0
#11 0x00000000004158c9 in main (argc=1, argv=0x7fff14e61b68) at src/core/nginx.c:405
        i = 62
        log = (ngx_log_t *) 0x726b00
        cycle = (ngx_cycle_t *) 0x9c8e10
        init_cycle = {conf_ctx = 0x0, pool = 0x9c8370, log = 0x726b00, new_log = {log_level = 0, file = 0x0, connection = 0, handler = 0, data = 0x0, 
    action = 0x0}, files = 0x0, free_connections = 0x0, free_connection_n = 0, reusable_connections_queue = {prev = 0x0, next = 0x0}, listening = {
    elts = 0x0, nelts = 0, size = 0, nalloc = 0, pool = 0x0}, pathes = {elts = 0x0, nelts = 0, size = 0, nalloc = 0, pool = 0x0}, open_files = {last = 0x0, 
    part = {elts = 0x0, nelts = 0, next = 0x0}, size = 0, nalloc = 0, pool = 0x0}, shared_memory = {last = 0x0, part = {elts = 0x0, nelts = 0, next = 0x0}, 
    size = 0, nalloc = 0, pool = 0x0}, connection_n = 0, files_n = 0, connections = 0x0, read_events = 0x0, write_events = 0x0, old_cycle = 0x0, 
  conf_file = {len = 37, data = 0x4ece50 "/usr/local/nginx/etc/nginx/nginx.conf"}, conf_param = {len = 0, data = 0x0}, conf_prefix = {len = 27, 
    data = 0x4ece50 "/usr/local/nginx/etc/nginx/nginx.conf"}, prefix = {len = 17, data = 0x4ece3c "/usr/local/nginx/"}, lock_file = {len = 0, data = 0x0}, 
  hostname = {len = 0, data = 0x0}}
        ccf = (ngx_core_conf_t *) 0x9c9db0
(gdb) 
bug?
#0  0x00000000004259b1 in ngx_rbtree_min (node=0x0, sentinel=0x728d00) at src/core/ngx_rbtree.h:75
75          while (node->left != sentinel) {

0 件のコメント:

コメントを投稿