need desperate help with zombie channels
Hi, I am having difficulty trying to figure out why there are bunch of zombie channels in my system. It seems to me that these zombies come from apr_thread pool. Does anyone have any idea what may be the cause of these problems? freeswitch@internal> show channels uuid,direction,created,created_epoch,name,state,cid_name,cid_num,ip_addr,dest,application,application_data,dialplan,context,read_codec,read_rate,write_codec,write_rate,secure b789468a-4412-490b-bc66-32f149ba4d1d,outbound,2009-11-13 20:15:35,1258114535,sofia/external/999100@192.168.1.116:9342 ,CS_REPORTING,a88999001,a88999001,192.168.1.116,999100@192.168.1.116:9342 ,,,XML,default,,,,, 7e1ecaaa-b2d8-47a0-9982-25cd44186d4e,outbound,2009-11-13 20:15:35,1258114535,sofia/external/999100@192.168.1.116:9342 ,CS_REPORTING,a88999001,a88999001,192.168.1.116,999100@192.168.1.116:9342 ,,,XML,default,,,,, 01fa2ff6-f807-4ef0-b988-70a9fe8c4536,outbound,2009-11-13 20:15:35,1258114535,sofia/external/999100@192.168.1.116:9342 ,CS_EXCHANGE_MEDIA,a88999001,a88999001,192.168.1.116, 999100@192.168.1.116:9342,incre_call_stat,125 165 182 235 13 3184093 0,XML,default,,,,, 0271541f-f0b5-482c-b05d-b196f85121be,inbound,2009-11-13 20:15:35,1258114535,sofia/external/88999001@192.168.1.116:7342 ,CS_EXECUTE,sipp,88999001,192.168.1.116,88999100,hangup,NORMAL_CLEARING,XML,default,,,,, 7e4ccfec-a4ad-4817-9a82-f1166b34576f,outbound,2009-11-13 20:15:35,1258114536,sofia/external/999100@192.168.1.116:9342 ,CS_CONSUME_MEDIA,a88999001,a88999001,192.168.1.116, 999100@192.168.1.116:9342,,,XML,default,,,,, 5 total. freeswitch@internal> uuid_kill b789468a-4412-490b-bc66-32f149ba4d1d -ERR No Such Channel! These channels actually do not exist in the system! Here is my gcore output with 5 zombies out of 100K test calls : Thread 21 (process 8946): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 128000} #2 0x00002b3cb3bfb8ca in switch_console_loop () at src/switch_console.c:819 arg = 1 thread = (switch_thread_t *) 0x2aaab00320d0 thd_attr = (switch_threadattr_t *) 0x2aaab0032070 pool = (switch_memory_pool_t *) 0x2aaab0031f88 __func__ = "switch_console_loop" __PRETTY_FUNCTION__ = "switch_console_loop" #3 0x0000000000402884 in main (argc=1, argv=<value optimized out>) at src/switch.c:753 pid_path = "/usr/local/freeswitch/log/freeswitch.pid", '\0' <repeats 219 times> pid_buffer = "8946", '\0' <repeats 27 times> old_pid_buffer = '\0' <repeats 31 times> pid_len = 4 old_pid_len = 4198811 err = 0x2b3cb3cec77d "Success" ---Type <return> to continue, or q <return> to quit--- nf = 0 runas_user = <value optimized out> runas_group = <value optimized out> nc = 0 pid = <value optimized out> x = <value optimized out> opts = <value optimized out> opts_str = '\0' <repeats 1023 times> local_argv = {0x7ffff6f08c15 "./freeswitch", 0x0 <repeats 1023 times>} arg_argv = {0x0 <repeats 128 times>} alt_dirs = 0 known_opt = <value optimized out> high_prio = 0 flags = 65 ret = <value optimized out> destroy_status = <value optimized out> fd = (switch_file_t *) 0xb6293e0 pool = (switch_memory_pool_t *) 0xb629368 rlp = {rlim_cur = 245760, rlim_max = 245760} waste = 0 __PRETTY_FUNCTION__ = "main" Thread 20 (process 20699): ---Type <return> to continue, or q <return> to quit--- #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 0} #2 0x00002aaaab35e926 in read_packet (listener=0x2aaae7523d08, event=0x2aab3b5ab058, timeout=0) at mod_event_socket.c:1255 do_sleep = 1 '\001' mlen = 0 bytes = 0 mbuf = '\0' <repeats 2047 times> buf = '\0' <repeats 1023 times> len = 123 status = SWITCH_STATUS_BREAK count = <value optimized out> start = 1258117263 pop = (void *) 0x2aaad12f6540 ptr = 0x2aab3b5a98a0 "" crcount = 0 '\0' channel = (switch_channel_t *) 0x0 clen = <value optimized out> __func__ = "read_packet" __PRETTY_FUNCTION__ = "read_packet" ---Type <return> to continue, or q <return> to quit--- #3 0x00002aaaab36347a in listener_run (thread=<value optimized out>, obj=0x2aaae7523d08) at mod_event_socket.c:2093 listener = (listener_t *) 0x0 buf = '\0' <repeats 1023 times> len = 1024 status = <value optimized out> event = (switch_event_t *) 0x0 reply = "\000OK log level [7]", '\0' <repeats 493 times> session = (switch_core_session_t *) 0x0 channel = <value optimized out> revent = (switch_event_t *) 0x0 var = <value optimized out> __PRETTY_FUNCTION__ = "listener_run" __func__ = "listener_run" #4 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 19 (process 14505): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. ---Type <return> to continue, or q <return> to quit--- #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0x2aaaaaf49798, data=0x7afe0080) at misc/apr_queue.c:276 rv = 0 #2 0x00002b3cb3c206be in switch_event_dispatch_thread ( thread=<value optimized out>, obj=<value optimized out>) at src/switch_event.c:248 pop = (void *) 0x0 event = (switch_event_t *) 0x0 queue = (switch_queue_t *) 0x2aaaaaf49798 my_id = 1 __func__ = "switch_event_dispatch_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 18 (process 9334): #0 0x0000003054e0d2cb in read () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3cd50c8 in read_char (el=0x2aaab0028180, cp=0x4027002f "") at read.c:294 num_read = 1076297860 tried = 0 ---Type <return> to continue, or q <return> to quit--- #2 0x00002b3cb3cd4ceb in el_gets (el=0x2aaab0028180, nread=0x40270084) at read.c:241 cmdnum = 112 'p' num = -1321754256 ch = 0 '\0' #3 0x00002b3cb3bfc4bb in console_thread (thread=<value optimized out>, obj=<value optimized out>) at src/switch_console.c:464 arg = 1 count = 1 line = 0x2aaab0034e70 "\n" pool = (switch_memory_pool_t *) 0x2aaab0031f88 __func__ = "console_thread" __PRETTY_FUNCTION__ = "console_thread" #4 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 17 (process 9333): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 ---Type <return> to continue, or q <return> to quit--- tv = {tv_sec = 0, tv_usec = 0} #2 0x00002b3cb3c53895 in softtimer_runtime () at src/switch_time.c:464 current_ms = 692 x = 690 tick = 292 ts = <value optimized out> last = 1258117283599783 fwd_errs = 0 rev_errs = 0 __func__ = "softtimer_runtime" #3 0x00002b3cb3c1a347 in switch_loadable_module_exec (thread=0x0, obj=0x0) at src/switch_loadable_module.c:94 status = <value optimized out> ts = (switch_core_thread_session_t *) 0x0 module = (switch_loadable_module_t *) 0xb6c4e00 __PRETTY_FUNCTION__ = "switch_loadable_module_exec" __func__ = "switch_loadable_module_exec" #4 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 16 (process 9332): ---Type <return> to continue, or q <return> to quit--- #0 0x0000003054e0d4eb in accept () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c707a4 in apr_socket_accept (new=0x416b4020, sock=0xbcfde38, connection_context=0x2aaacda27718) at network_io/unix/sockets.c:187 No locals. #2 0x00002aaaab35f889 in mod_event_socket_runtime () at mod_event_socket.c:2324 pool = (switch_memory_pool_t *) 0xbcfdc88 listener_pool = (switch_memory_pool_t *) 0x2aaacda27718 rv = <value optimized out> sa = (switch_sockaddr_t *) 0xbcfdd68 inbound_socket = (switch_socket_t *) 0x2aaacda277f8 listener = <value optimized out> x = <value optimized out> __func__ = "mod_event_socket_runtime" #3 0x00002b3cb3c1a347 in switch_loadable_module_exec (thread=0x14f, obj=0x2aaacda27948) at src/switch_loadable_module.c:94 status = <value optimized out> ts = (switch_core_thread_session_t *) 0x2aaacda27948 module = (switch_loadable_module_t *) 0x2aaaac0058c0 __PRETTY_FUNCTION__ = "switch_loadable_module_exec" __func__ = "switch_loadable_module_exec" #4 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 ---Type <return> to continue, or q <return> to quit--- No symbol table info available. #5 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 15 (process 9330): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 55000} #2 0x00002aaab503cc4c in node_thread_run (thread=<value optimized out>, obj=<value optimized out>) at mod_fifo.c:580 val = (void *) 0x0 var = (const void *) 0x0 idle_consumers = <value optimized out> hi = (switch_hash_index_t *) 0x0 ppl_waiting = 0 consumer_total = 1087699264 node = <value optimized out> #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. ---Type <return> to continue, or q <return> to quit--- Thread 14 (process 9329): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 100} #2 0x00002aaab44d77be in sofia_profile_worker_thread_run ( thread=<value optimized out>, obj=<value optimized out>) at sofia.c:763 profile = (sofia_profile_t *) 0xbce2310 ireg_loops = 18 gateway_loops = 0 loops = 72 qsize = 4294966782 pop = (void *) 0x0 __PRETTY_FUNCTION__ = "sofia_profile_worker_thread_run" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 13 (process 9328): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 ---Type <return> to continue, or q <return> to quit--- No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 0} #2 0x00002aaab44d77be in sofia_profile_worker_thread_run ( thread=<value optimized out>, obj=<value optimized out>) at sofia.c:763 profile = (sofia_profile_t *) 0x2aaab000eb10 ireg_loops = 5 gateway_loops = 0 loops = 93 qsize = 4294966782 pop = (void *) 0x0 __PRETTY_FUNCTION__ = "sofia_profile_worker_thread_run" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 12 (process 9327): #0 0x00000030542d3368 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x00002aaab45c9c9c in su_epoll_port_wait_events (self=0xbce71c0, tout=1000) at su_epoll_port.c:495 ---Type <return> to continue, or q <return> to quit--- j = 198076976 n = 0 events = 0 index = 10922 version = 3 M = 4 ev = 0x41204ef0 __PRETTY_FUNCTION__ = "su_epoll_port_wait_events" #2 0x00002aaab45d1079 in su_base_port_run (self=0xbce71c0) at su_base_port.c:349 tout = 1000 tout2 = 0 __PRETTY_FUNCTION__ = "su_base_port_run" #3 0x00002aaab45c6c51 in su_port_run (self=0xbce71c0) at su_port.h:326 base = (su_virtual_port_t *) 0xbce71c0 #4 0x00002aaab45c6c29 in su_root_run (self=0xbce72a0) at su_root.c:819 __PRETTY_FUNCTION__ = "su_root_run" #5 0x00002aaab45d8d58 in su_pthread_port_clone_main (varg=0x404f7ac0) at su_pthread_port.c:324 arg = (struct clone_args *) 0x0 task = {{sut_port = 0xbce71c0, sut_root = 0xbce72a0}} zap = 1 #6 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 ---Type <return> to continue, or q <return> to quit--- No symbol table info available. #7 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 11 (process 9326): #0 0x00000030542d3368 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x00002aaab45c9c9c in su_epoll_port_wait_events (self=0xbce78b0, tout=1000) at su_epoll_port.c:495 j = -1342070512 n = 10922 events = 0 index = 10922 version = 3 M = 4 ev = 0x411c8ef0 __PRETTY_FUNCTION__ = "su_epoll_port_wait_events" #2 0x00002aaab45d1079 in su_base_port_run (self=0xbce78b0) at su_base_port.c:349 tout = 1000 tout2 = 0 __PRETTY_FUNCTION__ = "su_base_port_run" #3 0x00002aaab45c6c51 in su_port_run (self=0xbce78b0) at su_port.h:326 ---Type <return> to continue, or q <return> to quit--- base = (su_virtual_port_t *) 0xbce78b0 #4 0x00002aaab45c6c29 in su_root_run (self=0x2aaab001a060) at su_root.c:819 __PRETTY_FUNCTION__ = "su_root_run" #5 0x00002aaab45d8d58 in su_pthread_port_clone_main (varg=0x404bbac0) at su_pthread_port.c:324 arg = (struct clone_args *) 0x0 task = {{sut_port = 0xbce78b0, sut_root = 0x2aaab001a060}} zap = 1 #6 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #7 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 10 (process 9325): #0 0x00000030542d3368 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x00002aaab45c9c9c in su_epoll_port_wait_events (self=0xbce6c30, tout=1000) at su_epoll_port.c:495 j = -1268971119 n = 10922 events = 0 index = 0 version = 1 ---Type <return> to continue, or q <return> to quit--- M = 4 ev = 0x404f7c40 __PRETTY_FUNCTION__ = "su_epoll_port_wait_events" #2 0x00002aaab45d11d4 in su_base_port_step (self=0xbce6c30, tout=1000) at su_base_port.c:467 now = {tv_sec = 3467106082, tv_usec = 971475} __PRETTY_FUNCTION__ = "su_base_port_step" #3 0x00002aaab45c6d6a in su_port_step (self=0xbce6c30, tout=1000) at su_port.h:340 base = (su_virtual_port_t *) 0xbce6c30 #4 0x00002aaab45c6d32 in su_root_step (self=0xbce4650, tout=1000) at su_root.c:858 __PRETTY_FUNCTION__ = "su_root_step" #5 0x00002aaab44e5c3a in sofia_profile_thread_run ( thread=<value optimized out>, obj=<value optimized out>) at sofia.c:973 profile = (sofia_profile_t *) 0xbce2310 pool = <value optimized out> node = (sip_alias_node_t *) 0x0 s_event = (switch_event_t *) 0x0 sanity = <value optimized out> worker_thread = (switch_thread_t *) 0xbce36a0 st = SWITCH_STATUS_SUCCESS __func__ = "sofia_profile_thread_run" ---Type <return> to continue, or q <return> to quit--- __PRETTY_FUNCTION__ = "sofia_profile_thread_run" #6 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #7 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 9 (process 9324): #0 0x00000030542d3368 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x00002aaab45c9c9c in su_epoll_port_wait_events (self=0xbcdffb0, tout=1000) at su_epoll_port.c:495 j = -1268971119 n = 10922 events = 0 index = 0 version = 1 M = 4 ev = 0x404bbc40 __PRETTY_FUNCTION__ = "su_epoll_port_wait_events" #2 0x00002aaab45d11d4 in su_base_port_step (self=0xbcdffb0, tout=1000) at su_base_port.c:467 now = {tv_sec = 3467106083, tv_usec = 525146} __PRETTY_FUNCTION__ = "su_base_port_step" ---Type <return> to continue, or q <return> to quit--- #3 0x00002aaab45c6d6a in su_port_step (self=0xbcdffb0, tout=1000) at su_port.h:340 base = (su_virtual_port_t *) 0xbcdffb0 #4 0x00002aaab45c6d32 in su_root_step (self=0xbcdfe00, tout=1000) at su_root.c:858 __PRETTY_FUNCTION__ = "su_root_step" #5 0x00002aaab44e5c3a in sofia_profile_thread_run ( thread=<value optimized out>, obj=<value optimized out>) at sofia.c:973 profile = (sofia_profile_t *) 0x2aaab000eb10 pool = <value optimized out> node = (sip_alias_node_t *) 0x0 s_event = (switch_event_t *) 0x0 sanity = <value optimized out> worker_thread = (switch_thread_t *) 0x2aaab000fea0 st = SWITCH_STATUS_SUCCESS __func__ = "sofia_profile_thread_run" __PRETTY_FUNCTION__ = "sofia_profile_thread_run" #6 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #7 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 8 (process 8999): ---Type <return> to continue, or q <return> to quit--- #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 444000} #2 0x00002b3cb3c14e2a in switch_scheduler_task_thread ( thread=<value optimized out>, obj=<value optimized out>) at src/switch_scheduler.c:171 __func__ = "switch_scheduler_task_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 7 (process 8998): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 100} #2 0x00002b3cb3c054f5 in switch_core_sql_thread ( thread=<value optimized out>, obj=<value optimized out>) at src/switch_core_sqldb.c:220 ---Type <return> to continue, or q <return> to quit--- pop = (void *) 0x2aaabf3d6220 itterations = 0 trans = 0 '\0' nothing_in_queue = 1 '\001' len = 100 sql_len = 4844546 sqlbuf = 0x2aab135c7010 "" sql = <value optimized out> newlen = <value optimized out> lc = 0 __PRETTY_FUNCTION__ = "switch_core_sql_thread" __func__ = "switch_core_sql_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 6 (process 8995): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0xb64c158, data=0x40893088) at misc/apr_queue.c:276 ---Type <return> to continue, or q <return> to quit--- rv = 0 #2 0x00002b3cb3c48ff1 in log_thread (t=<value optimized out>, obj=<value optimized out>) at src/switch_log.c:288 pop = (void *) 0x0 node = (switch_log_node_t *) 0x0 binding = (switch_log_binding_t *) 0x0 __func__ = "log_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 5 (process 8951): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0x2aaaaac355a8, data=0x40bec070) at misc/apr_queue.c:276 rv = 0 #2 0x00002b3cb3c1fb14 in switch_event_thread (thread=<value optimized out>, obj=<value optimized out>) at src/switch_event.c:291 pop = (void *) 0x0 event = <value optimized out> ---Type <return> to continue, or q <return> to quit--- queue = (switch_queue_t *) 0x2aaaaac355a8 index = 0 my_id = 2 __func__ = "switch_event_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 4 (process 8950): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0x2aaaaab705a8, data=0x4060a070) at misc/apr_queue.c:276 rv = 0 #2 0x00002b3cb3c1fb14 in switch_event_thread (thread=<value optimized out>, obj=<value optimized out>) at src/switch_event.c:291 pop = (void *) 0x0 event = <value optimized out> queue = (switch_queue_t *) 0x2aaaaab705a8 index = 0 my_id = 1 ---Type <return> to continue, or q <return> to quit--- __func__ = "switch_event_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 3 (process 8949): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0xb638fa8, data=0x405ce070) at misc/apr_queue.c:276 rv = 0 #2 0x00002b3cb3c1fb14 in switch_event_thread (thread=<value optimized out>, obj=<value optimized out>) at src/switch_event.c:291 pop = (void *) 0x0 event = <value optimized out> queue = (switch_queue_t *) 0xb638fa8 index = 0 my_id = 0 __func__ = "switch_event_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. ---Type <return> to continue, or q <return> to quit--- #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. Thread 2 (process 8948): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0x2aaaaacfa5a8, data=0x40592080) at misc/apr_queue.c:276 rv = 0 #2 0x00002b3cb3c206be in switch_event_dispatch_thread ( thread=<value optimized out>, obj=<value optimized out>) at src/switch_event.c:248 pop = (void *) 0x0 event = (switch_event_t *) 0x0 queue = (switch_queue_t *) 0x2aaaaacfa5a8 my_id = 0 __func__ = "switch_event_dispatch_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. ---Type <return> to continue, or q <return> to quit--- Thread 1 (process 8947): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 451000} #2 0x00002b3cb3c00c95 in pool_thread (thread=<value optimized out>, obj=<value optimized out>) at src/switch_core_memory.c:490 x = <value optimized out> #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. (gdb) (gdb) (gdb)
From the looks of that you probably have an equal number of zombie event socket processes. We do not get involved in load testing. Consider consulting@freeswitch.orgfor professional help.
On Fri, Nov 13, 2009 at 7:13 AM, Juan Backson <juanbackson@gmail.com> wrote:
Hi,
I am having difficulty trying to figure out why there are bunch of zombie channels in my system. It seems to me that these zombies come from apr_thread pool.
Does anyone have any idea what may be the cause of these problems?
freeswitch@internal> show channels
uuid,direction,created,created_epoch,name,state,cid_name,cid_num,ip_addr,dest,application,application_data,dialplan,context,read_codec,read_rate,write_codec,write_rate,secure b789468a-4412-490b-bc66-32f149ba4d1d,outbound,2009-11-13 20:15:35,1258114535,sofia/external/999100@192.168.1.116:9342 ,CS_REPORTING,a88999001,a88999001,192.168.1.116,999100@192.168.1.116:9342 ,,,XML,default,,,,, 7e1ecaaa-b2d8-47a0-9982-25cd44186d4e,outbound,2009-11-13 20:15:35,1258114535,sofia/external/999100@192.168.1.116:9342 ,CS_REPORTING,a88999001,a88999001,192.168.1.116,999100@192.168.1.116:9342 ,,,XML,default,,,,, 01fa2ff6-f807-4ef0-b988-70a9fe8c4536,outbound,2009-11-13 20:15:35,1258114535,sofia/external/999100@192.168.1.116:9342 ,CS_EXCHANGE_MEDIA,a88999001,a88999001,192.168.1.116, 999100@192.168.1.116:9342,incre_call_stat,125 165 182 235 13 3184093 0,XML,default,,,,, 0271541f-f0b5-482c-b05d-b196f85121be,inbound,2009-11-13 20:15:35,1258114535,sofia/external/88999001@192.168.1.116:7342 ,CS_EXECUTE,sipp,88999001,192.168.1.116,88999100,hangup,NORMAL_CLEARING,XML,default,,,,, 7e4ccfec-a4ad-4817-9a82-f1166b34576f,outbound,2009-11-13 20:15:35,1258114536,sofia/external/999100@192.168.1.116:9342 ,CS_CONSUME_MEDIA,a88999001,a88999001,192.168.1.116, 999100@192.168.1.116:9342,,,XML,default,,,,,
5 total.
freeswitch@internal> uuid_kill b789468a-4412-490b-bc66-32f149ba4d1d -ERR No Such Channel!
These channels actually do not exist in the system!
Here is my gcore output with 5 zombies out of 100K test calls :
Thread 21 (process 8946): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 128000} #2 0x00002b3cb3bfb8ca in switch_console_loop () at src/switch_console.c:819 arg = 1 thread = (switch_thread_t *) 0x2aaab00320d0 thd_attr = (switch_threadattr_t *) 0x2aaab0032070 pool = (switch_memory_pool_t *) 0x2aaab0031f88 __func__ = "switch_console_loop" __PRETTY_FUNCTION__ = "switch_console_loop" #3 0x0000000000402884 in main (argc=1, argv=<value optimized out>) at src/switch.c:753 pid_path = "/usr/local/freeswitch/log/freeswitch.pid", '\0' <repeats 219 times> pid_buffer = "8946", '\0' <repeats 27 times> old_pid_buffer = '\0' <repeats 31 times> pid_len = 4 old_pid_len = 4198811 err = 0x2b3cb3cec77d "Success" ---Type <return> to continue, or q <return> to quit--- nf = 0 runas_user = <value optimized out> runas_group = <value optimized out> nc = 0 pid = <value optimized out> x = <value optimized out> opts = <value optimized out> opts_str = '\0' <repeats 1023 times> local_argv = {0x7ffff6f08c15 "./freeswitch", 0x0 <repeats 1023 times>} arg_argv = {0x0 <repeats 128 times>} alt_dirs = 0 known_opt = <value optimized out> high_prio = 0 flags = 65 ret = <value optimized out> destroy_status = <value optimized out> fd = (switch_file_t *) 0xb6293e0 pool = (switch_memory_pool_t *) 0xb629368 rlp = {rlim_cur = 245760, rlim_max = 245760} waste = 0 __PRETTY_FUNCTION__ = "main"
Thread 20 (process 20699): ---Type <return> to continue, or q <return> to quit--- #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 0} #2 0x00002aaaab35e926 in read_packet (listener=0x2aaae7523d08, event=0x2aab3b5ab058, timeout=0) at mod_event_socket.c:1255 do_sleep = 1 '\001' mlen = 0 bytes = 0 mbuf = '\0' <repeats 2047 times> buf = '\0' <repeats 1023 times> len = 123 status = SWITCH_STATUS_BREAK count = <value optimized out> start = 1258117263 pop = (void *) 0x2aaad12f6540 ptr = 0x2aab3b5a98a0 "" crcount = 0 '\0' channel = (switch_channel_t *) 0x0 clen = <value optimized out> __func__ = "read_packet" __PRETTY_FUNCTION__ = "read_packet" ---Type <return> to continue, or q <return> to quit--- #3 0x00002aaaab36347a in listener_run (thread=<value optimized out>, obj=0x2aaae7523d08) at mod_event_socket.c:2093 listener = (listener_t *) 0x0 buf = '\0' <repeats 1023 times> len = 1024 status = <value optimized out> event = (switch_event_t *) 0x0 reply = "\000OK log level [7]", '\0' <repeats 493 times> session = (switch_core_session_t *) 0x0 channel = <value optimized out> revent = (switch_event_t *) 0x0 var = <value optimized out> __PRETTY_FUNCTION__ = "listener_run" __func__ = "listener_run" #4 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 19 (process 14505): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. ---Type <return> to continue, or q <return> to quit--- #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0x2aaaaaf49798, data=0x7afe0080) at misc/apr_queue.c:276 rv = 0 #2 0x00002b3cb3c206be in switch_event_dispatch_thread ( thread=<value optimized out>, obj=<value optimized out>) at src/switch_event.c:248 pop = (void *) 0x0 event = (switch_event_t *) 0x0 queue = (switch_queue_t *) 0x2aaaaaf49798 my_id = 1 __func__ = "switch_event_dispatch_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 18 (process 9334): #0 0x0000003054e0d2cb in read () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3cd50c8 in read_char (el=0x2aaab0028180, cp=0x4027002f "") at read.c:294 num_read = 1076297860 tried = 0 ---Type <return> to continue, or q <return> to quit--- #2 0x00002b3cb3cd4ceb in el_gets (el=0x2aaab0028180, nread=0x40270084) at read.c:241 cmdnum = 112 'p' num = -1321754256 ch = 0 '\0' #3 0x00002b3cb3bfc4bb in console_thread (thread=<value optimized out>, obj=<value optimized out>) at src/switch_console.c:464 arg = 1 count = 1 line = 0x2aaab0034e70 "\n" pool = (switch_memory_pool_t *) 0x2aaab0031f88 __func__ = "console_thread" __PRETTY_FUNCTION__ = "console_thread" #4 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 17 (process 9333): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 ---Type <return> to continue, or q <return> to quit--- tv = {tv_sec = 0, tv_usec = 0} #2 0x00002b3cb3c53895 in softtimer_runtime () at src/switch_time.c:464 current_ms = 692 x = 690 tick = 292 ts = <value optimized out> last = 1258117283599783 fwd_errs = 0 rev_errs = 0 __func__ = "softtimer_runtime" #3 0x00002b3cb3c1a347 in switch_loadable_module_exec (thread=0x0, obj=0x0) at src/switch_loadable_module.c:94 status = <value optimized out> ts = (switch_core_thread_session_t *) 0x0 module = (switch_loadable_module_t *) 0xb6c4e00 __PRETTY_FUNCTION__ = "switch_loadable_module_exec" __func__ = "switch_loadable_module_exec" #4 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 16 (process 9332): ---Type <return> to continue, or q <return> to quit--- #0 0x0000003054e0d4eb in accept () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c707a4 in apr_socket_accept (new=0x416b4020, sock=0xbcfde38, connection_context=0x2aaacda27718) at network_io/unix/sockets.c:187 No locals. #2 0x00002aaaab35f889 in mod_event_socket_runtime () at mod_event_socket.c:2324 pool = (switch_memory_pool_t *) 0xbcfdc88 listener_pool = (switch_memory_pool_t *) 0x2aaacda27718 rv = <value optimized out> sa = (switch_sockaddr_t *) 0xbcfdd68 inbound_socket = (switch_socket_t *) 0x2aaacda277f8 listener = <value optimized out> x = <value optimized out> __func__ = "mod_event_socket_runtime" #3 0x00002b3cb3c1a347 in switch_loadable_module_exec (thread=0x14f, obj=0x2aaacda27948) at src/switch_loadable_module.c:94 status = <value optimized out> ts = (switch_core_thread_session_t *) 0x2aaacda27948 module = (switch_loadable_module_t *) 0x2aaaac0058c0 __PRETTY_FUNCTION__ = "switch_loadable_module_exec" __func__ = "switch_loadable_module_exec" #4 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 ---Type <return> to continue, or q <return> to quit--- No symbol table info available. #5 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 15 (process 9330): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 55000} #2 0x00002aaab503cc4c in node_thread_run (thread=<value optimized out>, obj=<value optimized out>) at mod_fifo.c:580 val = (void *) 0x0 var = (const void *) 0x0 idle_consumers = <value optimized out> hi = (switch_hash_index_t *) 0x0 ppl_waiting = 0 consumer_total = 1087699264 node = <value optimized out> #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. ---Type <return> to continue, or q <return> to quit---
Thread 14 (process 9329): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 100} #2 0x00002aaab44d77be in sofia_profile_worker_thread_run ( thread=<value optimized out>, obj=<value optimized out>) at sofia.c:763 profile = (sofia_profile_t *) 0xbce2310 ireg_loops = 18 gateway_loops = 0 loops = 72 qsize = 4294966782 pop = (void *) 0x0 __PRETTY_FUNCTION__ = "sofia_profile_worker_thread_run" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 13 (process 9328): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 ---Type <return> to continue, or q <return> to quit--- No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 0} #2 0x00002aaab44d77be in sofia_profile_worker_thread_run ( thread=<value optimized out>, obj=<value optimized out>) at sofia.c:763 profile = (sofia_profile_t *) 0x2aaab000eb10 ireg_loops = 5 gateway_loops = 0 loops = 93 qsize = 4294966782 pop = (void *) 0x0 __PRETTY_FUNCTION__ = "sofia_profile_worker_thread_run" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 12 (process 9327): #0 0x00000030542d3368 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x00002aaab45c9c9c in su_epoll_port_wait_events (self=0xbce71c0, tout=1000) at su_epoll_port.c:495 ---Type <return> to continue, or q <return> to quit--- j = 198076976 n = 0 events = 0 index = 10922 version = 3 M = 4 ev = 0x41204ef0 __PRETTY_FUNCTION__ = "su_epoll_port_wait_events" #2 0x00002aaab45d1079 in su_base_port_run (self=0xbce71c0) at su_base_port.c:349 tout = 1000 tout2 = 0 __PRETTY_FUNCTION__ = "su_base_port_run" #3 0x00002aaab45c6c51 in su_port_run (self=0xbce71c0) at su_port.h:326 base = (su_virtual_port_t *) 0xbce71c0 #4 0x00002aaab45c6c29 in su_root_run (self=0xbce72a0) at su_root.c:819 __PRETTY_FUNCTION__ = "su_root_run" #5 0x00002aaab45d8d58 in su_pthread_port_clone_main (varg=0x404f7ac0) at su_pthread_port.c:324 arg = (struct clone_args *) 0x0 task = {{sut_port = 0xbce71c0, sut_root = 0xbce72a0}} zap = 1 #6 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 ---Type <return> to continue, or q <return> to quit--- No symbol table info available. #7 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 11 (process 9326): #0 0x00000030542d3368 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x00002aaab45c9c9c in su_epoll_port_wait_events (self=0xbce78b0, tout=1000) at su_epoll_port.c:495 j = -1342070512 n = 10922 events = 0 index = 10922 version = 3 M = 4 ev = 0x411c8ef0 __PRETTY_FUNCTION__ = "su_epoll_port_wait_events" #2 0x00002aaab45d1079 in su_base_port_run (self=0xbce78b0) at su_base_port.c:349 tout = 1000 tout2 = 0 __PRETTY_FUNCTION__ = "su_base_port_run" #3 0x00002aaab45c6c51 in su_port_run (self=0xbce78b0) at su_port.h:326 ---Type <return> to continue, or q <return> to quit--- base = (su_virtual_port_t *) 0xbce78b0 #4 0x00002aaab45c6c29 in su_root_run (self=0x2aaab001a060) at su_root.c:819 __PRETTY_FUNCTION__ = "su_root_run" #5 0x00002aaab45d8d58 in su_pthread_port_clone_main (varg=0x404bbac0) at su_pthread_port.c:324 arg = (struct clone_args *) 0x0 task = {{sut_port = 0xbce78b0, sut_root = 0x2aaab001a060}} zap = 1 #6 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #7 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 10 (process 9325): #0 0x00000030542d3368 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x00002aaab45c9c9c in su_epoll_port_wait_events (self=0xbce6c30, tout=1000) at su_epoll_port.c:495 j = -1268971119 n = 10922 events = 0 index = 0 version = 1 ---Type <return> to continue, or q <return> to quit--- M = 4 ev = 0x404f7c40 __PRETTY_FUNCTION__ = "su_epoll_port_wait_events" #2 0x00002aaab45d11d4 in su_base_port_step (self=0xbce6c30, tout=1000) at su_base_port.c:467 now = {tv_sec = 3467106082, tv_usec = 971475} __PRETTY_FUNCTION__ = "su_base_port_step" #3 0x00002aaab45c6d6a in su_port_step (self=0xbce6c30, tout=1000) at su_port.h:340 base = (su_virtual_port_t *) 0xbce6c30 #4 0x00002aaab45c6d32 in su_root_step (self=0xbce4650, tout=1000) at su_root.c:858 __PRETTY_FUNCTION__ = "su_root_step" #5 0x00002aaab44e5c3a in sofia_profile_thread_run ( thread=<value optimized out>, obj=<value optimized out>) at sofia.c:973 profile = (sofia_profile_t *) 0xbce2310 pool = <value optimized out> node = (sip_alias_node_t *) 0x0 s_event = (switch_event_t *) 0x0 sanity = <value optimized out> worker_thread = (switch_thread_t *) 0xbce36a0 st = SWITCH_STATUS_SUCCESS __func__ = "sofia_profile_thread_run" ---Type <return> to continue, or q <return> to quit--- __PRETTY_FUNCTION__ = "sofia_profile_thread_run" #6 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #7 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 9 (process 9324): #0 0x00000030542d3368 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x00002aaab45c9c9c in su_epoll_port_wait_events (self=0xbcdffb0, tout=1000) at su_epoll_port.c:495 j = -1268971119 n = 10922 events = 0 index = 0 version = 1 M = 4 ev = 0x404bbc40 __PRETTY_FUNCTION__ = "su_epoll_port_wait_events" #2 0x00002aaab45d11d4 in su_base_port_step (self=0xbcdffb0, tout=1000) at su_base_port.c:467 now = {tv_sec = 3467106083, tv_usec = 525146} __PRETTY_FUNCTION__ = "su_base_port_step" ---Type <return> to continue, or q <return> to quit--- #3 0x00002aaab45c6d6a in su_port_step (self=0xbcdffb0, tout=1000) at su_port.h:340 base = (su_virtual_port_t *) 0xbcdffb0 #4 0x00002aaab45c6d32 in su_root_step (self=0xbcdfe00, tout=1000) at su_root.c:858 __PRETTY_FUNCTION__ = "su_root_step" #5 0x00002aaab44e5c3a in sofia_profile_thread_run ( thread=<value optimized out>, obj=<value optimized out>) at sofia.c:973 profile = (sofia_profile_t *) 0x2aaab000eb10 pool = <value optimized out> node = (sip_alias_node_t *) 0x0 s_event = (switch_event_t *) 0x0 sanity = <value optimized out> worker_thread = (switch_thread_t *) 0x2aaab000fea0 st = SWITCH_STATUS_SUCCESS __func__ = "sofia_profile_thread_run" __PRETTY_FUNCTION__ = "sofia_profile_thread_run" #6 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #7 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 8 (process 8999): ---Type <return> to continue, or q <return> to quit--- #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 444000} #2 0x00002b3cb3c14e2a in switch_scheduler_task_thread ( thread=<value optimized out>, obj=<value optimized out>) at src/switch_scheduler.c:171 __func__ = "switch_scheduler_task_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 7 (process 8998): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 100} #2 0x00002b3cb3c054f5 in switch_core_sql_thread ( thread=<value optimized out>, obj=<value optimized out>) at src/switch_core_sqldb.c:220 ---Type <return> to continue, or q <return> to quit--- pop = (void *) 0x2aaabf3d6220 itterations = 0 trans = 0 '\0' nothing_in_queue = 1 '\001' len = 100 sql_len = 4844546 sqlbuf = 0x2aab135c7010 "" sql = <value optimized out> newlen = <value optimized out> lc = 0 __PRETTY_FUNCTION__ = "switch_core_sql_thread" __func__ = "switch_core_sql_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 6 (process 8995): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0xb64c158, data=0x40893088) at misc/apr_queue.c:276 ---Type <return> to continue, or q <return> to quit--- rv = 0 #2 0x00002b3cb3c48ff1 in log_thread (t=<value optimized out>, obj=<value optimized out>) at src/switch_log.c:288 pop = (void *) 0x0 node = (switch_log_node_t *) 0x0 binding = (switch_log_binding_t *) 0x0 __func__ = "log_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 5 (process 8951): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0x2aaaaac355a8, data=0x40bec070) at misc/apr_queue.c:276 rv = 0 #2 0x00002b3cb3c1fb14 in switch_event_thread (thread=<value optimized out>, obj=<value optimized out>) at src/switch_event.c:291 pop = (void *) 0x0 event = <value optimized out> ---Type <return> to continue, or q <return> to quit--- queue = (switch_queue_t *) 0x2aaaaac355a8 index = 0 my_id = 2 __func__ = "switch_event_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 4 (process 8950): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0x2aaaaab705a8, data=0x4060a070) at misc/apr_queue.c:276 rv = 0 #2 0x00002b3cb3c1fb14 in switch_event_thread (thread=<value optimized out>, obj=<value optimized out>) at src/switch_event.c:291 pop = (void *) 0x0 event = <value optimized out> queue = (switch_queue_t *) 0x2aaaaab705a8 index = 0 my_id = 1 ---Type <return> to continue, or q <return> to quit--- __func__ = "switch_event_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 3 (process 8949): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0xb638fa8, data=0x405ce070) at misc/apr_queue.c:276 rv = 0 #2 0x00002b3cb3c1fb14 in switch_event_thread (thread=<value optimized out>, obj=<value optimized out>) at src/switch_event.c:291 pop = (void *) 0x0 event = <value optimized out> queue = (switch_queue_t *) 0xb638fa8 index = 0 my_id = 0 __func__ = "switch_event_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. ---Type <return> to continue, or q <return> to quit--- #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 2 (process 8948): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0x2aaaaacfa5a8, data=0x40592080) at misc/apr_queue.c:276 rv = 0 #2 0x00002b3cb3c206be in switch_event_dispatch_thread ( thread=<value optimized out>, obj=<value optimized out>) at src/switch_event.c:248 pop = (void *) 0x0 event = (switch_event_t *) 0x0 queue = (switch_queue_t *) 0x2aaaaacfa5a8 my_id = 0 __func__ = "switch_event_dispatch_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
---Type <return> to continue, or q <return> to quit--- Thread 1 (process 8947): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 451000} #2 0x00002b3cb3c00c95 in pool_thread (thread=<value optimized out>, obj=<value optimized out>) at src/switch_core_memory.c:490 x = <value optimized out> #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. (gdb) (gdb) (gdb)
_______________________________________________ FreeSWITCH-users mailing list FreeSWITCH-users@lists.freeswitch.org http://lists.freeswitch.org/mailman/listinfo/freeswitch-users UNSUBSCRIBE:http://lists.freeswitch.org/mailman/options/freeswitch-users http://www.freeswitch.org
-- Anthony Minessale II FreeSWITCH http://www.freeswitch.org/ ClueCon http://www.cluecon.com/ Twitter: http://twitter.com/FreeSWITCH_wire AIM: anthm MSN:anthony_minessale@hotmail.com <MSN%3Aanthony_minessale@hotmail.com> GTALK/JABBER/PAYPAL:anthony.minessale@gmail.com<PAYPAL%3Aanthony.minessale@gmail.com> IRC: irc.freenode.net #freeswitch FreeSWITCH Developer Conference sip:888@conference.freeswitch.org <sip%3A888@conference.freeswitch.org> iax:guest@conference.freeswitch.org/888 googletalk:conf+888@conference.freeswitch.org<googletalk%3Aconf%2B888@conference.freeswitch.org> pstn:213-799-1400
Hi Juan, A similar thing happened to me. I was creating channels and bridging them with a JS script. I had to add a session.hangup(); statement at the end of the script. That solved my problem. Cheers, Nico On Fri, Nov 13, 2009 at 10:13 AM, Juan Backson <juanbackson@gmail.com>wrote:
Hi,
I am having difficulty trying to figure out why there are bunch of zombie channels in my system. It seems to me that these zombies come from apr_thread pool.
Does anyone have any idea what may be the cause of these problems?
freeswitch@internal> show channels
uuid,direction,created,created_epoch,name,state,cid_name,cid_num,ip_addr,dest,application,application_data,dialplan,context,read_codec,read_rate,write_codec,write_rate,secure b789468a-4412-490b-bc66-32f149ba4d1d,outbound,2009-11-13 20:15:35,1258114535,sofia/external/999100@192.168.1.116:9342 ,CS_REPORTING,a88999001,a88999001,192.168.1.116,999100@192.168.1.116:9342 ,,,XML,default,,,,, 7e1ecaaa-b2d8-47a0-9982-25cd44186d4e,outbound,2009-11-13 20:15:35,1258114535,sofia/external/999100@192.168.1.116:9342 ,CS_REPORTING,a88999001,a88999001,192.168.1.116,999100@192.168.1.116:9342 ,,,XML,default,,,,, 01fa2ff6-f807-4ef0-b988-70a9fe8c4536,outbound,2009-11-13 20:15:35,1258114535,sofia/external/999100@192.168.1.116:9342 ,CS_EXCHANGE_MEDIA,a88999001,a88999001,192.168.1.116, 999100@192.168.1.116:9342,incre_call_stat,125 165 182 235 13 3184093 0,XML,default,,,,, 0271541f-f0b5-482c-b05d-b196f85121be,inbound,2009-11-13 20:15:35,1258114535,sofia/external/88999001@192.168.1.116:7342 ,CS_EXECUTE,sipp,88999001,192.168.1.116,88999100,hangup,NORMAL_CLEARING,XML,default,,,,, 7e4ccfec-a4ad-4817-9a82-f1166b34576f,outbound,2009-11-13 20:15:35,1258114536,sofia/external/999100@192.168.1.116:9342 ,CS_CONSUME_MEDIA,a88999001,a88999001,192.168.1.116, 999100@192.168.1.116:9342,,,XML,default,,,,,
5 total.
freeswitch@internal> uuid_kill b789468a-4412-490b-bc66-32f149ba4d1d -ERR No Such Channel!
These channels actually do not exist in the system!
Here is my gcore output with 5 zombies out of 100K test calls :
Thread 21 (process 8946): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 128000} #2 0x00002b3cb3bfb8ca in switch_console_loop () at src/switch_console.c:819 arg = 1 thread = (switch_thread_t *) 0x2aaab00320d0 thd_attr = (switch_threadattr_t *) 0x2aaab0032070 pool = (switch_memory_pool_t *) 0x2aaab0031f88 __func__ = "switch_console_loop" __PRETTY_FUNCTION__ = "switch_console_loop" #3 0x0000000000402884 in main (argc=1, argv=<value optimized out>) at src/switch.c:753 pid_path = "/usr/local/freeswitch/log/freeswitch.pid", '\0' <repeats 219 times> pid_buffer = "8946", '\0' <repeats 27 times> old_pid_buffer = '\0' <repeats 31 times> pid_len = 4 old_pid_len = 4198811 err = 0x2b3cb3cec77d "Success" ---Type <return> to continue, or q <return> to quit--- nf = 0 runas_user = <value optimized out> runas_group = <value optimized out> nc = 0 pid = <value optimized out> x = <value optimized out> opts = <value optimized out> opts_str = '\0' <repeats 1023 times> local_argv = {0x7ffff6f08c15 "./freeswitch", 0x0 <repeats 1023 times>} arg_argv = {0x0 <repeats 128 times>} alt_dirs = 0 known_opt = <value optimized out> high_prio = 0 flags = 65 ret = <value optimized out> destroy_status = <value optimized out> fd = (switch_file_t *) 0xb6293e0 pool = (switch_memory_pool_t *) 0xb629368 rlp = {rlim_cur = 245760, rlim_max = 245760} waste = 0 __PRETTY_FUNCTION__ = "main"
Thread 20 (process 20699): ---Type <return> to continue, or q <return> to quit--- #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 0} #2 0x00002aaaab35e926 in read_packet (listener=0x2aaae7523d08, event=0x2aab3b5ab058, timeout=0) at mod_event_socket.c:1255 do_sleep = 1 '\001' mlen = 0 bytes = 0 mbuf = '\0' <repeats 2047 times> buf = '\0' <repeats 1023 times> len = 123 status = SWITCH_STATUS_BREAK count = <value optimized out> start = 1258117263 pop = (void *) 0x2aaad12f6540 ptr = 0x2aab3b5a98a0 "" crcount = 0 '\0' channel = (switch_channel_t *) 0x0 clen = <value optimized out> __func__ = "read_packet" __PRETTY_FUNCTION__ = "read_packet" ---Type <return> to continue, or q <return> to quit--- #3 0x00002aaaab36347a in listener_run (thread=<value optimized out>, obj=0x2aaae7523d08) at mod_event_socket.c:2093 listener = (listener_t *) 0x0 buf = '\0' <repeats 1023 times> len = 1024 status = <value optimized out> event = (switch_event_t *) 0x0 reply = "\000OK log level [7]", '\0' <repeats 493 times> session = (switch_core_session_t *) 0x0 channel = <value optimized out> revent = (switch_event_t *) 0x0 var = <value optimized out> __PRETTY_FUNCTION__ = "listener_run" __func__ = "listener_run" #4 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 19 (process 14505): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. ---Type <return> to continue, or q <return> to quit--- #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0x2aaaaaf49798, data=0x7afe0080) at misc/apr_queue.c:276 rv = 0 #2 0x00002b3cb3c206be in switch_event_dispatch_thread ( thread=<value optimized out>, obj=<value optimized out>) at src/switch_event.c:248 pop = (void *) 0x0 event = (switch_event_t *) 0x0 queue = (switch_queue_t *) 0x2aaaaaf49798 my_id = 1 __func__ = "switch_event_dispatch_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 18 (process 9334): #0 0x0000003054e0d2cb in read () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3cd50c8 in read_char (el=0x2aaab0028180, cp=0x4027002f "") at read.c:294 num_read = 1076297860 tried = 0 ---Type <return> to continue, or q <return> to quit--- #2 0x00002b3cb3cd4ceb in el_gets (el=0x2aaab0028180, nread=0x40270084) at read.c:241 cmdnum = 112 'p' num = -1321754256 ch = 0 '\0' #3 0x00002b3cb3bfc4bb in console_thread (thread=<value optimized out>, obj=<value optimized out>) at src/switch_console.c:464 arg = 1 count = 1 line = 0x2aaab0034e70 "\n" pool = (switch_memory_pool_t *) 0x2aaab0031f88 __func__ = "console_thread" __PRETTY_FUNCTION__ = "console_thread" #4 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 17 (process 9333): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 ---Type <return> to continue, or q <return> to quit--- tv = {tv_sec = 0, tv_usec = 0} #2 0x00002b3cb3c53895 in softtimer_runtime () at src/switch_time.c:464 current_ms = 692 x = 690 tick = 292 ts = <value optimized out> last = 1258117283599783 fwd_errs = 0 rev_errs = 0 __func__ = "softtimer_runtime" #3 0x00002b3cb3c1a347 in switch_loadable_module_exec (thread=0x0, obj=0x0) at src/switch_loadable_module.c:94 status = <value optimized out> ts = (switch_core_thread_session_t *) 0x0 module = (switch_loadable_module_t *) 0xb6c4e00 __PRETTY_FUNCTION__ = "switch_loadable_module_exec" __func__ = "switch_loadable_module_exec" #4 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #5 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 16 (process 9332): ---Type <return> to continue, or q <return> to quit--- #0 0x0000003054e0d4eb in accept () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c707a4 in apr_socket_accept (new=0x416b4020, sock=0xbcfde38, connection_context=0x2aaacda27718) at network_io/unix/sockets.c:187 No locals. #2 0x00002aaaab35f889 in mod_event_socket_runtime () at mod_event_socket.c:2324 pool = (switch_memory_pool_t *) 0xbcfdc88 listener_pool = (switch_memory_pool_t *) 0x2aaacda27718 rv = <value optimized out> sa = (switch_sockaddr_t *) 0xbcfdd68 inbound_socket = (switch_socket_t *) 0x2aaacda277f8 listener = <value optimized out> x = <value optimized out> __func__ = "mod_event_socket_runtime" #3 0x00002b3cb3c1a347 in switch_loadable_module_exec (thread=0x14f, obj=0x2aaacda27948) at src/switch_loadable_module.c:94 status = <value optimized out> ts = (switch_core_thread_session_t *) 0x2aaacda27948 module = (switch_loadable_module_t *) 0x2aaaac0058c0 __PRETTY_FUNCTION__ = "switch_loadable_module_exec" __func__ = "switch_loadable_module_exec" #4 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 ---Type <return> to continue, or q <return> to quit--- No symbol table info available. #5 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 15 (process 9330): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 55000} #2 0x00002aaab503cc4c in node_thread_run (thread=<value optimized out>, obj=<value optimized out>) at mod_fifo.c:580 val = (void *) 0x0 var = (const void *) 0x0 idle_consumers = <value optimized out> hi = (switch_hash_index_t *) 0x0 ppl_waiting = 0 consumer_total = 1087699264 node = <value optimized out> #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. ---Type <return> to continue, or q <return> to quit---
Thread 14 (process 9329): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 100} #2 0x00002aaab44d77be in sofia_profile_worker_thread_run ( thread=<value optimized out>, obj=<value optimized out>) at sofia.c:763 profile = (sofia_profile_t *) 0xbce2310 ireg_loops = 18 gateway_loops = 0 loops = 72 qsize = 4294966782 pop = (void *) 0x0 __PRETTY_FUNCTION__ = "sofia_profile_worker_thread_run" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 13 (process 9328): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 ---Type <return> to continue, or q <return> to quit--- No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 0} #2 0x00002aaab44d77be in sofia_profile_worker_thread_run ( thread=<value optimized out>, obj=<value optimized out>) at sofia.c:763 profile = (sofia_profile_t *) 0x2aaab000eb10 ireg_loops = 5 gateway_loops = 0 loops = 93 qsize = 4294966782 pop = (void *) 0x0 __PRETTY_FUNCTION__ = "sofia_profile_worker_thread_run" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 12 (process 9327): #0 0x00000030542d3368 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x00002aaab45c9c9c in su_epoll_port_wait_events (self=0xbce71c0, tout=1000) at su_epoll_port.c:495 ---Type <return> to continue, or q <return> to quit--- j = 198076976 n = 0 events = 0 index = 10922 version = 3 M = 4 ev = 0x41204ef0 __PRETTY_FUNCTION__ = "su_epoll_port_wait_events" #2 0x00002aaab45d1079 in su_base_port_run (self=0xbce71c0) at su_base_port.c:349 tout = 1000 tout2 = 0 __PRETTY_FUNCTION__ = "su_base_port_run" #3 0x00002aaab45c6c51 in su_port_run (self=0xbce71c0) at su_port.h:326 base = (su_virtual_port_t *) 0xbce71c0 #4 0x00002aaab45c6c29 in su_root_run (self=0xbce72a0) at su_root.c:819 __PRETTY_FUNCTION__ = "su_root_run" #5 0x00002aaab45d8d58 in su_pthread_port_clone_main (varg=0x404f7ac0) at su_pthread_port.c:324 arg = (struct clone_args *) 0x0 task = {{sut_port = 0xbce71c0, sut_root = 0xbce72a0}} zap = 1 #6 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 ---Type <return> to continue, or q <return> to quit--- No symbol table info available. #7 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 11 (process 9326): #0 0x00000030542d3368 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x00002aaab45c9c9c in su_epoll_port_wait_events (self=0xbce78b0, tout=1000) at su_epoll_port.c:495 j = -1342070512 n = 10922 events = 0 index = 10922 version = 3 M = 4 ev = 0x411c8ef0 __PRETTY_FUNCTION__ = "su_epoll_port_wait_events" #2 0x00002aaab45d1079 in su_base_port_run (self=0xbce78b0) at su_base_port.c:349 tout = 1000 tout2 = 0 __PRETTY_FUNCTION__ = "su_base_port_run" #3 0x00002aaab45c6c51 in su_port_run (self=0xbce78b0) at su_port.h:326 ---Type <return> to continue, or q <return> to quit--- base = (su_virtual_port_t *) 0xbce78b0 #4 0x00002aaab45c6c29 in su_root_run (self=0x2aaab001a060) at su_root.c:819 __PRETTY_FUNCTION__ = "su_root_run" #5 0x00002aaab45d8d58 in su_pthread_port_clone_main (varg=0x404bbac0) at su_pthread_port.c:324 arg = (struct clone_args *) 0x0 task = {{sut_port = 0xbce78b0, sut_root = 0x2aaab001a060}} zap = 1 #6 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #7 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 10 (process 9325): #0 0x00000030542d3368 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x00002aaab45c9c9c in su_epoll_port_wait_events (self=0xbce6c30, tout=1000) at su_epoll_port.c:495 j = -1268971119 n = 10922 events = 0 index = 0 version = 1 ---Type <return> to continue, or q <return> to quit--- M = 4 ev = 0x404f7c40 __PRETTY_FUNCTION__ = "su_epoll_port_wait_events" #2 0x00002aaab45d11d4 in su_base_port_step (self=0xbce6c30, tout=1000) at su_base_port.c:467 now = {tv_sec = 3467106082, tv_usec = 971475} __PRETTY_FUNCTION__ = "su_base_port_step" #3 0x00002aaab45c6d6a in su_port_step (self=0xbce6c30, tout=1000) at su_port.h:340 base = (su_virtual_port_t *) 0xbce6c30 #4 0x00002aaab45c6d32 in su_root_step (self=0xbce4650, tout=1000) at su_root.c:858 __PRETTY_FUNCTION__ = "su_root_step" #5 0x00002aaab44e5c3a in sofia_profile_thread_run ( thread=<value optimized out>, obj=<value optimized out>) at sofia.c:973 profile = (sofia_profile_t *) 0xbce2310 pool = <value optimized out> node = (sip_alias_node_t *) 0x0 s_event = (switch_event_t *) 0x0 sanity = <value optimized out> worker_thread = (switch_thread_t *) 0xbce36a0 st = SWITCH_STATUS_SUCCESS __func__ = "sofia_profile_thread_run" ---Type <return> to continue, or q <return> to quit--- __PRETTY_FUNCTION__ = "sofia_profile_thread_run" #6 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #7 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 9 (process 9324): #0 0x00000030542d3368 in epoll_wait () from /lib64/libc.so.6 No symbol table info available. #1 0x00002aaab45c9c9c in su_epoll_port_wait_events (self=0xbcdffb0, tout=1000) at su_epoll_port.c:495 j = -1268971119 n = 10922 events = 0 index = 0 version = 1 M = 4 ev = 0x404bbc40 __PRETTY_FUNCTION__ = "su_epoll_port_wait_events" #2 0x00002aaab45d11d4 in su_base_port_step (self=0xbcdffb0, tout=1000) at su_base_port.c:467 now = {tv_sec = 3467106083, tv_usec = 525146} __PRETTY_FUNCTION__ = "su_base_port_step" ---Type <return> to continue, or q <return> to quit--- #3 0x00002aaab45c6d6a in su_port_step (self=0xbcdffb0, tout=1000) at su_port.h:340 base = (su_virtual_port_t *) 0xbcdffb0 #4 0x00002aaab45c6d32 in su_root_step (self=0xbcdfe00, tout=1000) at su_root.c:858 __PRETTY_FUNCTION__ = "su_root_step" #5 0x00002aaab44e5c3a in sofia_profile_thread_run ( thread=<value optimized out>, obj=<value optimized out>) at sofia.c:973 profile = (sofia_profile_t *) 0x2aaab000eb10 pool = <value optimized out> node = (sip_alias_node_t *) 0x0 s_event = (switch_event_t *) 0x0 sanity = <value optimized out> worker_thread = (switch_thread_t *) 0x2aaab000fea0 st = SWITCH_STATUS_SUCCESS __func__ = "sofia_profile_thread_run" __PRETTY_FUNCTION__ = "sofia_profile_thread_run" #6 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #7 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 8 (process 8999): ---Type <return> to continue, or q <return> to quit--- #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 444000} #2 0x00002b3cb3c14e2a in switch_scheduler_task_thread ( thread=<value optimized out>, obj=<value optimized out>) at src/switch_scheduler.c:171 __func__ = "switch_scheduler_task_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 7 (process 8998): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 100} #2 0x00002b3cb3c054f5 in switch_core_sql_thread ( thread=<value optimized out>, obj=<value optimized out>) at src/switch_core_sqldb.c:220 ---Type <return> to continue, or q <return> to quit--- pop = (void *) 0x2aaabf3d6220 itterations = 0 trans = 0 '\0' nothing_in_queue = 1 '\001' len = 100 sql_len = 4844546 sqlbuf = 0x2aab135c7010 "" sql = <value optimized out> newlen = <value optimized out> lc = 0 __PRETTY_FUNCTION__ = "switch_core_sql_thread" __func__ = "switch_core_sql_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 6 (process 8995): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0xb64c158, data=0x40893088) at misc/apr_queue.c:276 ---Type <return> to continue, or q <return> to quit--- rv = 0 #2 0x00002b3cb3c48ff1 in log_thread (t=<value optimized out>, obj=<value optimized out>) at src/switch_log.c:288 pop = (void *) 0x0 node = (switch_log_node_t *) 0x0 binding = (switch_log_binding_t *) 0x0 __func__ = "log_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 5 (process 8951): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0x2aaaaac355a8, data=0x40bec070) at misc/apr_queue.c:276 rv = 0 #2 0x00002b3cb3c1fb14 in switch_event_thread (thread=<value optimized out>, obj=<value optimized out>) at src/switch_event.c:291 pop = (void *) 0x0 event = <value optimized out> ---Type <return> to continue, or q <return> to quit--- queue = (switch_queue_t *) 0x2aaaaac355a8 index = 0 my_id = 2 __func__ = "switch_event_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 4 (process 8950): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0x2aaaaab705a8, data=0x4060a070) at misc/apr_queue.c:276 rv = 0 #2 0x00002b3cb3c1fb14 in switch_event_thread (thread=<value optimized out>, obj=<value optimized out>) at src/switch_event.c:291 pop = (void *) 0x0 event = <value optimized out> queue = (switch_queue_t *) 0x2aaaaab705a8 index = 0 my_id = 1 ---Type <return> to continue, or q <return> to quit--- __func__ = "switch_event_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 3 (process 8949): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0xb638fa8, data=0x405ce070) at misc/apr_queue.c:276 rv = 0 #2 0x00002b3cb3c1fb14 in switch_event_thread (thread=<value optimized out>, obj=<value optimized out>) at src/switch_event.c:291 pop = (void *) 0x0 event = <value optimized out> queue = (switch_queue_t *) 0xb638fa8 index = 0 my_id = 0 __func__ = "switch_event_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. ---Type <return> to continue, or q <return> to quit--- #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
Thread 2 (process 8948): #0 0x0000003054e0a899 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 No symbol table info available. #1 0x00002b3cb3c63b42 in apr_queue_pop (queue=0x2aaaaacfa5a8, data=0x40592080) at misc/apr_queue.c:276 rv = 0 #2 0x00002b3cb3c206be in switch_event_dispatch_thread ( thread=<value optimized out>, obj=<value optimized out>) at src/switch_event.c:248 pop = (void *) 0x0 event = (switch_event_t *) 0x0 queue = (switch_queue_t *) 0x2aaaaacfa5a8 my_id = 0 __func__ = "switch_event_dispatch_thread" #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available.
---Type <return> to continue, or q <return> to quit--- Thread 1 (process 8947): #0 0x00000030542cc4c2 in select () from /lib64/libc.so.6 No symbol table info available. #1 0x00002b3cb3c72df5 in apr_sleep (t=<value optimized out>) at time/unix/time.c:246 tv = {tv_sec = 0, tv_usec = 451000} #2 0x00002b3cb3c00c95 in pool_thread (thread=<value optimized out>, obj=<value optimized out>) at src/switch_core_memory.c:490 x = <value optimized out> #3 0x0000003054e06367 in start_thread () from /lib64/libpthread.so.0 No symbol table info available. #4 0x00000030542d2f7d in clone () from /lib64/libc.so.6 No symbol table info available. (gdb) (gdb) (gdb)
_______________________________________________ FreeSWITCH-users mailing list FreeSWITCH-users@lists.freeswitch.org http://lists.freeswitch.org/mailman/listinfo/freeswitch-users UNSUBSCRIBE:http://lists.freeswitch.org/mailman/options/freeswitch-users http://www.freeswitch.org
participants (3)
-
Anthony Minessale
-
Juan Backson
-
Nicolas Brenner