We recently upgraded from 2.0.4 to 2.1.8 and are now noticing occasional segfaults when handling received auth packets. Representative backtraces are below. In all cases, all threads are idle except one, which is receiving an auth packet. In the first case, auth_socket_recv() passes a NULL packet to received_request(), which is strange since auth_socket_recv() checks for that case immediately before. In the second case, received_request() gets a bogus pointer to the packet, apparently from rad_recv(). I'm always hesitant to trust backtraces from optimized binaries, but the code paths relative to the packet pointers being passed around are bizarre and strike me as stack or heap corruption. Any ideas? john Program received signal SIGSEGV, Segmentation fault. [Switching to Thread 0x7f71e3b32ae0 (LWP 7677)] fr_packet_cmp (a=0x7f7179f94070, b=0x0) at packet.c:139 139 packet.c: No such file or directory. in packet.c Thread 161 (Thread 0x7f71067fc950 (LWP 12500)): #0 0x00007f71e2d5abd1 in sem_wait () from /lib/libpthread.so.0 No symbol table info available. #1 0x000000000041e1bc in request_handler_thread (arg=<value optimized out>) at threads.c:453 fun = (RAD_REQUEST_FUNP) 0x408a50 <rad_accounting> self = (THREAD_HANDLE *) 0x7f71d4a33ea0 #2 0x00007f71e2d54fc7 in start_thread () from /lib/libpthread.so.0 No symbol table info available. #3 0x00007f71e209f59d in clone () from /lib/libc.so.6 No symbol table info available. #4 0x0000000000000000 in ?? () No symbol table info available. [snip remaining idle request handler threads] Thread 1 (Thread 0x7f71e3b32ae0 (LWP 7677)): #0 fr_packet_cmp (a=0x7f7179f94070, b=0x0) at packet.c:139 rcode = <value optimized out> #1 0x00007f71e339fac4 in fr_hash_table_find (ht=0x8c1200, data=0x7fffc7ed1a30) at hash.c:191 key = <value optimized out> entry = <value optimized out> reversed = 3663427046 #2 0x00007f71e339fb09 in fr_hash_table_finddata (ht=0x7f7179f94070, data=0x0) at hash.c:491 node = <value optimized out> #3 0x00007f71e33ad4cb in fr_packet_list_find (pl=<value optimized out>, request=0x7f7179f94070) at packet.c:581 No locals. #4 0x0000000000427b49 in received_request (listener=0x15505d0, packet=0x0, prequest=0x5b0000, client=0x7f71e33b0fc0) at event.c:2775 packet_p = <value optimized out> request = (REQUEST *) 0x0 #5 0x0000000000415ac6 in auth_socket_recv (listener=0x15505d0, pfun=0x7fffc7ed1b98, prequest=0x7fffc7ed1b90) at listen.c:820 rcode = <value optimized out> code = 1 src_port = 21682 packet = (RADIUS_PACKET *) 0x7f7179f94070 fun = (RAD_REQUEST_FUNP) 0x409020 <rad_authenticate> client = (RADCLIENT *) 0xa9b4b0 src_ipaddr = {af = 2, ipaddr = {ip4addr = {s_addr = 1342711882}, ip6addr = {in6_u = {u6_addr8 = "J(\bP�*&�p\177\000\000�\001\\\001", u6_addr16 = {10314, 20488, 10976, 64550, 32624, 0, 448, 348}, u6_addr32 = {1342711882, 4230359776, 32624, 22806976}}}}} #6 0x0000000000422ffe in event_socket_handler (xel=<value optimized out>, fd=<value optimized out>, ctx=<value optimized out>) at event.c:3347 listener = (rad_listen_t *) 0x15505d0 fun = <value optimized out> request = <value optimized out> #7 0x00007f71e33ae766 in fr_event_loop (el=0x15c01c0) at event.c:412 i = 1 rcode = <value optimized out> maxfd = 21 when = {tv_sec = 1264965841, tv_usec = 427558} wake = <value optimized out> read_fds = {fds_bits = {524288, 0 <repeats 15 times>}} master_fds = {fds_bits = {3670024, 0 <repeats 15 times>}} #8 0x000000000041c377 in main (argc=1, argv=<value optimized out>) at radiusd.c:398 rcode = 22599360 argval = <value optimized out> spawn_flag = 1 dont_fork = 0 flag = <value optimized out> act = {__sigaction_handler = {sa_handler = 0x41c600 <sig_fatal>, sa_sigaction = 0x41c600 <sig_fatal>}, sa_mask = {__val = {0 <repeats 16 times>}}, sa_flags = 0, sa_restorer = 0} Program received signal SIGSEGV, Segmentation fault. [Switching to Thread 0x7f90c1422ae0 (LWP 11822)] fr_packet_cmp (a=0x7f90213b6ee0, b=0x144) at packet.c:139 139 packet.c: No such file or directory. in packet.c Thread 162 (Thread 0x7f8f97fe6950 (LWP 12010)): #0 0x00007f90c064abd1 in sem_wait () from /lib/libpthread.so.0 No symbol table info available. #1 0x000000000041e1bc in request_handler_thread (arg=<value optimized out>) at threads.c:453 fun = (RAD_REQUEST_FUNP) 0x409020 <rad_authenticate> self = (THREAD_HANDLE *) 0x7f9050c65670 #2 0x00007f90c0644fc7 in start_thread () from /lib/libpthread.so.0 No symbol table info available. #3 0x00007f90bf98f59d in clone () from /lib/libc.so.6 No symbol table info available. #4 0x0000000000000000 in ?? () No symbol table info available. [snip remaining idle request handler threads] Thread 1 (Thread 0x7f90c1422ae0 (LWP 11822)): #0 fr_packet_cmp (a=0x7f90213b6ee0, b=0x144) at packet.c:139 rcode = <value optimized out> #1 0x00007f90c0c8fac4 in fr_hash_table_find (ht=0xf64ce0, data=0x7fff3e4a1110) at hash.c:191 key = <value optimized out> entry = <value optimized out> reversed = 1832451347 #2 0x00007f90c0c8fb09 in fr_hash_table_finddata (ht=0x7f90213b6ee0, data=0x144) at hash.c:491 node = <value optimized out> #3 0x00007f90c0c9d4cb in fr_packet_list_find (pl=<value optimized out>, request=0x7f90213b6ee0) at packet.c:581 No locals. #4 0x0000000000427b49 in received_request (listener=0xf623d0, packet=0x144, prequest=0x380000, client=0x7f90c0ca0fc0) at event.c:2775 packet_p = <value optimized out> request = (REQUEST *) 0x0 #5 0x0000000000415ac6 in auth_socket_recv (listener=0xf623d0, pfun=0x7fff3e4a1278, prequest=0x7fff3e4a1270) at listen.c:820 rcode = <value optimized out> code = 1 src_port = 49156 packet = (RADIUS_PACKET *) 0x7f90213b6ee0 fun = (RAD_REQUEST_FUNP) 0x409020 <rad_authenticate> client = (RADCLIENT *) 0x113b8e0 src_ipaddr = {af = 2, ipaddr = {ip4addr = {s_addr = 2061970506}, ip6addr = {in6_u = {u6_addr8 = "J,�z�\0259!\220\177\000\000 \220@", u6_addr16 = {11338, 31463, 5568, 8505, 32656, 0, 36896, 64}, u6_addr32 = {2061970506, 557389248, 32656, 4231200}}}}} #6 0x0000000000422ffe in event_socket_handler (xel=<value optimized out>, fd=<value optimized out>, ctx=<value optimized out>) at event.c:3347 listener = (rad_listen_t *) 0xf623d0 fun = <value optimized out> request = <value optimized out> #7 0x00007f90c0c9e766 in fr_event_loop (el=0xf5eab0) at event.c:412 i = 1 rcode = <value optimized out> maxfd = 21 when = {tv_sec = 1264551371, tv_usec = 36937} wake = <value optimized out> read_fds = {fds_bits = {524288, 0 <repeats 15 times>}} master_fds = {fds_bits = {3670024, 0 <repeats 15 times>}} #8 0x000000000041c377 in main (argc=1, argv=<value optimized out>) at radiusd.c:398 rcode = 29613264 argval = <value optimized out> spawn_flag = 1 dont_fork = 0 flag = <value optimized out> act = {__sigaction_handler = {sa_handler = 0x41c600 <sig_fatal>, sa_sigaction = 0x41c600 <sig_fatal>}, sa_mask = {__val = {0 <repeats 16 times>}}, sa_flags = 0, sa_restorer = 0} -- John Morrissey _o /\ ---- __o jwm@horde.net _-< \_ / \ ---- < \, www.horde.net/ __(_)/_(_)________/ \_______(_) /_(_)__