// Turned on debug level on one of the two affected systems:

# gzcat stunnel.3
...
2023.11.24 02:40:25 LOG6[358]: s_connect: connected 127.0.0.1:2049
2023.11.24 02:40:25 LOG6[358]: persistence: 127.0.0.1:2049 cached
2023.11.24 02:40:25 LOG5[358]: Service [tls-nfs-srv] connected remote server from 127.0.0.1:61144
2023.11.24 02:40:25 LOG7[358]: Setting remote socket options (FD=27)
2023.11.24 02:40:25 LOG7[358]: Option TCP_NODELAY set on remote socket
2023.11.24 02:40:25 LOG7[358]: Remote descriptor (FD=27) initialized
2023.11.24 02:50:03 LOG7[358]: Remove session callback
2023.11.24 02:50:03 LOG7[358]: TLS alert (write): fatal: bad record mac
2023.11.24 02:50:03 LOG3[358]: SSL_read: s3_pkt.c:534: error:1408F119:SSL routines:SSL3_GET_RECORD:decryption failed or bad record mac
2023.11.24 02:50:03 LOG5[358]: Connection reset/closed: 1340 byte(s) sent to TLS, 1860 byte(s) sent to socket
2023.11.24 02:50:03 LOG7[358]: Deallocating application specific data for session connect address
2023.11.24 02:50:03 LOG7[358]: remote_fd reset (FD=27)
2023.11.24 02:50:03 LOG7[358]: Remote descriptor (FD=27) closed
2023.11.24 02:50:03 LOG7[358]: Local descriptor (FD=26) closed
2023.11.24 02:50:03 LOG7[358]: Service [tls-nfs-srv] finished (7 left)
2023.11.24 02:50:03 LOG7[357]: TLS alert (read): fatal: bad record mac
2023.11.24 02:50:03 LOG7[357]: Remove session callback
2023.11.24 02:50:03 LOG3[357]: SSL_read: s3_pkt.c:1498: error:140943FC:SSL routines:ssl3_read_bytes:sslv3 alert bad record mac
2023.11.24 02:50:03 LOG5[357]: Connection reset/closed: 4044 byte(s) sent to TLS, 5356 byte(s) sent to socket
2023.11.24 02:50:03 LOG7[357]: Deallocating application specific data for session connect address
2023.11.24 02:50:03 LOG7[357]: remote_fd reset (FD=25)
2023.11.24 02:50:03 LOG7[357]: Remote descriptor (FD=25) closed
2023.11.24 02:50:03 LOG7[357]: Local descriptor (FD=24) closed
2023.11.24 02:50:03 LOG7[357]: Service [tls-nfs-srv] finished (6 left)
2023.11.24 02:50:13 LOG7[main]: Found 1 ready file descriptor(s)
2023.11.24 02:50:13 LOG7[main]: FD=4 events=0x1 revents=0x0
2023.11.24 02:50:13 LOG7[main]: FD=8 events=0x1 revents=0x0
2023.11.24 02:50:13 LOG7[main]: FD=9 events=0x1 revents=0x1
2023.11.24 02:50:13 LOG7[main]: FD=15 events=0x1 revents=0x0
2023.11.24 02:50:13 LOG7[main]: FD=16 events=0x1 revents=0x0
2023.11.24 02:50:13 LOG7[main]: FD=17 events=0x1 revents=0x0
2023.11.24 02:50:13 LOG7[main]: Service [tls-nfs-srv] accepted (FD=24) from 1.2.3.5:37532
2023.11.24 02:50:13 LOG7[359]: Service [tls-nfs-srv] started
2023.11.24 02:50:13 LOG7[359]: Setting local socket options (FD=24)
2023.11.24 02:50:13 LOG7[359]: Option TCP_NODELAY set on local socket
2023.11.24 02:50:13 LOG7[359]: Waiting for a libwrap process
2023.11.24 02:50:13 LOG7[359]: Acquired libwrap process #3
2023.11.24 02:50:13 LOG7[359]: Releasing libwrap process #3
2023.11.24 02:50:13 LOG7[359]: Released libwrap process #3
2023.11.24 02:50:13 LOG7[359]: Service [tls-nfs-srv] permitted by libwrap from 1.2.3.5:37532
2023.11.24 02:50:13 LOG5[359]: Service [tls-nfs-srv] accepted connection from 1.2.3.5:37532
2023.11.24 02:50:13 LOG6[359]: Peer certificate not required
2023.11.24 02:50:13 LOG7[359]: TLS state (accept): before/accept initialization
2023.11.24 02:50:13 LOG7[359]: Get session callback
2023.11.24 02:50:13 LOG7[359]: Initializing application specific data for session authenticated
2023.11.24 02:50:13 LOG7[359]: SNI: no virtual services defined
2023.11.24 02:50:13 LOG7[359]: TLS state (accept): SSLv3 read client hello A
2023.11.24 02:50:13 LOG7[359]: TLS state (accept): SSLv3 write server hello A
2023.11.24 02:50:13 LOG7[359]: TLS state (accept): SSLv3 write server done A
2023.11.24 02:50:13 LOG7[359]: TLS state (accept): SSLv3 flush data
2023.11.24 02:50:13 LOG7[359]: TLS state (accept): SSLv3 read client certificate A
2023.11.24 02:50:13 LOG5[359]: Key configured for PSK identity "aaaaa"
2023.11.24 02:50:13 LOG7[359]: TLS state (accept): SSLv3 read client key exchange A
2023.11.24 02:50:13 LOG7[359]: TLS state (accept): SSLv3 read certificate verify A
2023.11.24 02:50:13 LOG7[359]: TLS state (accept): SSLv3 read finished A
2023.11.24 02:50:13 LOG7[359]: TLS state (accept): SSLv3 write change cipher spec A
2023.11.24 02:50:13 LOG7[359]: TLS state (accept): SSLv3 write finished A
2023.11.24 02:50:13 LOG7[359]: TLS state (accept): SSLv3 flush data
2023.11.24 02:50:13 LOG7[359]: New session callback
2023.11.24 02:50:13 LOG6[359]: No peer certificate received
2023.11.24 02:50:13 LOG6[359]: Session id: 6CF18FF7EC3D4A17E76FE797DB352E5F2E885F5E8191F7A16E4B1E3137FA98DA
2023.11.24 02:50:13 LOG7[359]:    354 server accept(s) requested
2023.11.24 02:50:13 LOG7[359]:    351 server accept(s) succeeded
2023.11.24 02:50:13 LOG7[359]:      0 server renegotiation(s) requested
2023.11.24 02:50:13 LOG7[359]:      0 session reuse(s)
2023.11.24 02:50:13 LOG7[359]:      0 internal session cache item(s)
2023.11.24 02:50:13 LOG7[359]:      0 internal session cache fill-up(s)
2023.11.24 02:50:13 LOG7[359]:    303 internal session cache miss(es)
2023.11.24 02:50:13 LOG7[359]:      0 external session cache hit(s)
2023.11.24 02:50:13 LOG7[359]:     51 expired session(s) retrieved
2023.11.24 02:50:13 LOG6[359]: TLS accepted: new session negotiated
2023.11.24 02:50:13 LOG6[359]: TLSv1.2 ciphersuite: PSK-AES256-CBC-SHA (256-bit encryption)
2023.11.24 02:50:13 LOG7[359]: Compression: null, expansion: null
2023.11.24 02:50:13 LOG6[359]: s_connect: connecting 127.0.0.1:2049
2023.11.24 02:50:13 LOG6[359]: s_connect: connected 127.0.0.1:2049
2023.11.24 02:50:13 LOG6[359]: persistence: 127.0.0.1:2049 cached
2023.11.24 02:50:13 LOG5[359]: Service [tls-nfs-srv] connected remote server from 127.0.0.1:61251
2023.11.24 02:50:13 LOG7[359]: Setting remote socket options (FD=25)
2023.11.24 02:50:13 LOG7[359]: Option TCP_NODELAY set on remote socket
2023.11.24 02:50:13 LOG7[359]: Remote descriptor (FD=25) initialized
2023.11.24 02:50:13 LOG7[main]: Found 1 ready file descriptor(s)
2023.11.24 02:50:13 LOG7[main]: FD=4 events=0x1 revents=0x0
2023.11.24 02:50:13 LOG7[main]: FD=8 events=0x1 revents=0x0
2023.11.24 02:50:13 LOG7[main]: FD=9 events=0x1 revents=0x1
2023.11.24 02:50:13 LOG7[main]: FD=15 events=0x1 revents=0x0
2023.11.24 02:50:13 LOG7[main]: FD=16 events=0x1 revents=0x0
2023.11.24 02:50:13 LOG7[main]: FD=17 events=0x1 revents=0x0
2023.11.24 02:50:13 LOG7[main]: Service [tls-nfs-srv] accepted (FD=26) from 1.2.3.6:50653
2023.11.24 02:50:13 LOG7[360]: Service [tls-nfs-srv] started
2023.11.24 02:50:13 LOG7[360]: Setting local socket options (FD=26)
2023.11.24 02:50:13 LOG7[360]: Option TCP_NODELAY set on local socket
2023.11.24 02:50:13 LOG7[360]: Waiting for a libwrap process
2023.11.24 02:50:13 LOG7[360]: Acquired libwrap process #3
2023.11.24 02:50:13 LOG7[360]: Releasing libwrap process #3
2023.11.24 02:50:13 LOG7[360]: Released libwrap process #3
2023.11.24 02:50:13 LOG7[360]: Service [tls-nfs-srv] permitted by libwrap from 1.2.3.6:50653
2023.11.24 02:50:13 LOG5[360]: Service [tls-nfs-srv] accepted connection from 1.2.3.6:50653
2023.11.24 02:50:13 LOG6[360]: Peer certificate not required
2023.11.24 02:50:13 LOG7[360]: TLS state (accept): before/accept initialization
2023.11.24 02:50:13 LOG7[360]: Get session callback
2023.11.24 02:50:13 LOG7[360]: Initializing application specific data for session authenticated
2023.11.24 02:50:13 LOG7[360]: SNI: no virtual services defined
2023.11.24 02:50:13 LOG7[360]: TLS state (accept): SSLv3 read client hello A
2023.11.24 02:50:13 LOG7[360]: TLS state (accept): SSLv3 write server hello A
2023.11.24 02:50:13 LOG7[360]: TLS state (accept): SSLv3 write server done A
2023.11.24 02:50:13 LOG7[360]: TLS state (accept): SSLv3 flush data
2023.11.24 02:50:13 LOG7[360]: TLS state (accept): SSLv3 read client certificate A
2023.11.24 02:50:13 LOG5[360]: Key configured for PSK identity "bbbbbb"
2023.11.24 02:50:13 LOG7[360]: TLS state (accept): SSLv3 read client key exchange A
2023.11.24 02:50:13 LOG7[360]: TLS state (accept): SSLv3 read certificate verify A
2023.11.24 02:50:13 LOG7[360]: TLS state (accept): SSLv3 read finished A
2023.11.24 02:50:13 LOG7[360]: TLS state (accept): SSLv3 write change cipher spec A
2023.11.24 02:50:13 LOG7[360]: TLS state (accept): SSLv3 write finished A
2023.11.24 02:50:13 LOG7[360]: TLS state (accept): SSLv3 flush data
2023.11.24 02:50:13 LOG7[360]: New session callback
2023.11.24 02:50:13 LOG6[360]: No peer certificate received
2023.11.24 02:50:13 LOG6[360]: Session id: EC96B93ABB1381A8B39B45783C95E606A5D80A49FAE931D32E32D1CF88BD3123
2023.11.24 02:50:13 LOG7[360]:    355 server accept(s) requested
2023.11.24 02:50:13 LOG7[360]:    352 server accept(s) succeeded
2023.11.24 02:50:13 LOG7[360]:      0 server renegotiation(s) requested
2023.11.24 02:50:13 LOG7[360]:      0 session reuse(s)
2023.11.24 02:50:13 LOG7[360]:      1 internal session cache item(s)
2023.11.24 02:50:13 LOG7[360]:      0 internal session cache fill-up(s)
2023.11.24 02:50:13 LOG7[360]:    304 internal session cache miss(es)
2023.11.24 02:50:13 LOG7[360]:      0 external session cache hit(s)
2023.11.24 02:50:13 LOG7[360]:     51 expired session(s) retrieved
2023.11.24 02:50:13 LOG6[360]: TLS accepted: new session negotiated
2023.11.24 02:50:13 LOG6[360]: TLSv1.2 ciphersuite: PSK-AES256-CBC-SHA (256-bit encryption)
2023.11.24 02:50:13 LOG7[360]: Compression: null, expansion: null
2023.11.24 02:50:13 LOG6[360]: s_connect: connecting 127.0.0.1:2049
2023.11.24 02:50:13 LOG6[360]: s_connect: connected 127.0.0.1:2049
2023.11.24 02:50:13 LOG6[360]: persistence: 127.0.0.1:2049 cached
2023.11.24 02:50:13 LOG5[360]: Service [tls-nfs-srv] connected remote server from 127.0.0.1:61253
2023.11.24 02:50:13 LOG7[360]: Setting remote socket options (FD=27)
2023.11.24 02:50:13 LOG7[360]: Option TCP_NODELAY set on remote socket
2023.11.24 02:50:13 LOG7[360]: Remote descriptor (FD=27) initialized
2023.11.24 03:10:00 LOG7[main]: Found 1 ready file descriptor(s)
2023.11.24 03:10:00 LOG7[main]: FD=4 events=0x1 revents=0x1
2023.11.24 03:10:00 LOG7[main]: FD=8 events=0x1 revents=0x0
2023.11.24 03:10:00 LOG7[main]: FD=9 events=0x1 revents=0x0
2023.11.24 03:10:00 LOG7[main]: FD=15 events=0x1 revents=0x0
2023.11.24 03:10:00 LOG7[main]: FD=16 events=0x1 revents=0x0
2023.11.24 03:10:00 LOG7[main]: FD=17 events=0x1 revents=0x0
2023.11.24 03:10:00 LOG7[main]: Dispatching a signal from the signal pipe
2023.11.24 03:10:00 LOG7[main]: Processing SIGNAL_RELOAD_CONFIG
2023.11.24 03:10:00 LOG6[main]: Initializing inetd mode configuration
2023.11.24 03:10:00 LOG5[main]: Reading configuration from file /etc/opt/csw/stunnel/stunnel.conf
2023.11.24 03:10:00 LOG5[main]: UTF-8 byte order mark not detected
2023.11.24 03:10:00 LOG5[main]: FIPS mode disabled
2023.11.24 03:10:00 LOG6[main]: Compression disabled
...
2023.11.24 04:18:25 LOG7[371]: TLS state (accept): SSLv3 write finished A
2023.11.24 04:18:25 LOG7[371]: TLS state (accept): SSLv3 flush data
2023.11.24 04:18:25 LOG7[371]: New session callback
2023.11.24 04:18:25 LOG6[371]: No peer certificate received
2023.11.24 04:18:25 LOG6[371]: Session id: 8B0E89AA955F2AA817DBC16B0FA35420DA3C729C00A06BBA868FFE94C082D11E
2023.11.24 04:18:25 LOG7[371]:    366 server accept(s) requested
2023.11.24 04:18:25 LOG7[371]:    363 server accept(s) succeeded
2023.11.24 04:18:25 LOG7[371]:      0 server renegotiation(s) requested
2023.11.24 04:18:25 LOG7[371]:      0 session reuse(s)
2023.11.24 04:18:25 LOG7[371]:      1 internal session cache item(s)
2023.11.24 04:18:25 LOG7[371]:      0 internal session cache fill-up(s)
2023.11.24 04:18:25 LOG7[371]:    315 internal session cache miss(es)
2023.11.24 04:18:25 LOG7[371]:      0 external session cache hit(s)
2023.11.24 04:18:25 LOG7[371]:     51 expired session(s) retrieved
2023.11.24 04:18:25 LOG6[371]: TLS accepted: new session negotiated
2023.11.24 04:18:25 LOG6[371]: TLSv1.2 ciphersuite: PSK-AES256-CBC-SHA (256-bit encryption)
2023.11.24 04:18:25 LOG7[371]: Compression: null, expansion: null
2023.11.24 04:18:25 LOG6[371]: s_connect: connecting 127.0.0.1:2049
2023.11.24 04:18:25 LOG6[371]: s_connect: connected 127.0.0.1:2049
2023.11.24 04:18:25 LOG6[371]: persistence: 127.0.0.1:2049 cached
2023.11.24 04:18:25 LOG5[371]: Service [tls-nfs-srv] connected remote server from 127.0.0.1:62265
2023.11.24 04:18:25 LOG7[371]: Setting remote socket options (FD=25)
2023.11.24 04:18:25 LOG7[371]: Option TCP_NODELAY set on remote socket
2023.11.24 04:18:25 LOG7[371]: Remote descriptor (FD=25) initialized


# gzcat stunnel.3 | egrep 's3_pkt.c|ERROR|Initializing inetd' 
...
2023.11.24 01:56:09 LOG3[349]: SSL_read: s3_pkt.c:1498: error:140943FC:SSL routines:ssl3_read_bytes:sslv3 alert bad record mac
2023.11.24 02:10:51 LOG3[355]: SSL_read: s3_pkt.c:534: error:1408F119:SSL routines:SSL3_GET_RECORD:decryption failed or bad record mac
2023.11.24 02:15:45 LOG3[356]: SSL_read: s3_pkt.c:534: error:1408F119:SSL routines:SSL3_GET_RECORD:decryption failed or bad record mac
2023.11.24 02:40:15 LOG3[354]: SSL_read: s3_pkt.c:534: error:1408F119:SSL routines:SSL3_GET_RECORD:decryption failed or bad record mac
2023.11.24 02:50:03 LOG3[358]: SSL_read: s3_pkt.c:534: error:1408F119:SSL routines:SSL3_GET_RECORD:decryption failed or bad record mac
2023.11.24 02:50:03 LOG3[357]: SSL_read: s3_pkt.c:1498: error:140943FC:SSL routines:ssl3_read_bytes:sslv3 alert bad record mac
2023.11.24 03:10:00 LOG6[main]: Initializing inetd mode configuration
2023.11.24 03:14:33 LOG3[360]: SSL_read: s3_pkt.c:534: error:1408F119:SSL routines:SSL3_GET_RECORD:decryption failed or bad record mac
2023.11.24 03:14:33 LOG3[359]: SSL_read: s3_pkt.c:1498: error:140943FC:SSL routines:ssl3_read_bytes:sslv3 alert bad record mac
2023.11.24 03:24:21 LOG3[361]: SSL_read: s3_pkt.c:534: error:1408F119:SSL routines:SSL3_GET_RECORD:decryption failed or bad record mac
2023.11.24 03:24:21 LOG3[362]: SSL_read: s3_pkt.c:1498: error:140943FC:SSL routines:ssl3_read_bytes:sslv3 alert bad record mac
2023.11.24 03:29:15 LOG3[364]: SSL_read: s3_pkt.c:534: error:1408F119:SSL routines:SSL3_GET_RECORD:decryption failed or bad record mac
...
2023.11.24 04:08:27 LOG3[368]: SSL_read: s3_pkt.c:534: error:1408F119:SSL routines:SSL3_GET_RECORD:decryption failed or bad record mac
2023.11.24 04:18:15 LOG3[370]: SSL_read: s3_pkt.c:1498: error:140943FC:SSL routines:ssl3_read_bytes:sslv3 alert bad record mac


// stunnel crashed 14 min later in this example:
-rw-------   1 root     root     7241413 Nov 24 04:32 core.stunnel.1387.60001

// After stunnel crashed:
root@yyyyyy# gzcat stunnel.2 | head
2023.11.24 04:32:58 LOG6[ui]: Initializing inetd mode configuration
2023.11.24 04:32:58 LOG7[ui]: Clients allowed=125
2023.11.24 04:32:58 LOG5[ui]: stunnel 5.71 on x86_64-pc-solaris2.10 platform
2023.11.24 04:32:58 LOG5[ui]: Compiled/running with OpenSSL 1.0.2u  20 Dec 2019
2023.11.24 04:32:58 LOG5[ui]: Threading:PTHREAD Sockets:POLL,IPv6 TLS:ENGINE,OCSP,PSK,SNI Auth:LIBWRAP
2023.11.24 04:32:58 LOG7[ui]: errno: (*(___errno()))
2023.11.24 04:32:58 LOG6[ui]: Initializing inetd mode configuration
2023.11.24 04:32:58 LOG5[ui]: Reading configuration from file /etc/opt/csw/stunnel/stunnel.conf
2023.11.24 04:32:58 LOG5[ui]: UTF-8 byte order mark not detected
2023.11.24 04:32:58 LOG5[ui]: FIPS mode disabled


root@yyyyyy# gzcat stunnel.2 | egrep 's3_pkt.c|ERROR|Initializing inetd'
2023.11.24 04:32:58 LOG6[ui]: Initializing inetd mode configuration
2023.11.24 04:32:58 LOG6[ui]: Initializing inetd mode configuration
2023.11.24 04:32:58 LOG6[ui]: Initializing inetd mode configuration
2023.11.24 04:32:58 LOG6[ui]: Initializing inetd mode configuration
2023.11.25 03:10:00 LOG6[main]: Initializing inetd mode configuration


// The core files also vary:

root@yyyyyy# mdb /var/tmp/cores-stunnel/core.stunnel.
core.stunnel.1387.60001   core.stunnel.15107.60001  core.stunnel.21017.60001  core.stunnel.23951.60001  core.stunnel.339.60001    core.stunnel.8971.60001
root@yyyyyy# mdb /var/tmp/cores-stunnel/core.stunnel.1387.60001
Loading modules: [ libc.so.1 ld.so.1 ]
>
> ::status
debugging core file of stunnel (32-bit) from yyyyyy
file: /opt/csw/bin/stunnel
initial argv: /opt/csw/bin/stunnel /etc/opt/csw/stunnel/stunnel.conf start
threading model: multi-threaded
status: process terminated by SIGSEGV (Segmentation Fault)
>
> $c
libcrypto.so.1.0.0`sk_value+0x17(0, 806da38, 806d9d5, 0, 0, 0)
0x809b938()
>
> $q



root@yyyyyy# pstack /var/tmp/cores-stunnel/core.stunnel.1387.60001
core '/var/tmp/cores-stunnel/core.stunnel.1387.60001' of 1387:  /opt/csw/bin/stunnel /etc/opt/csw/stunnel/stunnel.conf start
-----------------  lwp# 1 / thread# 1  --------------------
 feeec785 __pollsys (80fbfe8, 6, 0, 0) + 15
 fee912a2 poll     (80fbfe8, 6, ffffffff, 80ead90) + 52
 0806bb3f s_poll_wait (80dbae8, ffffffff, ffffffff, 8073fff) + 58
 08073fb4 daemon_loop (febfbc7c, feffa910, 8047e18, 8075e9f, 3, 8047e48) + 54
 08076011 ???????? (3, 8047e48, feffa910, 8075e91)
 08075e9f main     (3, 8047e48, 8047e58, 8047e3c) + 1f
 0805989d _start   (3, 8047ee0, 8047ef5, 8047f17, 0, 8047f1d) + 7d
-----------------  lwp# 3 / thread# 3  --------------------
 feeec785 __pollsys (80dce48, 1, feb5ef60, 0) + 15
 fee912a2 poll     (80dce48, 1, 3e8, 0) + 52
 0806bb11 s_poll_wait (80f0028, 1, 0, 806bb7c) + 2a
 0806bb98 s_poll_sleep (1) + 28
 080735bc ???????? (feee7d09, fef50000, feb5efec, feee938e, 0, 0)
 0807356c ???????? (0)
 feee938e _thr_setup (fea50200) + 4e
 feee9690 _lwp_start (fea50200, 0, 0, feb5eff8, feee9690, fea50200)
-----------------  lwp# 4 / thread# 4  --------------------
 feeec785 __pollsys (80f5a60, 1, fe8def40, 0) + 15
 fee912a2 poll     (80f5a60, 1, 5265c00, 0) + 52
 0806bb11 s_poll_wait (80fa4f8, 15180, 0, 806bb7c) + 2a
 0806bb98 s_poll_sleep (15180) + 28
 080736e2 ???????? (feee7d09, fef50000, fe8defec, feee938e, 0, 0)
 0807358c ???????? (0)
 feee938e _thr_setup (fea50a00) + 4e
 feee9690 _lwp_start (fea50a00, 0, 0, fe8deff8, feee9690, fea50a00)
-----------------  lwp# 7 / thread# 7  --------------------
 feeec785 __pollsys (80dbb40, 3, fe79fe70, 0) + 15
 fee912a2 poll     (80dbb40, 3, 493e0, 0) + 52
 0806bb11 s_poll_wait (80f2df0, 12c, 0, 0) + 2a
 0805c12e ???????? (813d580, 0, fe79ffb8, 805aec6)
 0805af0a ???????? (813d580)
 0805aad3 client_thread (813d580) + 43
 feee938e _thr_setup (fea52200) + 4e
 feee9690 _lwp_start (fea52200, 0, 0, fe79fff8, feee9690, fea52200)
-----------------  lwp# 188 / thread# 188  --------------------
 feeec785 __pollsys (813cc20, 3, fe7dfe70, 0) + 15
 fee912a2 poll     (813cc20, 3, 493e0, 0) + 52
 0806bb11 s_poll_wait (8197d18, 12c, 0, 0) + 2a
 0805c12e ???????? (811c250, 0, fe7dffb8, 805aec6)
 0805af0a ???????? (811c250)
 0805aad3 client_thread (811c250) + 43
 feee938e _thr_setup (fea51200) + 4e
 feee9690 _lwp_start (fea51200, 0, 0, fe7dfff8, feee9690, fea51200)
-----------------  lwp# 189 / thread# 189  --------------------
 feeec785 __pollsys (80ee0c8, 3, fe73fe70, 0) + 15
 fee912a2 poll     (80ee0c8, 3, 493e0, 0) + 52
 0806bb11 s_poll_wait (812a8c8, 12c, 0, 0) + 2a
 0805c12e ???????? (81333d8, 0, fe73ffb8, 805aec6)
 0805af0a ???????? (81333d8)
 0805aad3 client_thread (81333d8) + 43
 feee938e _thr_setup (fea53a00) + 4e
 feee9690 _lwp_start (fea53a00, 0, 0, fe73fff8, feee9690, fea53a00)
-----------------  lwp# 192 / thread# 192  --------------------
 feeec785 __pollsys (816e480, 3, fe75fe70, 0) + 15
 fee912a2 poll     (816e480, 3, 493e0, 0) + 52
 0806bb11 s_poll_wait (81977e0, 12c, 0, 0) + 2a
 0805c12e ???????? (8176998, 0, fe75ffb8, 805aec6)
 0805af0a ???????? (8176998)
 0805aad3 client_thread (8176998) + 43
 feee938e _thr_setup (fea53200) + 4e
 feee9690 _lwp_start (fea53200, 0, 0, fe75fff8, feee9690, fea53200)
-----------------  lwp# 193 / thread# 193  --------------------
 feeec785 __pollsys (8163db0, 3, fe7bfe70, 0) + 15
 fee912a2 poll     (8163db0, 3, 493e0, 0) + 52
 0806bb11 s_poll_wait (8197d70, 12c, 0, 0) + 2a
 0805c12e ???????? (814f168, 0, fe7bffb8, 805aec6)
 0805af0a ???????? (814f168)
 0805aad3 client_thread (814f168) + 43
 feee938e _thr_setup (fea51a00) + 4e
 feee9690 _lwp_start (fea51a00, 0, 0, fe7bfff8, feee9690, fea51a00)
-----------------  lwp# 194 / thread# 194  --------------------
 feeec785 __pollsys (8161570, 3, fe77fe70, 0) + 15
 fee912a2 poll     (8161570, 3, 493e0, 0) + 52
 0806bb11 s_poll_wait (812a6d0, 12c, 0, 0) + 2a
 0805c12e ???????? (81582c8, 0, fe77ffb8, 805aec6)
 0805af0a ???????? (81582c8)
 0805aad3 client_thread (81582c8) + 43
 feee938e _thr_setup (fea52a00) + 4e
 feee9690 _lwp_start (fea52a00, 0, 0, fe77fff8, feee9690, fea52a00)
-----------------  lwp# 374 / thread# 374  --------------------
 fed04b37 sk_value (0, 806da38, 806d9d5, 0, 0, 0) + 17
 0809b938 ???????? ()
-----------------  lwp# 376 / thread# 376  --------------------
 feeec785 __pollsys (8197540, 3, fe71fe70, 0) + 15
 fee912a2 poll     (8197540, 3, 493e0, 0) + 52
 0806bb11 s_poll_wait (80db440, 12c, 0, 0) + 2a
 0805c12e ???????? (81645c0, 0, fe71ffb8, 805aec6)
 0805af0a ???????? (81645c0)
 0805aad3 client_thread (81645c0) + 43
 feee938e _thr_setup (fea54200) + 4e
 feee9690 _lwp_start (fea54200, 0, 0, fe71fff8, feee9690, fea54200)


// Some examples of other core files:

root@xxxxxx# mdb core.stunnel.20880.60001
Loading modules: [ libc.so.1 ld.so.1 ]
> ::status
debugging core file of stunnel (32-bit) from xxxxxx
file: /opt/csw/bin/stunnel
initial argv: /opt/csw/bin/stunnel /etc/opt/csw/stunnel/stunnel.conf start
threading model: multi-threaded
status: process terminated by SIGSEGV (Segmentation Fault)
>
> $c
libssl.so.1.0.0`freelist_extract+0x57()
> $q


root@xxxxxx# mdb core.stunnel.3751.60001
Loading modules: [ libc.so.1 ld.so.1 ]
> ::status
debugging core file of stunnel (32-bit) from xxxxxx
file: /opt/csw/bin/stunnel
initial argv: /opt/csw/bin/stunnel /etc/opt/csw/stunnel/stunnel.conf start
threading model: multi-threaded
status: process terminated by SIGSEGV (Segmentation Fault)
>
> $c
libcrypto.so.1.0.0`getrn+0x74()
>
> $q


root@xxxxxx# mdb core.stunnel.12169.60001
Loading modules: [ libc.so.1 ld.so.1 ]
> ::status
debugging core file of stunnel (32-bit) from xxxxxx
file: /opt/csw/bin/stunnel
initial argv: /opt/csw/bin/stunnel /etc/opt/csw/stunnel/stunnel.conf start
threading model: multi-threaded
status: process terminated by SIGABRT (Abort)
>
> $c
libc.so.1`_lwp_kill+0x15(94, 6)
libc.so.1`raise+0x1f(6)
libc.so.1`abort+0xcd(45544e49, 4c414e52, 52524520, 203a524f, 64616544, 6e616320)
fatal_debug+0xbd(8076a84, 80769c4, 0, 0)
0x805a309(80fe000, 80769c4, 0, 0)
str_detach_debug+0x1c(80fe000, 80769c4)
str_free_debug+0x3c(80fe000, 80769c4, 0, fef51588)
0x8059b68(80fe000, fe75fe68, fee86a35, fec6008a)
libcrypto.so.1.0.0`CRYPTO_free+0x3c(814c860)
client_thread+0x43(814c860)
libc.so.1`_thr_setup+0x4e(fea53200)
libc.so.1`_lwp_start(fea53200, 0, 0, fe75fff8, feee9690, fea53200)
>