dtrace userland and postgresql

fluca1978 · Mar 29, 2011

Hi all,

I'm trying to compile PostgreSQL with the support for dtrace, and since it is not clear to me how to do this through the ports I've compiled from source with the --enable-dtrace option. I've followed the steps reported here to enable dtrace userland, and the compiling went fine. But when I launch initdb I got the error:

Code:

fgets failure: No such file or directory
The program "postgres" is needed by initdb but was not found in the
same directory as "/usr/local/pgsql/bin/initdb".
Check your installation

Of course the program is there. I've tried to recompile the postgresql without dtrace and then the initdb and other tools just work, so it must be a problem with dtrace userland.

The following is the output of truss:

Code:

__sysctl(0xbfbfe754,0x2,0xbfbfe75c,0xbfbfe760,0x0,0x0) = 0 (0x0)
mmap(0x0,336,PROT_READ|PROT_WRITE,MAP_ANON,-1,0x0) = 671678464 (0x28090000)
munmap(0x28090000,336)                           = 0 (0x0)
__sysctl(0xbfbfe7b8,0x2,0x28086e3c,0xbfbfe7c0,0x0,0x0) = 0 (0x0)
mmap(0x0,32768,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0x0) = 671678464 (0x28090000)
issetugid(0x2807fa47,0xbfbfe880,0x104,0x0,0x0,0x0) = 0 (0x0)
open("/etc/libmap.conf",O_RDONLY,0666)           ERR#2 'No such file or directory'
access("/usr/local/pgsql/lib/libc.so.7",0)       ERR#2 'No such file or directory'
open("/var/run/ld-elf.so.hints",O_RDONLY,00)     = 2 (0x2)
read(2,"Ehnt\^A\0\0\0\M^@\0\0\0-\0\0\0\0"...,128) = 128 (0x80)
lseek(2,0x80,SEEK_SET)                           = 128 (0x80)
read(2,"/lib:/usr/lib:/usr/lib/compat:/u"...,45) = 45 (0x2d)
close(2)                                         = 0 (0x0)
access("/lib/libc.so.7",0)                       = 0 (0x0)
open("/lib/libc.so.7",O_RDONLY,00)               = 2 (0x2)
fstat(2,{ mode=-r--r--r-- ,inode=1813528,size=1346561,blksize=16384 }) = 0 (0x0)
pread(0x2,0x28085d80,0x1000,0x0,0x0,0x0)         = 4096 (0x1000)
mmap(0x0,1155072,PROT_NONE,MAP_PRIVATE|MAP_ANON|MAP_NOCORE,-1,0x0) = 671711232 (0x28098000)
mmap(0x28098000,1036288,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE,2,0x0) = 671711232 (0x28098000)
mmap(0x28195000,24576,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED,2,0xfd000) = 672747520 (0x28195000)
mprotect(0x2819b000,94208,PROT_READ|PROT_WRITE)  = 0 (0x0)
close(2)                                         = 0 (0x0)
sysarch(0xa,0xbfbfe820,0x2805639b,0x280852f8,0x28068f09,0x280852f8) = 0 (0x0)
mmap(0x0,712,PROT_READ|PROT_WRITE,MAP_ANON,-1,0x0) = 672866304 (0x281b2000)
munmap(0x281b2000,712)                           = 0 (0x0)
mmap(0x0,21904,PROT_READ|PROT_WRITE,MAP_ANON,-1,0x0) = 672866304 (0x281b2000)
munmap(0x281b2000,21904)                         = 0 (0x0)
sigprocmask(SIG_BLOCK,SIGHUP|SIGINT|SIGQUIT|SIGKILL|SIGPIPE|SIGALRM|SIGTERM|SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHLD|SIGTTIN|SIGTTOU|SIGIO|
SIGXCPU|SIGXFSZ|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|SIGUSR1|SIGUSR2,0x0) = 0 (0x0)
sigprocmask(SIG_SETMASK,0x0,0x0)                 = 0 (0x0)
__sysctl(0xbfbfe7d4,0x2,0x2819c740,0xbfbfe7dc,0x0,0x0) = 0 (0x0)
sigprocmask(SIG_BLOCK,SIGHUP|SIGINT|SIGQUIT|SIGKILL|SIGPIPE|SIGALRM|SIGTERM|SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHLD|SIGTTIN|SIGTTOU|SIGIO|
SIGXCPU|SIGXFSZ|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|SIGUSR1|SIGUSR2,0x0) = 0 (0x0)
sigprocmask(SIG_SETMASK,0x0,0x0)                 = 0 (0x0)
__sysctl(0xbfbfdab0,0x2,0x281a1400,0xbfbfdabc,0x0,0x0) = 0 (0x0)
__sysctl(0xbfbfd9e4,0x2,0xbfbfd97c,0xbfbfd9ec,0x2818d5b4,0xc) = 0 (0x0)
__sysctl(0xbfbfd97c,0x2,0x281a14f8,0xbfbfda28,0x0,0x0) = 0 (0x0)
readlink("/etc/malloc.conf",0xbfbfdac3,1024)     ERR#2 'No such file or directory'
issetugid(0x2818c2d1,0xbfbfdac3,0x400,0xbfbfdabc,0x0,0x0) = 0 (0x0)
break(0x8100000)                                 = 0 (0x0)
__sysctl(0xbfbfdd44,0x2,0xbfbfdd4c,0xbfbfdd50,0x0,0x0) = 0 (0x0)
mmap(0x0,1048576,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0x0) = 672866304 (0x281b2000)
mmap(0x282b2000,319488,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0x0) = 673914880 (0x282b2000)
munmap(0x281b2000,319488)                        = 0 (0x0)
__getcwd("/usr/local/pgsql/bin",1024)            = 0 (0x0)
stat("/usr/local/pgsql/bin/initdb",{ mode=-rwxr-xr-x ,inode=1250720,size=61669,blksize=16384 }) = 0 (0x0)
access("/usr/local/pgsql/bin/initdb",4)          = 0 (0x0)
access("/usr/local/pgsql/bin/initdb",1)          = 0 (0x0)
__getcwd("/usr/local/pgsql/bin",1024)            = 0 (0x0)
chdir("/usr/local/pgsql/bin")                    = 0 (0x0)
lstat("initdb",{ mode=-rwxr-xr-x ,inode=1250720,size=61669,blksize=16384 }) = 0 (0x0)
__getcwd("/usr/local/pgsql/bin",1024)            = 0 (0x0)
chdir("/usr/local/pgsql/bin")                    = 0 (0x0)
__getcwd("/usr/local/pgsql/bin",1024)            = 0 (0x0)
stat("/usr/local/pgsql/bin/initdb",{ mode=-rwxr-xr-x ,inode=1250720,size=61669,blksize=16384 }) = 0 (0x0)
access("/usr/local/pgsql/bin/initdb",4)          = 0 (0x0)
access("/usr/local/pgsql/bin/initdb",1)          = 0 (0x0)
__getcwd("/usr/local/pgsql/bin",1024)            = 0 (0x0)
chdir("/usr/local/pgsql/bin")                    = 0 (0x0)
lstat("initdb",{ mode=-rwxr-xr-x ,inode=1250720,size=61669,blksize=16384 }) = 0 (0x0)
__getcwd("/usr/local/pgsql/bin",1024)            = 0 (0x0)
chdir("/usr/local/pgsql/bin")                    = 0 (0x0)
stat("/usr/local/pgsql/bin/postgres",{ mode=-rwxr-xr-x ,inode=1250110,size=4715827,blksize=16384 }) = 0 (0x0)
access("/usr/local/pgsql/bin/postgres",4)        = 0 (0x0)
access("/usr/local/pgsql/bin/postgres",1)        = 0 (0x0)
pipe(0xbfbfdae4)                                 = 0 (0x0)
vfork(0x2b,0x0,0x28184a31,0x0,0x2820c090,0x0)    = 13972 (0x3694)
fcntl(2,F_GETFL,)                                = 2 (0x2)
close(3)                                         = 0 (0x0)
fstat(2,{ mode=p--------- ,inode=0,size=0,blksize=4096 }) = 0 (0x0)
read(2,0x28211000,4096)                          = 0 (0x0)
stat("/usr/share/nls/C/libc.cat",0xbfbfcdb4)     ERR#2 'No such file or directory'
stat("/usr/share/nls/libc/C",0xbfbfcdb4)         ERR#2 'No such file or directory'
stat("/usr/local/share/nls/C/libc.cat",0xbfbfcdb4) ERR#2 'No such file or directory'
stat("/usr/local/share/nls/libc/C",0xbfbfcdb4)   ERR#2 'No such file or directory'
writev(0x2,0xbfbfd2c8,0x4,0x4d91bc87,0x0,0xbfbfd2c8) ERR#32 'Broken pipe'
SIGNAL 13 (SIGPIPE)
process exit, rval = 0

Am I doing something wrong with my installation?

Thanks

achix · Mar 29, 2011

You could try with initdb --debug to see what is going on. If this does not help, then you will have to dig into port/exec.c:305 to print some debugging info from there. This routine should return 0 for a found target. In your case apparently it returns -1, as implied by the initdb.c message.

The two files you should play with for a start are /bin/initdb/initdb.c and port/exec.c

fluca1978 · Mar 30, 2011

Thanks, I modified the sources and found that the problem was in postgres (binary): each time it is called it end up with a SIGFAULT. I post the truss and ldd output, I need a little help to understand what is going on here. Again, if compiled without --enable-dtrace everything works.

Thanks.

Code:

__sysctl(0xbfbfe744,0x2,0xbfbfe74c,0xbfbfe750,0x0,0x0) = 0 (0x0)
mmap(0x0,336,PROT_READ|PROT_WRITE,MAP_ANON,-1,0x0) = 675897344 (0x28496000)
munmap(0x28496000,336)                           = 0 (0x0)
__sysctl(0xbfbfe7a8,0x2,0x2848ce3c,0xbfbfe7b0,0x0,0x0) = 0 (0x0)
mmap(0x0,32768,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANON,-1,0x0) = 675897344 (0x28496000)
issetugid(0x28485a47,0xbfbfe870,0x104,0x0,0x0,0x0) = 0 (0x0)
open("/etc/libmap.conf",O_RDONLY,0666)           ERR#2 'No such file or directory'
access("/usr/local/pgsql/lib/libm.so.5",0)       ERR#2 'No such file or directory'
open("/var/run/ld-elf.so.hints",O_RDONLY,00)     = 2 (0x2)
read(2,"Ehnt\^A\0\0\0\M^@\0\0\0-\0\0\0\0"...,128) = 128 (0x80)
lseek(2,0x80,SEEK_SET)                           = 128 (0x80)
read(2,"/lib:/usr/lib:/usr/lib/compat:/u"...,45) = 45 (0x2d)
close(2)                                         = 0 (0x0)
access("/lib/libm.so.5",0)                       = 0 (0x0)
open("/lib/libm.so.5",O_RDONLY,00)               = 2 (0x2)
fstat(2,{ mode=-r--r--r-- ,inode=1813507,size=146183,blksize=16384 }) = 0 (0x0)
pread(0x2,0x2848bd80,0x1000,0x0,0x0,0x0)         = 4096 (0x1000)
mmap(0x0,106496,PROT_NONE,MAP_PRIVATE|MAP_ANON|MAP_NOCORE,-1,0x0) = 675930112 (0x2849e000)
mmap(0x2849e000,102400,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE,2,0x0) = 675930112 (0x2849e000)
mmap(0x284b7000,4096,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED,2,0x18000) = 676032512 (0x284b7000)
close(2)                                         = 0 (0x0)
access("/usr/local/pgsql/lib/libc.so.7",0)       ERR#2 'No such file or directory'
access("/lib/libc.so.7",0)                       = 0 (0x0)
open("/lib/libc.so.7",O_RDONLY,027757763634)     = 2 (0x2)
fstat(2,{ mode=-r--r--r-- ,inode=1813528,size=1346561,blksize=16384 }) = 0 (0x0)
pread(0x2,0x2848bd80,0x1000,0x0,0x0,0x0)         = 4096 (0x1000)
mmap(0x0,1155072,PROT_NONE,MAP_PRIVATE|MAP_ANON|MAP_NOCORE,-1,0x0) = 676036608 (0x284b8000)
mmap(0x284b8000,1036288,PROT_READ|PROT_EXEC,MAP_PRIVATE|MAP_FIXED|MAP_NOCORE,2,0x0) = 676036608 (0x284b8000)
mmap(0x285b5000,24576,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_FIXED,2,0xfd000) = 677072896 (0x285b5000)
mprotect(0x285bb000,94208,PROT_READ|PROT_WRITE)  = 0 (0x0)
close(2)                                         = 0 (0x0)
sysarch(0xa,0xbfbfe810,0x2845c39b,0x2848b2f8,0x2846ef09,0x2848b2f8) = 0 (0x0)
mmap(0x0,48744,PROT_READ|PROT_WRITE,MAP_ANON,-1,0x0) = 677191680 (0x285d2000)
munmap(0x285d2000,48744)                         = 0 (0x0)
mmap(0x0,2072,PROT_READ|PROT_WRITE,MAP_ANON,-1,0x0) = 677191680 (0x285d2000)
munmap(0x285d2000,2072)                          = 0 (0x0)
mmap(0x0,21904,PROT_READ|PROT_WRITE,MAP_ANON,-1,0x0) = 677191680 (0x285d2000)
munmap(0x285d2000,21904)                         = 0 (0x0)
sigprocmask(SIG_BLOCK,SIGHUP|SIGINT|SIGQUIT|SIGKILL|SIGPIPE|SIGALRM|SIGTERM|SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHLD|SIGTTIN|SIGTTOU|SIGIO|
SIGXCPU|SIGXFSZ|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|SIGUSR1|SIGUSR2,0x0) = 0 (0x0)
sigprocmask(SIG_SETMASK,0x0,0x0)                 = 0 (0x0)
__sysctl(0xbfbfe7c4,0x2,0x8460670,0xbfbfe7cc,0x0,0x0) = 0 (0x0)
sigprocmask(SIG_BLOCK,SIGHUP|SIGINT|SIGQUIT|SIGKILL|SIGPIPE|SIGALRM|SIGTERM|SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHLD|SIGTTIN|SIGTTOU|SIGIO|
SIGXCPU|SIGXFSZ|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|SIGUSR1|SIGUSR2,0x0) = 0 (0x0)
sigprocmask(SIG_SETMASK,0x0,0x0)                 = 0 (0x0)
sigprocmask(SIG_BLOCK,SIGHUP|SIGINT|SIGQUIT|SIGKILL|SIGPIPE|SIGALRM|SIGTERM|SIGURG|SIGSTOP|SIGTSTP|SIGCONT|SIGCHLD|SIGTTIN|SIGTTOU|SIGIO|
SIGXCPU|SIGXFSZ|SIGVTALRM|SIGPROF|SIGWINCH|SIGINFO|SIGUSR1|SIGUSR2,0x0) = 0 (0x0)
sigprocmask(SIG_SETMASK,0x0,0x0)                 = 0 (0x0)
SIGNAL 11 (SIGSEGV)
process exit, rval = 0

Code:

/usr/local/pgsql/bin/postgres:
        libm.so.5 => /lib/libm.so.5 (0x2849e000)
        libc.so.7 => /lib/libc.so.7 (0x284b8000)

achix · Mar 30, 2011

I would advise subscribing to postgresql lists, (-admin, -performance).
This specific one could be best served in -hackers.
Be aware that although PostgreSQL might not be huge as in lines of code, it is still a big piece of software, with a long faithful hacker community behind it.
You should talk to them. Some of them (Tom Lane) could complain a little bit about you not using Linux but others will be fine.

fluca1978 · Mar 30, 2011

I know quite well the PostgreSQL universe, and I'm already subscribed to such lists, but since the problem raised since I compiled with dtrace I thougth it was best asking on FreeBSD forums first.
Thanks, I'll update this post once I know something more.

fluca1978 · Apr 1, 2011

Debugging the process postgres I found that it receives SIGSEV as soon as it enters dtrace_dof_init() (the first instruction causes the SIGSEV). I don't know what dtrace_dof_init is but I'm pretty sure there is a problem with dtrace (or at least my dtrace configuration). The stacktrace shows that the function called before dtrace_dof_init is __do_global_ctors_aux() that sounds to me something related to c constructors (libc?). Any suggestion on how to investigate more?

achix · Apr 2, 2011

Luca,

I was just reading Greg Smith's book about pgsql performance (PACKT publishing), and on page 412, he says that FreeBSD's implementation of dtrace has two issues:

1) It is limited to root
2) The probes are not very comprehensive yet.

Taking into account, that in the Linux world you can actually rely only on RedHat or Fedora for an emulation of dtrace, that leaves Solaris and MacOS as the only two systems with solid dtrace implementations.

fluca1978 · Apr 5, 2011

Thanks for the suggestion, I guess every IllumOS based systems (Nexenta, OpenIndiana, ecc.) are valid too. Any way as stated here, as well as in the book you cited, there is effectively some problems between PostgreSQL and FreeBSD Dtrace implementation. What is the way to notify FreeBSD developers of the problem, so that they can investigate more on the subject?

achix · Apr 7, 2011

Maybe you should track the dtrace committers.
John Birrell maybe?

Anyways, the natural place to talk about it is in the FreeBSD hacker list, (the same thing you did about PostgreSQL).

PS
Anyway, why did you feel the need to use dtrace?
Just curious?

achix · Apr 7, 2011

oopss : update...

I am so sorry. It seems that the person who did the porting John Birrell, has passed away.
RIP to this fine engineer.