Thanks for the hint on using syscall.
I found some information which can be summarized as:
; function parameter (when linked with external libraries):
; r9 ; 6th param
; r8 ; 5th param
; rcx ; 4th param
; rdx ; 3rd param
; rsi ; 2nd param
; rdi ; 1st param
; call library
so the way I made a successful printf call was to not push the parameter onto the stack but rather set the rdi register ie
the working snippet of code equivalent to printf("Hello 64-bit world!\n")
msg: db 'Hello 64-bit world!',0xA,0x0
...
mov edi, msg
call printf
Success!
The curious thing was when I tried to extend this to more general library calls ie from openGL. Rather than using rdi, I had to use the xmm registers
half: dd 0.5
neghalf: dd -0.5
...
xorps xmm2, xmm2
movss xmm1, [neghalf]
movss xmm0, [neghalf]
call glVertex3f ; glVertex3f(-.5, -.5, 0);
p.s. If interested, the following is a complete working openGL version of Hello World running successfully on my amd64 with
"nasm -g -f elf64 hello_gl64.asm && gcc hello_gl64.o -lGL -lglut"
Note following is a modified version from 32bit example (
http://www.cs.lmu.edu/~ray/notes/nasmexamples/) to work on my amd64.
global main
extern printf
extern glClear
extern glBegin
extern glEnd
extern glColor3f
extern glVertex3f
extern glFlush
extern glutInit
extern glutInitDisplayMode
extern glutInitWindowPosition
extern glutInitWindowSize
extern glutCreateWindow
extern glutDisplayFunc
extern glutMainLoop
section .text
msg: db 'Hello 64-bit world!',0xA,0x0
zero: dd 0.0
one: dd 1.0
half: dd 0.5
neghalf:dd -0.5
display:
;pseudo constructor to cleanly deal with local variables
push rbp ; must save old ebp
mov rbp, rsp ; point ebp to this frame
mov edi, 16384
call glClear ; glClear(GL_COLOR_BUFFER_BIT);
mov edi, 9
call glBegin ; glBegin(GL_POLYGON);
xorps xmm2, xmm2
xorps xmm1, xmm1
movss xmm0, [one]
call glColor3f ; glColor3f(1, 0, 0);
xorps xmm2, xmm2
movss xmm1, [neghalf]
movss xmm0, [neghalf]
call glVertex3f ; glVertex3f(-.5, -.5, 0);
xorps xmm2, xmm2
movss xmm1, [one]
xorps xmm0, xmm0
call glColor3f ; glColor3f(0, 1, 0);
xorps xmm2, xmm2
movss xmm1, [neghalf]
movss xmm0, [half]
call glVertex3f ; glVertex3f(.5, -.5, 0);
movss xmm2, [one]
xorps xmm1, xmm1
xorps xmm0, xmm0
call glColor3f ; glColor3f(0, 0, 1);
xorps xmm2, xmm2
movss xmm1, [half]
xorps xmm0, xmm0
call glVertex3f ; glVertex3f(0, .5, 0);
call glEnd ; glEnd();
call glFlush ; glFlush();
mov edi, msg
call printf
leave
ret
main:
;pseudo init to cleanly deal with local variables
push rbp ; must save old ebp
mov rbp, rsp ; point ebp to this frame
sub rsp, 16 ; make space for 4? locals
;mov [rbp-4], edi
;mov [rbp-16], rsi
mov rsi, [rbp-16] ; argv
lea rdi, [rbp-4] ; argc
call glutInit ; glutInit(&argc, argv)
mov edi,0
call glutInitDisplayMode ; glutInitDisplayMode(GLUT_RGB);
mov esi, 80
mov edi, 80
call glutInitWindowPosition ; glutInitWindowPosition(80,80);
mov esi, 300
mov edi, 400
call glutInitWindowSize ; glutInitWindowSize(400,300);
mov edi, msg
call glutCreateWindow ; glutCreateWindow(msg);
mov edi, display
call glutDisplayFunc ; glutDisplayFunc(display);
call glutMainLoop ; glutMainLoop();
;pseudo cleanup for local variables helper
mov rsp, rbp ; clean up locals
pop rbp ; restore old ebp
;return success
mov eax, 0
ret