After much experimentation and even though GDB really doesn't like this, I've settled on STDCALL, probably from having done it that way so long, it just seems easier to design algos
This is primarily designed to facilitate non-canonical input in my apps, but additional functionality can be incorporated using mode SET = 1 and modifying whatever flags outside the procedure.
SYS_IOCTL equ 16
TCGETS equ 0x5401
TCSETS equ 0x5402
MASK equ ICANON | ECHO ; ICANON = 2, ECHO = 8
ARG1 equ 16
ARG2 equ 24
global Termios
extern FmtErr
section .text
; =============================================================================================
; bool Termios ( int Function, char *Termios_struct )
; ENTER: ARG1 = Function # 0 = GET, 1 = SET, 2 = TOGGLE
; ARG2 = Pointer to termios structure
; LEAVE: RAX = True (-1), function failed, otherwise False (0).
; ---------------------------------------------------------------------------------------------
Termios: push rbp
mov rbp, rsp ; So ARG1 can be addressed conventionally
push rdx
push rsi ; Preserve
push rdi
; RDX & RDI are common to all modes
xor rdi, rdi ; File descriptor ( can be STDIN or STDOUT )
mov rdx, [rbp + ARG2] ; Pointer termios structure
; Get function mode from caller and because setting is more probably we'll set it
; as default
mov rcx, [rbp + ARG1] ; Get function mode
mov rax, rcx ; Simple way of resetting RAX
mov ax, TCGETS ; Most probable = 5402H
cmp cl, 1
jb .DoFunc
; Toggle bits 1 and 0 of AL. Hopefully kernel never changes this number
.Set xor al, 3 ; Now = 5401H
cmp cl, 1 ; Are we just setting new functionality
jz .DoFunc
; Caller could have done this, but this method save a bit of code overall because
; it might be called multiple times in app.
xor byte [rdx + 12], MASK ; Toggle NON-CANONICAL on/off
; Finish setting up registers.
.DoFunc mov rsi, rax ; Move function into RSI
mov ax, SYS_IOCTL
syscall ; Execute function
or rax, rax
jge .Exit
push 0 ; Have proc find syscall address
push rax ; Unmodifed error code
call FmtErr ; Display error
or rax, -1 ; Set error condition
.Exit pop rdi
pop rsi ; Retrieve
pop rdx
leave ; Kill frame
ret 16 ; Waste 2 arguments