Win64 Structure Alignment (Win API)

[encryptor256]

Hello!

Problemo:

Today i found a nasty surprise, that my structures are aligned incorrectly,
some structure sizes were not correct and structure member alignment too.
So, i went to investigate.

Each of my structure and member offsets i tested in Win64 Pelles C,
structures like PAINTSTRUCT, MSG, CREATESTRUCTA and so.

Then i tested with NASMX, to find out, if there are correct structure sizes, and i found that there is not too.

Attached modified demo1.asm, that gives a message boxes:
1. PAINTSTRUCT_size: 68 (should be 72)
2. LOGFONT_size: 60 (should be 60)
3. MSG_size: 40 (should be 48)
4. Wrong(wParam) MSG_offsets:  hwnd: ( 0 ), message: ( 8 ), wParam: ( 12 ), lParam: ( 20 ), time: ( 28 ), pt.x: ( 32 ), pt.y: ( 36 )

This is how it should be:
MSG.hwnd, size 8, offset 0
MSG.message, size 4, offset 8
MSG.wParam, size 8, offset 16
....
After MSG.message there needs to be a 4x padding bytes, so, when touching wParam, then , the touch, is aligned on 8x byte boundaries.

MSDN: Aggregates and Unions:

• "Structure size must be an integral multiple of its alignment, which may require padding after the last member. Since structures and unions can be grouped in arrays, each array element of a structure or union must begin and end at the proper alignment previously determined."

Some ideas, structure size 44 + 4x bytes of padding:

Code: [Select]

MSG.hwnd equ (0)
MSG.message equ (8)
MSG.wParam equ (8+(4+4))
MSG.lParam equ (8+(4+4)+8)
MSG.time equ (8+(4+4)+8+8)
MSG.pt.x equ (8+(4+4)+8+8+4)
MSG.pt.x equ (8+(4+4)+8+8+4+4)
So, last offset is 40, which estimates in 44 bytes of structure size!
Largest member unit set size is 8.
L = 8
SZ = 44
X = SZ - SZ / L * L = 44 - 40 = 4    ; Determine if SZ divides with L without a remainder, which X becomes.
IF X != 0
 X = L - X = 4
ENDIF
Final structure size:
SZ = SZ + X = 44 + 4 = 48.

This is how it should be,
a testing macro named: structureoffsets

Code: [Select]

%macro structureoffsets 3-*
      %define structname %1
      %assign loopcount ((%0-1)/2)
      %assign alignment 1


      ; Determine alignment
      %rotate 1
      %rep loopcount
            %if (%2 > alignment) && (%2 <= 8)
                  %assign alignment %2
            %endif
            %rotate 2
      %endrep

      ;%warning ALIGNMENT alignment

      ; First pair doesnt count
;       %assign %$structoffset 0
;       %rotate 1
;       %define %[%$structname].%[%1] %$structoffset


      %assign correctioncount 0
      %define max ((%0-1)/2)
      ;%define structname %1
      %assign structsize 0
      %rotate 1
      %rep max
            %if %[%2] = 8
                  %assign mod (%[structsize]-(%[structsize]/8)*8)
                  %if mod != 0
                        %assign structsize %[structsize]+(8-%[mod])
                        %assign correctioncount 1
                  %endif
                  %undef mod
            %endif
            %define %[structname].%[%1] %[structsize]
            %assign structsize structsize+%[%2]
            %rotate 2
      %endrep
      ; ---
      %assign mod (%[structsize]-(%[structsize]/%[alignment])*%[alignment])
      %if mod != 0
            %assign structsize %[structsize]+(%[alignment]-%[mod])
      %endif
      %undef mod
      %undef correctioncount
      ; ---
      %define %[structname]_SIZE %[structsize]
      %warning "zzzstruct:" %[structname] %[structname]_SIZE
      %undef max
      %undef structname

%endmacro

These are structures,
defined by test macro structureoffsets:

Code: [Select]

structureoffsets WNDCLASSEXA,\
                cbSize,4,\
                style,4,\
                lpfnWndProc,8,\
                cbClsExtra,4,\
                cbWndExtra,4,\
                hInstance,8,\
                hIcon,8,\
                hCursor,8,\
                hbrBackground,8,\
                lpszMenuName,8,\
                lpszClassName,8,\
                hIconSm,8

structureoffsets MSG,\
                hwnd,8,\
                message,4,\
                wParam,8,\
                lParam,8,\
                time,4,\
                pt.x,4,\
                pt.y,4
;

structureoffsets POINT,\
                x,4,\
                y,4

structureoffsets CREATESTRUCTA,\
                lpCreateParams,8,\
                hInstance,8,\
                hMenu,8,\
                hwndParent,8,\
                cy,4,\
                cx,4,\
                y,4,\
                x,4,\
                style,4,\
                lpszName,8,\
                lpszClass,8,\
                dwExStyle,4

structureoffsets RECT,\
                left,4,\
                top,4,\
                right,4,\
                bottom,4

structureoffsets PAINTSTRUCT,\
                hdc,8,\
                fErase,4,\
                rcPaint,RECT_SIZE,\
                fRestore,4,\
                fIncUpdate,4,\
                rgbReserved,32
;

structureoffsets NMHDR,\
                hwndFrom,8,\
                idFrom,8,\
                code,4
;


%define LF_FACESIZE  32
structureoffsets LOGFONTA,\
                lfHeight,4,\
                lfWidth,4,\
                lfEscapement,4,\
                lfOrientation,4,\
                lfWeight,4,\
                lfItalic,1,\
                lfUnderline,1,\
                lfStrikeOut,1,\
                lfCharSet,1,\
                lfOutPrecision,1,\
                lfClipPrecision,1,\
                lfQuality,1,\
                lfPitchAndFamily,1,\
                lfFaceName,LF_FACESIZE


structureoffsets WINDOWPOS,\
                hwnd,8,\
                hwndInsertAfter,8,\
                x,4,\
                y,4,\
                cx,4,\
                cy,4,\
                flags,4
;


structureoffsets STARTUPINFOA,\
            cb,4,\
            lpReserved,8,\
            lpDesktop,8,\
            lpTitle,8,\
            dwX,4,\
            dwY,4,\
            dwXSize,4,\
            dwYSize,4,\
            dwXCountChars,4,\
            dwYCountChars,4,\
            dwFillAttribute,4,\
            dwFlags,4,\
            wShowWindow,2,\
            cbReserved2,2,\
            lpReserved2,8,\
            hStdInput,8,\
            hStdOutput,8,\
            hStdError,8

structureoffsets PROCESS_INFORMATION,\
            hProcess,8,\
            hThread,8,\
            dwProcessId,4,\
            dwThreadId,4

structureoffsets SECURITY_ATTRIBUTES,\
            nLength,8,\
            lpSecurityDescriptor,8,\
            bInheritHandle,8

This is output of definition in compile time,
it prints structure name and size, like MYSTRUCT SIZE:

Code: [Select]

structures.inc:15: warning: (structureoffsets:50) "zzzstruct:" WNDCLASSEXA 80
structures.inc:24: warning: (structureoffsets:50) "zzzstruct:" MSG 48
structures.inc:29: warning: (structureoffsets:50) "zzzstruct:" POINT 8
structures.inc:43: warning: (structureoffsets:50) "zzzstruct:" CREATESTRUCTA 80
structures.inc:49: warning: (structureoffsets:50) "zzzstruct:" RECT 16
structures.inc:60: warning: (structureoffsets:50) "zzzstruct:" PAINTSTRUCT 72
structures.inc:66: warning: (structureoffsets:50) "zzzstruct:" NMHDR 24
structures.inc:85: warning: (structureoffsets:50) "zzzstruct:" LOGFONTA 60
structures.inc:95: warning: (structureoffsets:50) "zzzstruct:" WINDOWPOS 40
structures.inc:117: warning: (structureoffsets:50) "zzzstruct:" STARTUPINFOA 104
structures.inc:123: warning: (structureoffsets:50) "zzzstruct:" PROCESS_INFORMATION 24
structures.inc:128: warning: (structureoffsets:50) "zzzstruct:" SECURITY_ATTRIBUTES 24

Here, SIZE OF PAINTSTRUCT in x64, Jeremy Gordon find out, that there was a problem him too, with assembly and structure alignment definitions.

Bye!

[Rob Neff]

Thanks for the report, encryptor256!

One of the macros originally developed for structure alignment, NASMX_ALIGN ( reference this old thread ) was designed to partly solve that issue.  However, it was never fully implemented in all published structure definitions.  Subsequently, I've re-examined the source for the structure macros and can confirm we are not supporting x64 struct alignment properly. 

I've not been 100% satisfied with that support and would much rather have the NASMX_RESERVE and NASMX_ENDSTRUCT macros become much smarter when dealing with structure alignment issues.  We'll obviously have to research this a bit further and rework it for the next release.

[Bryant Keller]

The problem with MSG.message is that it's defined as uint32_t instead of uint_t. Also, nasmx.inc has:

Code: [Select]

%ixdefine uint_t uint32_t
instead of:

Code: [Select]

%ixdefine uint_t uint%[__BITS__]_t
Which would probably be better.

I assume that this is also the problem with PAINTSTRUCT. Rather instead of using uint32_t it's using int32_t and should be using a int_t with the same int%[__BITS__]_t style definition in nasmx.inc.

Fair warning, I don't have Windows or a 64-bit machine and the best I could do is a quick "read through" of the current NASMX release in relation to these structures. I would also be willing to bet that this issue is affecting other structures in the Win64 platform.

Providing basic alignment padding shouldn't work much differently than UNION support does. You figure out the size of the largest member of the structure, then append an 'align %{$largest_member_size}, resb 1'. The difficult part would be doing this for every single member, this would resort to the same kind of late binding that I did with objects.inc (which is far from being a trivial task).

[encryptor256]

Hi!

can confirm we are not supporting x64 struct alignment properly.

Thanks, for the confirmation, now we know, officially, that there is small problem.

The problem with MSG.message is that...

MSG.message on Win64 remains 32bit, other members,
like hwnd, wParam, lParam are now 64bit, because,
they are pointers or they can hold a pointer,
for example WM_CREATE message delivers CREATESTRUCT pointer into lParam.

Well, we can say what ever we want, but we cannot deny the truth - The Problem Might Be The Whole Microsoft Windows.
Also Intel processors! Why do we have to support very very old CPU types, backward compability, come on!, let's fly with the wind and go forward in time!
Let's cut old cpu support and old Ms Windows base and go with acceleration!

Q: What's wrong with you, why are you insulting Intel and Microsoft?
A: I'm not! Intel, Microsoft and others, ... They are just a business!

Solution to structure alignment.
I think Win64 is built on C, so we have to learn, how C aligns structures (according to BITS),
then implement new nasm.exe command switch, something like: "-cstruct", which means,
when ever compiler finds structure, which starts with "struct" definition, then it aligns
the structure according to C rules, + according to __BITS__ too, so,
"-cstruct" + BITS 32 : defines structures and aligns them on 32 bit boundaries.
"-cstruct" + BITS 64 : defines structures and aligns them on 64 bit boundaries.

Q: Why do we have to support C structures in NASM?
A: Why not! NASM is built on that C\C++ thingy too, so, why couldn't we!

NASM switch "-s": redirect error messages to stdout.
NASM switch "-se": send output to email.
NASM switch "-sc": call me, when compile is done

[Rob Neff]

Let's cut old cpu support and old Ms Windows base and go with acceleration!

Unfortunately, there is a huge segment of the population out there still running Linux and WinXP on 32-bit machines.  We can't just suddenly force them all to upgrade, nor can we afford to ignore them as potential customers of ours.  Also, while there are very few, if any, 16-bit assembly programmer positions on the job market today, there are still classes ( as referenced by the various threads on these very forums ) that teach it to this day.

Solution to structure alignment.
I think Win64 is built on C, so we have to learn, how C aligns structures (according to BITS),

And therein you find the problem.  My research has shown no structure alignment requirements leaving the compiler writers free to implement it how they see fit.  Factor in the various operating systems and compilers available on the market today and we have quite a bit of non-standardness.  Think of the various alignment and packing command line switches and macros available to the developer to use when creating his or her structures and you'll find that it's going to be rather difficult to replicate in a portable way.

Since NASMX is limited to the subset of CPUs targeted by Nasm ( Namely, Intel x86 and x64 CPUs ) we can design towards a standard implementation and yet permit the flexibility of allowing choice to the developer - mainly in the area of pragmas and switches.  Thus, our first job is to implement natural alignment per structure member and overall correct alignment of structure size based on target CPU.  Secondly, we need to implement a way for programmers to modify the default behavior - eg: something along the lines of http://gcc.gnu.org/onlinedocs/gcc/Structure-Packing-Pragmas.html.  Finally, we'll need to verify that our published definitions conform to the expectations of the OS itself since obviously some currently do not.

Q: Why do we have to support C structures in NASM?
A: Why not! NASM is built on that C\C++ thingy too, so, why couldn't we!

See my original suggestion in this thread.  Note, however, that Nasm's free-form basically allows us to define the structures however we see fit - which neatly side-steps the issue and puts the onus on the developer to work out alignment issues.

[Bryant Keller]

Solution to structure alignment.
I think Win64 is built on C, so we have to learn, how C aligns structures (according to BITS),
then implement new nasm.exe command switch, something like: "-cstruct", which means,
when ever compiler finds structure, which starts with "struct" definition, then it aligns
the structure according to C rules, + according to __BITS__ too, so,
"-cstruct" + BITS 32 : defines structures and aligns them on 32 bit boundaries.
"-cstruct" + BITS 64 : defines structures and aligns them on 64 bit boundaries.

Q: Why do we have to support C structures in NASM?
A: Why not! NASM is built on that C\C++ thingy too, so, why couldn't we!

The development of NASMX is largely independent of the Netwide Assembler. Early on, a design choice was made to not break backward compatibility with old NASM versions whenever possible, this is because distributions like Debian will only allow new versions of software into their repositories after the software has been stable for some time.

Another thing to keep in mind is that NASM doesn't technically support structures.  There are a lot of features that NASM has that aren't actually hard-coded in. If you look at the source code, you'll notice that there is a bunch of macros in a file called standard.mac, one of which is the implementation of STRUC and ENDSTRUC. These are loaded into the nasm.exe file at compile time so they are always available to you. As you can tell by that file, structure support is just some fancy macro wrapper for the absolute directive. This makes since because structures are just made up of a bunch of label offsets to its members.

[encryptor256]

Let's cut old cpu support and old Ms Windows base and go with acceleration!

there is a huge segment of the population out there still running Linux and WinXP on 32-bit machines.  We can't just suddenly force them all to upgrade

a design choice was made to not break backward compatibility

Okay, okay, i agree, i wasn't thinking straight on that, sorry for that,
i blame early morning, when i posted that hostile idea, to cut backward compatibility.

Another thing to keep in mind is that NASM doesn't technically support structures.  There are a lot of features that NASM has that aren't actually hard-coded in. If you look at the source code, you'll notice that there is a bunch of macros in a file called standard.mac, one of which is the implementation of STRUC and ENDSTRUC.

This is something new, i was thinking otherwise, but it seems, that it's now other-otherwise, was thinking, that "struct" is not a macro, but it is.

Well, it seems, you both, already have some bright ideas, about, how to solve structure alignment, at least that part where NASMX supported Win64, maybe.

[Rob Neff]

Ok.  I'm 99% finished with part 1 - correcting the struc alignment issue.
I'm still working on the union stuff which is rather broken at the moment but struc's are fine now I believe.
I'll work on part 2 - pack()'ing - later on.

Can you download a snapshot and test please?

Confirmation is appreciated.

Thanks.

[encryptor256]

Ok.  I'm 99% finished with part 1 - correcting the struc alignment issue.
I'm still working on the union stuff which is rather broken at the moment but struc's are fine now I believe.
I'll work on part 2 - pack()'ing - later on.

Can you download a snapshot and test please?

Confirmation is appreciated.

Thanks.

Confirm!

Tested some winAPI and custom made structures.
Verified by brains and compared against Win64 Pelles C.
Result:
Structure Alignment now is correct!
* Well, at least, didn't managed to break it by finding any miss alignments.

Structure size's and member offset's reported, are correct.
By correct, i mean, they were equal.

Added attachment: "win64_demo1.asm", a file, where i did some testing, this file was derived from NASMX/WIN64/DEMO1.

Here is Win64 Pelles C, structure size + alignment + offset tester code, can be used for other structures too:

Code: [Select]

        #define ins(x) wsprintf(buffer,"%s\r\n%s: %d",buffer,#x,(int)&m.##x - (int)&m)
#define sz(x) wsprintf(buffer,"%s\r\n%s: %d",buffer,#x,sizeof(x))
#define initial(x) wsprintf(buffer,"\r\nSize: %d\r\nAligned: %d\r\n",sizeof(x),__alignof(x))

char buffer[4096];

        // TEST: MSG
// -------------------------------------------
MSG m;

initial(MSG);

ins(hwnd);
        ins(message);
        ins(wParam);
        ins(lParam);
        ins(time);
        ins(pt.x);
        ins(pt.y);
MessageBox(0,buffer,0,0);

Structures tested and can be found in attachment or elsewhere: CREATESTRUCT, STARTUPINFO, WNDCLASSEX, MyStructureD, MyStructureC, MyStructureB, MyStructureA, PAINTSTRUCT, LOGFONT, MSG.

Bye.

[Rob Neff]

Thank you for your efforts, encryptor256. 

Now, can I ask another favor?  I'd like you to write a console app, using NASMX only, where the program tests the generated struc size and offsets against known values.  It needs to assemble in both 32 and 64-bit mode.  The values to compare against must be defined such that, for example, MSG.wParam offset will change depending on BITS.  The program should printf() a message if it finds an error and return a positive value, otherwise printf() a success message and return zero (0).

I would like to see quite a few self-made strucs that test for the various permutations - kind of like your MyStructureX - but be ruthless in definitions trying to break it.  If possible, only %include 'nasmx.inc' and %include 'test.inc'.

I would like to include this work into NASMX as a way to automate testing and to prevent any regressions of future changes.  Please ensure NASMX copyright with yourself as the author for the source code.  Thanks again for your hard work.

[encryptor256]

write a console app, using NASMX only, where the program tests the generated struc size and offsets against known values.  It needs to assemble in both 32 and 64-bit mode.  The values to compare against must be defined such that, for example, MSG.wParam offset will change depending on BITS.  The program should printf() a message if it finds an error and return a positive value, otherwise printf() a success message and return zero (0).

I would like to see quite a few self-made strucs that test for the various permutations - kind of like your MyStructureX - but be ruthless in definitions trying to break it.  If possible, only %include 'nasmx.inc' and %include 'test.inc'.

Well, it seems like, our minds work with different frequencies, and my english is too bad, to understand the task.
It will take, too much time, to find out, what's on your mind.
AND, task is, too specific, to NASMX, cannot do it, don't have a big experience with it, sorry.

[Rob Neff]

OK.  Then I'll add that task to my to-do list if no one else offers to.  Thanks anyways.

[Mathi]

Hi Rob,
   Mind if i try it.?
I was thinking of a script (perl) which can do this automatically.

-Mathi

[Rob Neff]

Absolutely, Mathi!  Thank you for the assist! 

Please create a directory named 'test' under the trunk/nasmx root directory and put as much code in there as you deem necessary.  You'll probably need to synchronize your svn first.  There's no rush since I'll be finalizing the struc/union macros within the next few days. 

I have other work remaining for this next release as well.  If you have anything else you'd like to check in then now is the time. 

Edit: Grammar.

[Rob Neff]

Progress!  Structure and union definitions ( including nested ) and natural size alignment are now fully implemented!
You can download the latest snapshot to play with.

Nested unions containing nested structures/unions were causing me grief.  I spent the whole day rewriting the label offset algorithm due to the fact that, at macro instantion, there's no possible way to know the correct label offset if you're in a union because you don't yet know the alignment of future union members.

Consider the following structure definition as contained in WINSOCK.INC

Code: [Select]

NASMX_STRUC SOCKADDR_IN
    NASMX_RESERVE sin_family, int16_t, 1
    NASMX_RESERVE sin_port, uint16_t, 1
    NASMX_STRUC sin_addr, IN_ADDR
        NASMX_UNION S_un
            NASMX_STRUC S_un_b
                NASMX_RESERVE s_b1, uint8_t, 1
                NASMX_RESERVE s_b2, uint8_t, 1
                NASMX_RESERVE s_b3, uint8_t, 1
                NASMX_RESERVE s_b4, uint8_t, 1
            NASMX_ENDSTRUC
            NASMX_STRUC S_un_w
                NASMX_RESERVE s_w1, uint16_t, 1
                NASMX_RESERVE s_w2, uint16_t, 1
            NASMX_ENDSTRUC
    NASMX_RESERVE S_addr,   uint32_t, 1
        NASMX_ENDUNION
    NASMX_ENDSTRUC
    NASMX_RESERVE sin_zero, int8_t, 8
NASMX_ENDSTRUC

Yes, NASMX now portably and correctly supports those types of complex structure definitions. 

As you can see in the above structure, when the programmer specifies the union we cannot create offsets for the union members until after all members have been observed because at that point we do not know how large of an alignment we need yet.  That is the fallout of alignment.  Subsequently, the offset labeling now occurs in two phases using a form of iterative recursion.

Now that that's out of the way I'll begin working on providing packing and alignment macros to allow the programmer to override the default alignment requirements detected by NASMX during assembly.

Please let us know the results of any testing you do with NASMX regarding structures/unions here in this thread.
Thank you and happy testing!