CIS 4615 meeting -*- Outline -*- * calling conventions Based on chapter 6 of The IDA Pro Book by Chris Eagle, No Starch press, 2011. Using also material from chapter 4 of the book Practical Malware Analysis: The Hands-On Guide to Dissecting Malicious Software by Michael Sikorski and Andrew Honig, No Starch Press, 2012. See also the Intel architecture manuals at http://www.intel.com/content/www/us/en/processors/ architectures-software-developer-manuals.html ** stack frames (background) ------------------------------------------ STACK FRAME FOR A FUNCTION high addrs EBP --> +--------------------------+ | | | | | | | | | | | v | | | | | | | | | ESP --> | | +--------------------------+ low addrs push 0xC pop [eax+4] ------------------------------------------ Q: Does push increment or decrement ESP? decrements it Q: Where does the top of the stack go when pop is executed? Into the specified destination address Q: What happens if there is no space for the push? An exception happens ** _cdecl calling convention ------------------------------------------ FUNCTION CALL AND RETURN (GCC, ...) Using the _cdecl calling convention in x86 CALLER CALLEE ======================================== push argn ; last argument (rightmost) ... push arg1 ; first argument (leftmost) call fun ; pushes AFTER, sets EIP to fun push ebp mov ebp, esp ; work of fun pop ebp ret ; jump to AFTER AFTER: times n pop ------------------------------------------ This is for x86 (32 bit), for x64, arguments are typically in registers and use RBP and RIP for 64 bit and for 16 bit programs, enter and leave can also be used instead of such function preludes and epilogues In x86 one can also use pusha and popa (16 bit) or pushad and popad (32 bit) to push all the general registers and pop them Often used by shell code but not by compilers; doesn't work in 64 bit mode. Note that the arguments are pushed on in reverse order Q: Which argument is closest to where ebp points? the first argument! ------------------------------------------ SUMMARY POINTS FOR _CDECL arguments pushed on the stack in caller pops result in Benefits: first argument near top of stack, so works well with ------------------------------------------ ... right-to-left order ... arguments off the stack (cleans up the stack) ... eax register ... variable number of argument functions, like printf, where the early arguments determine the later ones Q: If a function has a variable number of arguments (like printf), who can best clean up the stack: the caller or callee? The caller ------------------------------------------ STACK FRAME FOR _CDECL high address +--------------------------------+ | arg n | +--------------------------------+ | ... | +--------------------------------+ | arg 1 | +--------------------------------+ | return address | +================================+ EBP ->| old EBP | +--------------------------------+ | local 1 | +--------------------------------+ | ... | +--------------------------------+ ESP ->| local M | +--------------------------------+ low address ------------------------------------------ visualize this over time. Q: So, in a 32 bit program, what is the address of arg 1? EBP+8 Q: What is the address of arg 2? EBP+12 ------------------------------------------ EXAMPLE C CODE #include void swap(int *x, int *y) { int temp = *x; *x = *y; *y = temp; } int main(int argc, char *argv[]) { int i = 3; int n = 4; printf("i = %d, n = %d\n", i, n); swap(&i, &n); printf("i = %d, n = %d\n", i, n); return 0; } ------------------------------------------ look at the executable in IDA Pro ------------------------------------------ ASSEMBLY CODE (from gcc -S -masm=intel) .text .globl _swap .def _swap; .scl 2; .type 32; .endef _swap: LFB7: .cfi_startproc push ebp .cfi_def_cfa_offset 8 .cfi_offset 5, -8 mov ebp, esp .cfi_def_cfa_register 5 sub esp, 16 mov eax, DWORD PTR [ebp+8] mov eax, DWORD PTR [eax] mov DWORD PTR [ebp-4], eax mov eax, DWORD PTR [ebp+12] mov edx, DWORD PTR [eax] mov eax, DWORD PTR [ebp+8] mov DWORD PTR [eax], edx mov eax, DWORD PTR [ebp+12] mov edx, DWORD PTR [ebp-4] mov DWORD PTR [eax], edx leave .cfi_restore 5 .cfi_def_cfa 4, 4 ret .cfi_endproc LFE7: .def ___main; .scl 2; .type 32; .endef .section .rdata,"dr" LC0: .ascii "i = %d, n = %d\12\0" .text .globl _main .def _main; .scl 2; .type 32; .endef _main: LFB8: .cfi_startproc push ebp .cfi_def_cfa_offset 8 .cfi_offset 5, -8 mov ebp, esp .cfi_def_cfa_register 5 and esp, -16 sub esp, 32 call ___main mov DWORD PTR [esp+28], 3 mov DWORD PTR [esp+24], 4 mov edx, DWORD PTR [esp+24] mov eax, DWORD PTR [esp+28] mov DWORD PTR [esp+8], edx mov DWORD PTR [esp+4], eax mov DWORD PTR [esp], OFFSET FLAT:LC0 call _printf lea eax, [esp+24] mov DWORD PTR [esp+4], eax lea eax, [esp+28] mov DWORD PTR [esp], eax call _swap mov edx, DWORD PTR [esp+24] mov eax, DWORD PTR [esp+28] mov DWORD PTR [esp+8], edx mov DWORD PTR [esp+4], eax mov DWORD PTR [esp], OFFSET FLAT:LC0 call _printf mov eax, 0 leave .cfi_restore 5 .cfi_def_cfa 4, 4 ret .cfi_endproc ------------------------------------------ ** stdcall (standard) calling convention ------------------------------------------ MICROSOFT "STANDARD" CALLING CONVENTION Using the _stdcall calling convention in x86 CALLER CALLEE ======================================== push argn ; last argument (rightmost) ... push arg1 ; first argument (leftmost) call fun ; pushes AFTER, sets EIP to fun push ebp mov ebp, esp ; work of fun pop ebp ret n*4 ; clean stack and return AFTER: ------------------------------------------ This is for x86 (32 bit), One could also use pushad and popad (32 bit) to push all the general registers and pop them Note that the arguments are pushed on in reverse order Q: Is that the same as cdecl? Yes Q: Which argument is closest to where ebp points? the first argument! (same as with _cdecl) ------------------------------------------ SUMMARY POINTS FOR _STDCALL arguments pushed on the stack in right-to-left order callee pops result in eax Benefits: cleanup code only in callee so ------------------------------------------ ... right-to-left order ... arguments off the stack (cleans up the stack) ... smaller object code size (may thus be faster) Q: Will this work with variable argument functions? No ------------------------------------------ STACK FRAME FOR _STDECL high address +--------------------------------+ | arg n | +--------------------------------+ | ... | +--------------------------------+ | arg 1 | +--------------------------------+ | return address | +--------------------------------+ | old EBP | +================================+ EBP ->| local 1 | +--------------------------------+ | ... | +--------------------------------+ ESP ->| local M | +--------------------------------+ low address ------------------------------------------ visualize this over time. Q: So, in a 32 bit program, what is the address of arg 1? EBP+8 Q: What is the address of arg 2? EBP+12 ------------------------------------------ EXAMPLE C CODE #include int demo_stdcall(int w, int x, int y, int z) { return w+x+y+z; } int main() { int res; res = demo_stdcall(1,22,333,4444); printf("res is %d", res); return 0; } ------------------------------------------ look at the executable in IDA Pro ------------------------------------------ .text:00401280 ; Attributes: bp-based frame .text:00401280 .text:00401280 _main proc near ; CODE XREF: start-82 .text:00401280 .text:00401280 res = dword ptr -4 .text:00401280 .text:00401280 push ebp .text:00401281 mov ebp, esp .text:00401283 push ecx ; save ecx .text:00401284 push 115Ch ; z (which is 4444) .text:00401289 push 14Dh ; y (which is 333) .text:0040128E push 16h ; x (= 22) .text:00401290 push 1 ; w .text:00401292 call demo_stdcall .text:00401297 add esp, 10h ; recover space from call .text:0040129A mov [ebp+res], eax ; save result in res .text:0040129D mov eax, [ebp+res] ; put res back in eax .text:004012A0 push eax ; push res on stack for call to printf .text:004012A1 push offset aResIsD ; "res is %d" .text:004012A6 call sub_401300 ; call printf("res is %d\n", res); .text:004012AB add esp, 8 ; recover stack from call .text:004012AE xor eax, eax ; return 0 .text:004012B0 mov esp, ebp .text:004012B2 pop ebp .text:004012B3 retn .text:004012B3 _main endp .text:00401260 ; Attributes: bp-based frame .text:00401260 .text:00401260 demo_stdcall proc near ; CODE XREF: _main+12 .text:00401260 .text:00401260 w = dword ptr 8 .text:00401260 x = dword ptr 0Ch .text:00401260 y = dword ptr 10h .text:00401260 z = dword ptr 14h .text:00401260 .text:00401260 push ebp .text:00401261 mov ebp, esp .text:00401263 mov eax, [ebp+w] .text:00401266 add eax, [ebp+x] .text:00401269 add eax, [ebp+y] .text:0040126C add eax, [ebp+z] .text:0040126F pop ebp .text:00401270 retn .text:00401270 demo_stdcall endp ------------------------------------------ Also look at vs/swap.exe in IDA pro ** fastcall calling convention ------------------------------------------ FASTCALL CALLING CONVENTION Like stdcall but: passes up to 2 parameters in registers (arg1 in ecx, arg2 in edx) rest on stack (in right-to-left order ------------------------------------------