2014-05-08 15:21:52 -04:00
/ *
* linux/ a r c h / x86 _ 6 4 / m c o u n t _ 6 4 . S
*
* Copyright ( C ) 2 0 1 4 S t e v e n R o s t e d t , R e d H a t I n c
* /
# include < l i n u x / l i n k a g e . h >
# include < a s m / p t r a c e . h >
# include < a s m / f t r a c e . h >
.code64
.section .entry .text , " ax"
# ifdef C O N F I G _ F U N C T I O N _ T R A C E R
# ifdef C C _ U S I N G _ F E N T R Y
# define f u n c t i o n _ h o o k _ _ f e n t r y _ _
# else
# define f u n c t i o n _ h o o k m c o u n t
# endif
2014-11-24 18:08:48 -05:00
/* All cases save the original rbp (8 bytes) */
# ifdef C O N F I G _ F R A M E _ P O I N T E R
# ifdef C C _ U S I N G _ F E N T R Y
/* Save parent and function stack frames (rip and rbp) */
# define M C O U N T _ F R A M E _ S I Z E ( 8 + 1 6 * 2 )
# else
/* Save just function stack frame (rip and rbp) */
# define M C O U N T _ F R A M E _ S I Z E ( 8 + 1 6 )
# endif
# else
/* No need to save a stack frame */
# define M C O U N T _ F R A M E _ S I Z E 8
# endif / * C O N F I G _ F R A M E _ P O I N T E R * /
2014-11-24 14:26:38 -05:00
/* Size of stack used to save mcount regs in save_mcount_regs */
2014-11-24 18:08:48 -05:00
# define M C O U N T _ R E G _ S I Z E ( S S + 8 + M C O U N T _ F R A M E _ S I Z E )
2014-11-24 14:26:38 -05:00
2014-11-24 11:43:39 -05:00
/ *
* gcc - p g o p t i o n a d d s a c a l l t o ' m c o u n t ' i n m o s t f u n c t i o n s .
* When - m f e n t r y i s u s e d , t h e c a l l i s t o ' f e n t r y ' a n d n o t ' m c o u n t '
* and i s d o n e b e f o r e t h e f u n c t i o n ' s s t a c k f r a m e i s s e t u p .
* They b o t h r e q u i r e a s e t o f r e g s t o b e s a v e d b e f o r e c a l l i n g
* any C c o d e a n d r e s t o r e d b e f o r e r e t u r n i n g b a c k t o t h e f u n c t i o n .
*
* On b o o t u p , a l l t h e s e c a l l s a r e c o n v e r t e d i n t o n o p s . W h e n t r a c i n g
* is e n a b l e d , t h e c a l l c a n j u m p t o e i t h e r f t r a c e _ c a l l e r o r
* ftrace_ r e g s _ c a l l e r . C a l l b a c k s ( t r a c i n g f u n c t i o n s ) t h a t r e q u i r e
* ftrace_ r e g s _ c a l l e r ( l i k e k p r o b e s ) n e e d t o h a v e p t _ r e g s p a s s e d t o
* it. F o r t h i s r e a s o n , t h e s i z e o f t h e p t _ r e g s s t r u c t u r e w i l l b e
* allocated o n t h e s t a c k a n d t h e r e q u i r e d m c o u n t r e g i s t e r s w i l l
* be s a v e d i n t h e l o c a t i o n s t h a t p t _ r e g s h a s t h e m i n .
* /
2014-11-24 21:38:40 -05:00
/ *
* @added: the amount of stack added before calling this
*
* After t h i s i s c a l l e d , t h e f o l l o w i n g r e g i s t e r s c o n t a i n :
*
* % rdi - h o l d s t h e a d d r e s s t h a t c a l l e d t h e t r a m p o l i n e
* % rsi - h o l d s t h e p a r e n t f u n c t i o n ( t r a c e d f u n c t i o n ' s r e t u r n a d d r e s s )
* % rdx - h o l d s t h e o r i g i n a l % r b p
* /
2014-11-24 13:06:05 -05:00
.macro save_mcount_regs added=0
2014-11-24 18:08:48 -05:00
/* Always save the original rbp */
pushq % r b p
# ifdef C O N F I G _ F R A M E _ P O I N T E R
/ *
* Stack t r a c e s w i l l s t o p a t t h e f t r a c e t r a m p o l i n e i f t h e f r a m e p o i n t e r
* is n o t s e t u p p r o p e r l y . I f f e n t r y i s u s e d , w e n e e d t o s a v e a f r a m e
* pointer f o r t h e p a r e n t a s w e l l a s t h e f u n c t i o n t r a c e d , b e c a u s e t h e
* fentry i s c a l l e d b e f o r e t h e s t a c k f r a m e i s s e t u p , w h e r e a s m c o u n t
* is c a l l e d a f t e r w a r d .
* /
# ifdef C C _ U S I N G _ F E N T R Y
/* Save the parent pointer (skip orig rbp and our return address) */
pushq \ a d d e d + 8 * 2 ( % r s p )
pushq % r b p
movq % r s p , % r b p
/* Save the return address (now skip orig rbp, rbp and parent) */
pushq \ a d d e d + 8 * 3 ( % r s p )
# else
/* Can't assume that rip is before this (unless added was zero) */
pushq \ a d d e d + 8 ( % r s p )
# endif
pushq % r b p
movq % r s p , % r b p
# endif / * C O N F I G _ F R A M E _ P O I N T E R * /
/ *
* We a d d e n o u g h s t a c k t o s a v e a l l r e g s .
* /
subq $ ( M C O U N T _ R E G _ S I Z E - M C O U N T _ F R A M E _ S I Z E ) , % r s p
2014-11-24 11:30:58 -05:00
movq % r a x , R A X ( % r s p )
movq % r c x , R C X ( % r s p )
movq % r d x , R D X ( % r s p )
movq % r s i , R S I ( % r s p )
movq % r d i , R D I ( % r s p )
movq % r8 , R 8 ( % r s p )
movq % r9 , R 9 ( % r s p )
2014-11-24 18:08:48 -05:00
/ *
* Save t h e o r i g i n a l R B P . E v e n t h o u g h t h e m c o u n t A B I d o e s n o t
* require t h i s , i t h e l p s o u t c a l l e r s .
* /
movq M C O U N T _ R E G _ S I Z E - 8 ( % r s p ) , % r d x
movq % r d x , R B P ( % r s p )
2014-11-24 21:38:40 -05:00
/* Copy the parent address into %rsi (second parameter) */
# ifdef C C _ U S I N G _ F E N T R Y
movq M C O U N T _ R E G _ S I Z E + 8 + \ a d d e d ( % r s p ) , % r s i
# else
/* %rdx contains original %rbp */
movq 8 ( % r d x ) , % r s i
# endif
2014-11-24 11:30:58 -05:00
/* Move RIP to its proper location */
2014-11-24 14:26:38 -05:00
movq M C O U N T _ R E G _ S I Z E + \ a d d e d ( % r s p ) , % r d i
2014-11-24 13:21:09 -05:00
movq % r d i , R I P ( % r s p )
2014-11-24 21:38:40 -05:00
/ *
* Now % r d i ( t h e f i r s t p a r a m e t e r ) h a s t h e r e t u r n a d d r e s s o f
* where f t r a c e _ c a l l r e t u r n s . B u t t h e c a l l b a c k s e x p e c t t h e
* the a d d r e s s o f t h e c a l l i t s e l f .
* /
subq $ M C O U N T _ I N S N _ S I Z E , % r d i
2014-11-24 11:30:58 -05:00
.endm
2014-11-24 13:06:05 -05:00
.macro restore_mcount_regs
2014-11-24 11:30:58 -05:00
movq R 9 ( % r s p ) , % r9
movq R 8 ( % r s p ) , % r8
movq R D I ( % r s p ) , % r d i
movq R S I ( % r s p ) , % r s i
movq R D X ( % r s p ) , % r d x
movq R C X ( % r s p ) , % r c x
movq R A X ( % r s p ) , % r a x
2014-11-24 18:08:48 -05:00
/* ftrace_regs_caller can modify %rbp */
movq R B P ( % r s p ) , % r b p
2014-11-24 14:26:38 -05:00
addq $ M C O U N T _ R E G _ S I Z E , % r s p
2014-11-24 18:08:48 -05:00
2014-11-24 11:30:58 -05:00
.endm
2014-11-24 14:54:27 -05:00
# ifdef C O N F I G _ D Y N A M I C _ F T R A C E
ENTRY( f u n c t i o n _ h o o k )
retq
END( f u n c t i o n _ h o o k )
2014-05-08 15:21:52 -04:00
ENTRY( f t r a c e _ c a l l e r )
2014-11-24 21:38:40 -05:00
/* save_mcount_regs fills in first two parameters */
save_ m c o u n t _ r e g s
GLOBAL( f t r a c e _ c a l l e r _ o p _ p t r )
/* Load the ftrace_ops into the 3rd parameter */
movq f u n c t i o n _ t r a c e _ o p ( % r i p ) , % r d x
2014-05-08 15:21:52 -04:00
/* regs go into 4th parameter (but make it NULL) */
movq $ 0 , % r c x
GLOBAL( f t r a c e _ c a l l )
call f t r a c e _ s t u b
2014-11-24 11:43:39 -05:00
restore_ m c o u n t _ r e g s
ftrace/x86: Add dynamic allocated trampoline for ftrace_ops
The current method of handling multiple function callbacks is to register
a list function callback that calls all the other callbacks based on
their hash tables and compare it to the function that the callback was
called on. But this is very inefficient.
For example, if you are tracing all functions in the kernel and then
add a kprobe to a function such that the kprobe uses ftrace, the
mcount trampoline will switch from calling the function trace callback
to calling the list callback that will iterate over all registered
ftrace_ops (in this case, the function tracer and the kprobes callback).
That means for every function being traced it checks the hash of the
ftrace_ops for function tracing and kprobes, even though the kprobes
is only set at a single function. The kprobes ftrace_ops is checked
for every function being traced!
Instead of calling the list function for functions that are only being
traced by a single callback, we can call a dynamically allocated
trampoline that calls the callback directly. The function graph tracer
already uses a direct call trampoline when it is being traced by itself
but it is not dynamically allocated. It's trampoline is static in the
kernel core. The infrastructure that called the function graph trampoline
can also be used to call a dynamically allocated one.
For now, only ftrace_ops that are not dynamically allocated can have
a trampoline. That is, users such as function tracer or stack tracer.
kprobes and perf allocate their ftrace_ops, and until there's a safe
way to free the trampoline, it can not be used. The dynamically allocated
ftrace_ops may, although, use the trampoline if the kernel is not
compiled with CONFIG_PREEMPT. But that will come later.
Tested-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Tested-by: Jiri Kosina <jkosina@suse.cz>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2014-07-02 23:23:31 -04:00
/ *
* The c o p i e d t r a m p o l i n e m u s t c a l l f t r a c e _ r e t u r n a s i t
* still m a y n e e d t o c a l l t h e f u n c t i o n g r a p h t r a c e r .
* /
GLOBAL( f t r a c e _ c a l l e r _ e n d )
GLOBAL( f t r a c e _ r e t u r n )
2014-05-08 15:21:52 -04:00
# ifdef C O N F I G _ F U N C T I O N _ G R A P H _ T R A C E R
GLOBAL( f t r a c e _ g r a p h _ c a l l )
jmp f t r a c e _ s t u b
# endif
GLOBAL( f t r a c e _ s t u b )
retq
END( f t r a c e _ c a l l e r )
ENTRY( f t r a c e _ r e g s _ c a l l e r )
2014-11-24 13:06:05 -05:00
/* Save the current flags before any operations that can change them */
2014-05-08 15:21:52 -04:00
pushfq
2014-11-24 13:06:05 -05:00
/* added 8 bytes to save flags */
2014-11-24 21:38:40 -05:00
save_ m c o u n t _ r e g s 8
/* save_mcount_regs fills in first two parameters */
GLOBAL( f t r a c e _ r e g s _ c a l l e r _ o p _ p t r )
/* Load the ftrace_ops into the 3rd parameter */
movq f u n c t i o n _ t r a c e _ o p ( % r i p ) , % r d x
2014-05-08 15:21:52 -04:00
/* Save the rest of pt_regs */
movq % r15 , R 1 5 ( % r s p )
movq % r14 , R 1 4 ( % r s p )
movq % r13 , R 1 3 ( % r s p )
movq % r12 , R 1 2 ( % r s p )
movq % r11 , R 1 1 ( % r s p )
movq % r10 , R 1 0 ( % r s p )
movq % r b x , R B X ( % r s p )
/* Copy saved flags */
2014-11-24 14:26:38 -05:00
movq M C O U N T _ R E G _ S I Z E ( % r s p ) , % r c x
2014-05-08 15:21:52 -04:00
movq % r c x , E F L A G S ( % r s p )
/* Kernel segments */
movq $ _ _ K E R N E L _ D S , % r c x
movq % r c x , S S ( % r s p )
movq $ _ _ K E R N E L _ C S , % r c x
movq % r c x , C S ( % r s p )
2014-11-24 13:06:05 -05:00
/* Stack - skipping return address and flags */
2014-11-24 14:26:38 -05:00
leaq M C O U N T _ R E G _ S I Z E + 8 * 2 ( % r s p ) , % r c x
2014-05-08 15:21:52 -04:00
movq % r c x , R S P ( % r s p )
/* regs go into 4th parameter */
leaq ( % r s p ) , % r c x
GLOBAL( f t r a c e _ r e g s _ c a l l )
call f t r a c e _ s t u b
/* Copy flags back to SS, to restore them */
movq E F L A G S ( % r s p ) , % r a x
2014-11-24 14:26:38 -05:00
movq % r a x , M C O U N T _ R E G _ S I Z E ( % r s p )
2014-05-08 15:21:52 -04:00
/* Handlers can change the RIP */
movq R I P ( % r s p ) , % r a x
2014-11-24 14:26:38 -05:00
movq % r a x , M C O U N T _ R E G _ S I Z E + 8 ( % r s p )
2014-05-08 15:21:52 -04:00
/* restore the rest of pt_regs */
movq R 1 5 ( % r s p ) , % r15
movq R 1 4 ( % r s p ) , % r14
movq R 1 3 ( % r s p ) , % r13
movq R 1 2 ( % r s p ) , % r12
movq R 1 0 ( % r s p ) , % r10
movq R B X ( % r s p ) , % r b x
2014-11-24 13:06:05 -05:00
restore_ m c o u n t _ r e g s
2014-05-08 15:21:52 -04:00
/* Restore flags */
popfq
ftrace/x86: Add dynamic allocated trampoline for ftrace_ops
The current method of handling multiple function callbacks is to register
a list function callback that calls all the other callbacks based on
their hash tables and compare it to the function that the callback was
called on. But this is very inefficient.
For example, if you are tracing all functions in the kernel and then
add a kprobe to a function such that the kprobe uses ftrace, the
mcount trampoline will switch from calling the function trace callback
to calling the list callback that will iterate over all registered
ftrace_ops (in this case, the function tracer and the kprobes callback).
That means for every function being traced it checks the hash of the
ftrace_ops for function tracing and kprobes, even though the kprobes
is only set at a single function. The kprobes ftrace_ops is checked
for every function being traced!
Instead of calling the list function for functions that are only being
traced by a single callback, we can call a dynamically allocated
trampoline that calls the callback directly. The function graph tracer
already uses a direct call trampoline when it is being traced by itself
but it is not dynamically allocated. It's trampoline is static in the
kernel core. The infrastructure that called the function graph trampoline
can also be used to call a dynamically allocated one.
For now, only ftrace_ops that are not dynamically allocated can have
a trampoline. That is, users such as function tracer or stack tracer.
kprobes and perf allocate their ftrace_ops, and until there's a safe
way to free the trampoline, it can not be used. The dynamically allocated
ftrace_ops may, although, use the trampoline if the kernel is not
compiled with CONFIG_PREEMPT. But that will come later.
Tested-by: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Tested-by: Jiri Kosina <jkosina@suse.cz>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2014-07-02 23:23:31 -04:00
/ *
* As t h i s j m p t o f t r a c e _ r e t u r n c a n b e a s h o r t j u m p
* it m u s t n o t b e c o p i e d i n t o t h e t r a m p o l i n e .
* The t r a m p o l i n e w i l l a d d t h e c o d e t o j u m p
* to t h e r e t u r n .
* /
GLOBAL( f t r a c e _ r e g s _ c a l l e r _ e n d )
2014-05-08 15:21:52 -04:00
jmp f t r a c e _ r e t u r n
2014-06-25 11:59:45 -04:00
2014-05-08 15:21:52 -04:00
END( f t r a c e _ r e g s _ c a l l e r )
# else / * ! C O N F I G _ D Y N A M I C _ F T R A C E * /
ENTRY( f u n c t i o n _ h o o k )
cmpq $ f t r a c e _ s t u b , f t r a c e _ t r a c e _ f u n c t i o n
jnz t r a c e
2014-11-24 14:58:17 -05:00
fgraph_trace :
2014-05-08 15:21:52 -04:00
# ifdef C O N F I G _ F U N C T I O N _ G R A P H _ T R A C E R
cmpq $ f t r a c e _ s t u b , f t r a c e _ g r a p h _ r e t u r n
jnz f t r a c e _ g r a p h _ c a l l e r
cmpq $ f t r a c e _ g r a p h _ e n t r y _ s t u b , f t r a c e _ g r a p h _ e n t r y
jnz f t r a c e _ g r a p h _ c a l l e r
# endif
GLOBAL( f t r a c e _ s t u b )
retq
trace :
2014-11-24 21:38:40 -05:00
/* save_mcount_regs fills in first two parameters */
save_ m c o u n t _ r e g s
2014-05-08 15:21:52 -04:00
call * f t r a c e _ t r a c e _ f u n c t i o n
2014-11-24 11:43:39 -05:00
restore_ m c o u n t _ r e g s
2014-05-08 15:21:52 -04:00
2014-11-24 14:58:17 -05:00
jmp f g r a p h _ t r a c e
2014-05-08 15:21:52 -04:00
END( f u n c t i o n _ h o o k )
# endif / * C O N F I G _ D Y N A M I C _ F T R A C E * /
# endif / * C O N F I G _ F U N C T I O N _ T R A C E R * /
# ifdef C O N F I G _ F U N C T I O N _ G R A P H _ T R A C E R
ENTRY( f t r a c e _ g r a p h _ c a l l e r )
2014-11-24 21:38:40 -05:00
/* Saves rbp into %rdx */
2014-11-24 11:43:39 -05:00
save_ m c o u n t _ r e g s
2014-05-08 15:21:52 -04:00
# ifdef C C _ U S I N G _ F E N T R Y
2014-11-24 14:26:38 -05:00
leaq M C O U N T _ R E G _ S I Z E + 8 ( % r s p ) , % r d i
2014-05-08 15:21:52 -04:00
movq $ 0 , % r d x / * N o f r a m e p o i n t e r s n e e d e d * /
# else
2014-11-24 21:38:40 -05:00
/* Save address of the return address of traced function */
2014-11-24 18:08:48 -05:00
leaq 8 ( % r d x ) , % r d i
2014-11-24 21:38:40 -05:00
/* ftrace does sanity checks against frame pointers */
2014-11-24 18:08:48 -05:00
movq ( % r d x ) , % r d x
2014-05-08 15:21:52 -04:00
# endif
movq R I P ( % r s p ) , % r s i
subq $ M C O U N T _ I N S N _ S I Z E , % r s i
call p r e p a r e _ f t r a c e _ r e t u r n
2014-11-24 11:43:39 -05:00
restore_ m c o u n t _ r e g s
2014-05-08 15:21:52 -04:00
retq
END( f t r a c e _ g r a p h _ c a l l e r )
GLOBAL( r e t u r n _ t o _ h a n d l e r )
subq $ 2 4 , % r s p
/* Save the return values */
movq % r a x , ( % r s p )
movq % r d x , 8 ( % r s p )
movq % r b p , % r d i
call f t r a c e _ r e t u r n _ t o _ h a n d l e r
movq % r a x , % r d i
movq 8 ( % r s p ) , % r d x
movq ( % r s p ) , % r a x
addq $ 2 4 , % r s p
jmp * % r d i
# endif
