diff --git a/arch/x86/kernel/fpu/xstate.c b/arch/x86/kernel/fpu/xstate.c index 80dcf0417f30..80836b94189e 100644 --- a/arch/x86/kernel/fpu/xstate.c +++ b/arch/x86/kernel/fpu/xstate.c @@ -1084,20 +1084,10 @@ static inline bool xfeatures_mxcsr_quirk(u64 xfeatures) return true; } -static void fill_gap(struct membuf *to, unsigned *last, unsigned offset) +static void copy_feature(bool from_xstate, struct membuf *to, void *xstate, + void *init_xstate, unsigned int size) { - if (*last >= offset) - return; - membuf_write(to, (void *)&init_fpstate.xsave + *last, offset - *last); - *last = offset; -} - -static void copy_part(struct membuf *to, unsigned *last, unsigned offset, - unsigned size, void *from) -{ - fill_gap(to, last, offset); - membuf_write(to, from, size); - *last = offset + size; + membuf_write(to, from_xstate ? xstate : init_xstate, size); } /* @@ -1109,10 +1099,10 @@ static void copy_part(struct membuf *to, unsigned *last, unsigned offset, */ void copy_xstate_to_kernel(struct membuf to, struct xregs_state *xsave) { + const unsigned int off_mxcsr = offsetof(struct fxregs_state, mxcsr); + struct xregs_state *xinit = &init_fpstate.xsave; struct xstate_header header; - const unsigned off_mxcsr = offsetof(struct fxregs_state, mxcsr); - unsigned size = to.left; - unsigned last = 0; + unsigned int zerofrom; int i; /* @@ -1122,41 +1112,68 @@ void copy_xstate_to_kernel(struct membuf to, struct xregs_state *xsave) header.xfeatures = xsave->header.xfeatures; header.xfeatures &= xfeatures_mask_user(); - if (header.xfeatures & XFEATURE_MASK_FP) - copy_part(&to, &last, 0, off_mxcsr, &xsave->i387); - if (header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM)) - copy_part(&to, &last, off_mxcsr, - MXCSR_AND_FLAGS_SIZE, &xsave->i387.mxcsr); - if (header.xfeatures & XFEATURE_MASK_FP) - copy_part(&to, &last, offsetof(struct fxregs_state, st_space), - 128, &xsave->i387.st_space); - if (header.xfeatures & XFEATURE_MASK_SSE) - copy_part(&to, &last, xstate_offsets[XFEATURE_SSE], - 256, &xsave->i387.xmm_space); - /* - * Fill xsave->i387.sw_reserved value for ptrace frame: - */ - copy_part(&to, &last, offsetof(struct fxregs_state, sw_reserved), - 48, xstate_fx_sw_bytes); - /* - * Copy xregs_state->header: - */ - copy_part(&to, &last, offsetof(struct xregs_state, header), - sizeof(header), &header); + /* Copy FP state up to MXCSR */ + copy_feature(header.xfeatures & XFEATURE_MASK_FP, &to, &xsave->i387, + &xinit->i387, off_mxcsr); + + /* Copy MXCSR when SSE or YMM are set in the feature mask */ + copy_feature(header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM), + &to, &xsave->i387.mxcsr, &xinit->i387.mxcsr, + MXCSR_AND_FLAGS_SIZE); + + /* Copy the remaining FP state */ + copy_feature(header.xfeatures & XFEATURE_MASK_FP, + &to, &xsave->i387.st_space, &xinit->i387.st_space, + sizeof(xsave->i387.st_space)); + + /* Copy the SSE state - shared with YMM, but independently managed */ + copy_feature(header.xfeatures & XFEATURE_MASK_SSE, + &to, &xsave->i387.xmm_space, &xinit->i387.xmm_space, + sizeof(xsave->i387.xmm_space)); + + /* Zero the padding area */ + membuf_zero(&to, sizeof(xsave->i387.padding)); + + /* Copy xsave->i387.sw_reserved */ + membuf_write(&to, xstate_fx_sw_bytes, sizeof(xsave->i387.sw_reserved)); + + /* Copy the user space relevant state of @xsave->header */ + membuf_write(&to, &header, sizeof(header)); + + zerofrom = offsetof(struct xregs_state, extended_state_area); for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) { /* - * Copy only in-use xstates: + * The ptrace buffer is in non-compacted XSAVE format. + * In non-compacted format disabled features still occupy + * state space, but there is no state to copy from in the + * compacted init_fpstate. The gap tracking will zero this + * later. */ - if ((header.xfeatures >> i) & 1) { - void *src = __raw_xsave_addr(xsave, i); + if (!(xfeatures_mask_user() & BIT_ULL(i))) + continue; - copy_part(&to, &last, xstate_offsets[i], - xstate_sizes[i], src); - } + /* + * If there was a feature or alignment gap, zero the space + * in the destination buffer. + */ + if (zerofrom < xstate_offsets[i]) + membuf_zero(&to, xstate_offsets[i] - zerofrom); + copy_feature(header.xfeatures & BIT_ULL(i), &to, + __raw_xsave_addr(xsave, i), + __raw_xsave_addr(xinit, i), + xstate_sizes[i]); + + /* + * Keep track of the last copied state in the non-compacted + * target buffer for gap zeroing. + */ + zerofrom = xstate_offsets[i] + xstate_sizes[i]; } - fill_gap(&to, &last, size); + + if (to.left) + membuf_zero(&to, to.left); } /*