2017-02-25 02:01:12 +03:00
# ifndef __LZ4DEFS_H__
# define __LZ4DEFS_H__
2013-07-09 03:01:45 +04:00
/*
2017-02-25 02:01:12 +03:00
* lz4defs . h - - common and architecture specific defines for the kernel usage
* LZ4 - Fast LZ compression algorithm
* Copyright ( C ) 2011 - 2016 , Yann Collet .
* BSD 2 - Clause License ( http : //www.opensource.org/licenses/bsd-license.php)
* Redistribution and use in source and binary forms , with or without
* modification , are permitted provided that the following conditions are
* met :
* * Redistributions of source code must retain the above copyright
* notice , this list of conditions and the following disclaimer .
* * Redistributions in binary form must reproduce the above
* copyright notice , this list of conditions and the following disclaimer
* in the documentation and / or other materials provided with the
* distribution .
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* " AS IS " AND ANY EXPRESS OR IMPLIED WARRANTIES , INCLUDING , BUT NOT
* LIMITED TO , THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED . IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT , INDIRECT , INCIDENTAL ,
* SPECIAL , EXEMPLARY , OR CONSEQUENTIAL DAMAGES ( INCLUDING , BUT NOT
* LIMITED TO , PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES ; LOSS OF USE ,
* DATA , OR PROFITS ; OR BUSINESS INTERRUPTION ) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY , WHETHER IN CONTRACT , STRICT LIABILITY , OR TORT
* ( INCLUDING NEGLIGENCE OR OTHERWISE ) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE , EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE .
* You can contact the author at :
* - LZ4 homepage : http : //www.lz4.org
* - LZ4 source repository : https : //github.com/lz4/lz4
2013-07-09 03:01:45 +04:00
*
2017-02-25 02:01:12 +03:00
* Changed for kernel usage by :
* Sven Schmidt < 4 sschmid @ informatik . uni - hamburg . de >
2013-07-09 03:01:45 +04:00
*/
2017-02-25 02:01:12 +03:00
# include <asm/unaligned.h>
# include <linux/string.h> /* memset, memcpy */
# define FORCE_INLINE __always_inline
/*-************************************
* Basic Types
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
# include <linux/types.h>
typedef uint8_t BYTE ;
typedef uint16_t U16 ;
typedef uint32_t U32 ;
typedef int32_t S32 ;
typedef uint64_t U64 ;
typedef uintptr_t uptrval ;
/*-************************************
* Architecture specifics
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2016-04-10 00:05:34 +03:00
# if defined(CONFIG_64BIT)
2013-07-09 03:01:45 +04:00
# define LZ4_ARCH64 1
# else
# define LZ4_ARCH64 0
# endif
2017-02-25 02:01:12 +03:00
# if defined(__LITTLE_ENDIAN)
# define LZ4_LITTLE_ENDIAN 1
# else
# define LZ4_LITTLE_ENDIAN 0
2013-07-09 03:01:45 +04:00
# endif
2017-02-25 02:01:12 +03:00
/*-************************************
* Constants
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
# define MINMATCH 4
# define WILDCOPYLENGTH 8
# define LASTLITERALS 5
# define MFLIMIT (WILDCOPYLENGTH + MINMATCH)
2018-10-31 01:07:28 +03:00
/*
* ensure it ' s possible to write 2 x wildcopyLength
* without overflowing output buffer
*/
# define MATCH_SAFEGUARD_DISTANCE ((2 * WILDCOPYLENGTH) - MINMATCH)
2017-02-25 02:01:12 +03:00
/* Increase this value ==> compression run slower on incompressible data */
# define LZ4_SKIPTRIGGER 6
# define HASH_UNIT sizeof(size_t)
# define KB (1 << 10)
# define MB (1 << 20)
# define GB (1U << 30)
# define MAXD_LOG 16
# define MAX_DISTANCE ((1 << MAXD_LOG) - 1)
# define STEPSIZE sizeof(size_t)
# define ML_BITS 4
# define ML_MASK ((1U << ML_BITS) - 1)
2013-07-09 03:01:45 +04:00
# define RUN_BITS (8 - ML_BITS)
# define RUN_MASK ((1U << RUN_BITS) - 1)
2017-02-25 02:01:12 +03:00
/*-************************************
* Reading and writing into memory
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
static FORCE_INLINE U16 LZ4_read16 ( const void * ptr )
{
return get_unaligned ( ( const U16 * ) ptr ) ;
}
static FORCE_INLINE U32 LZ4_read32 ( const void * ptr )
{
return get_unaligned ( ( const U32 * ) ptr ) ;
}
static FORCE_INLINE size_t LZ4_read_ARCH ( const void * ptr )
{
return get_unaligned ( ( const size_t * ) ptr ) ;
}
static FORCE_INLINE void LZ4_write16 ( void * memPtr , U16 value )
{
put_unaligned ( value , ( U16 * ) memPtr ) ;
}
static FORCE_INLINE void LZ4_write32 ( void * memPtr , U32 value )
{
put_unaligned ( value , ( U32 * ) memPtr ) ;
}
static FORCE_INLINE U16 LZ4_readLE16 ( const void * memPtr )
{
return get_unaligned_le16 ( memPtr ) ;
}
static FORCE_INLINE void LZ4_writeLE16 ( void * memPtr , U16 value )
{
return put_unaligned_le16 ( value , memPtr ) ;
}
2020-08-15 03:30:10 +03:00
/*
* LZ4 relies on memcpy with a constant size being inlined . In freestanding
* environments , the compiler can ' t assume the implementation of memcpy ( ) is
* standard compliant , so apply its specialized memcpy ( ) inlining logic . When
* possible , use __builtin_memcpy ( ) to tell the compiler to analyze memcpy ( )
* as - if it were standard compliant , so it can inline it in freestanding
* environments . This is needed when decompressing the Linux Kernel , for example .
*/
# define LZ4_memcpy(dst, src, size) __builtin_memcpy(dst, src, size)
lib/lz4: explicitly support in-place decompression
LZ4 final literal copy could be overlapped when doing
in-place decompression, so it's unsafe to just use memcpy()
on an optimized memcpy approach but memmove() instead.
Upstream LZ4 has updated this years ago [1] (and the impact
is non-sensible [2] plus only a few bytes remain), this commit
just synchronizes LZ4 upstream code to the kernel side as well.
It can be observed as EROFS in-place decompression failure
on specific files when X86_FEATURE_ERMS is unsupported,
memcpy() optimization of commit 59daa706fbec ("x86, mem:
Optimize memcpy by avoiding memory false dependece") will
be enabled then.
Currently most modern x86-CPUs support ERMS, these CPUs just
use "rep movsb" approach so no problem at all. However, it can
still be verified with forcely disabling ERMS feature...
arch/x86/lib/memcpy_64.S:
ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \
- "jmp memcpy_erms", X86_FEATURE_ERMS
+ "jmp memcpy_orig", X86_FEATURE_ERMS
We didn't observe any strange on arm64/arm/x86 platform before
since most memcpy() would behave in an increasing address order
("copy upwards" [3]) and it's the correct order of in-place
decompression but it really needs an update to memmove() for sure
considering it's an undefined behavior according to the standard
and some unique optimization already exists in the kernel.
[1] https://github.com/lz4/lz4/commit/33cb8518ac385835cc17be9a770b27b40cd0e15b
[2] https://github.com/lz4/lz4/pull/717#issuecomment-497818921
[3] https://sourceware.org/bugzilla/show_bug.cgi?id=12518
Link: https://lkml.kernel.org/r/20201122030749.2698994-1-hsiangkao@redhat.com
Signed-off-by: Gao Xiang <hsiangkao@redhat.com>
Reviewed-by: Nick Terrell <terrelln@fb.com>
Cc: Yann Collet <yann.collet.73@gmail.com>
Cc: Miao Xie <miaoxie@huawei.com>
Cc: Chao Yu <yuchao0@huawei.com>
Cc: Li Guifu <bluce.liguifu@huawei.com>
Cc: Guo Xuenan <guoxuenan@huawei.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2020-12-16 07:44:03 +03:00
# define LZ4_memmove(dst, src, size) __builtin_memmove(dst, src, size)
2020-08-15 03:30:10 +03:00
2017-02-25 02:01:12 +03:00
static FORCE_INLINE void LZ4_copy8 ( void * dst , const void * src )
{
# if LZ4_ARCH64
U64 a = get_unaligned ( ( const U64 * ) src ) ;
put_unaligned ( a , ( U64 * ) dst ) ;
# else
U32 a = get_unaligned ( ( const U32 * ) src ) ;
U32 b = get_unaligned ( ( const U32 * ) src + 1 ) ;
put_unaligned ( a , ( U32 * ) dst ) ;
put_unaligned ( b , ( U32 * ) dst + 1 ) ;
# endif
}
/*
* customized variant of memcpy ,
* which can overwrite up to 7 bytes beyond dstEnd
*/
static FORCE_INLINE void LZ4_wildCopy ( void * dstPtr ,
const void * srcPtr , void * dstEnd )
{
BYTE * d = ( BYTE * ) dstPtr ;
const BYTE * s = ( const BYTE * ) srcPtr ;
BYTE * const e = ( BYTE * ) dstEnd ;
do {
LZ4_copy8 ( d , s ) ;
d + = 8 ;
s + = 8 ;
} while ( d < e ) ;
}
static FORCE_INLINE unsigned int LZ4_NbCommonBytes ( register size_t val )
{
# if LZ4_LITTLE_ENDIAN
return __ffs ( val ) > > 3 ;
2013-07-09 03:01:49 +04:00
# else
2017-02-25 02:01:12 +03:00
return ( BITS_PER_LONG - 1 - __fls ( val ) ) > > 3 ;
# endif
}
static FORCE_INLINE unsigned int LZ4_count (
const BYTE * pIn ,
const BYTE * pMatch ,
const BYTE * pInLimit )
{
const BYTE * const pStart = pIn ;
while ( likely ( pIn < pInLimit - ( STEPSIZE - 1 ) ) ) {
size_t const diff = LZ4_read_ARCH ( pMatch ) ^ LZ4_read_ARCH ( pIn ) ;
if ( ! diff ) {
pIn + = STEPSIZE ;
pMatch + = STEPSIZE ;
continue ;
}
pIn + = LZ4_NbCommonBytes ( diff ) ;
return ( unsigned int ) ( pIn - pStart ) ;
}
# if LZ4_ARCH64
if ( ( pIn < ( pInLimit - 3 ) )
& & ( LZ4_read32 ( pMatch ) = = LZ4_read32 ( pIn ) ) ) {
pIn + = 4 ;
pMatch + = 4 ;
}
2013-07-09 03:01:49 +04:00
# endif
2013-07-09 03:01:45 +04:00
2017-02-25 02:01:12 +03:00
if ( ( pIn < ( pInLimit - 1 ) )
& & ( LZ4_read16 ( pMatch ) = = LZ4_read16 ( pIn ) ) ) {
pIn + = 2 ;
pMatch + = 2 ;
}
2013-07-09 03:01:45 +04:00
2017-02-25 02:01:12 +03:00
if ( ( pIn < pInLimit ) & & ( * pMatch = = * pIn ) )
pIn + + ;
2013-07-09 03:01:45 +04:00
2017-02-25 02:01:12 +03:00
return ( unsigned int ) ( pIn - pStart ) ;
}
2013-07-09 03:01:45 +04:00
2017-02-25 02:01:12 +03:00
typedef enum { noLimit = 0 , limitedOutput = 1 } limitedOutput_directive ;
typedef enum { byPtr , byU32 , byU16 } tableType_t ;
2013-07-09 03:01:49 +04:00
2017-02-25 02:01:12 +03:00
typedef enum { noDict = 0 , withPrefix64k , usingExtDict } dict_directive ;
typedef enum { noDictIssue = 0 , dictSmall } dictIssue_directive ;
2013-07-09 03:01:49 +04:00
2017-02-25 02:01:12 +03:00
typedef enum { endOnOutputSize = 0 , endOnInputSize = 1 } endCondition_directive ;
2018-10-31 01:07:28 +03:00
typedef enum { decode_full_block = 0 , partial_decode = 1 } earlyEnd_directive ;
# define LZ4_STATIC_ASSERT(c) BUILD_BUG_ON(!(c))
2013-07-09 03:01:45 +04:00
2017-02-25 02:01:12 +03:00
# endif