linux-kernel - Re: [PATCH kernel v2] zstd: Fixing mixed module-builtin objects

lists.openwall.net		lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC
Open Source and information security mailing list archives
Hash Suite: Windows password security audit tool. GUI, reports in PDF.
[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Date:   Mon, 6 Jun 2022 15:12:37 +1000
From:   Alexey Kardashevskiy <aik@...abs.ru>
To:     Nathan Chancellor <nathan@...nel.org>,
        Nick Terrell <terrelln@...com>
Cc:     linux-kernel@...r.kernel.org,
        Nick Desaulniers <ndesaulniers@...gle.com>,
        Masahiro Yamada <masahiroy@...nel.org>
Subject: Re: [PATCH kernel v2] zstd: Fixing mixed module-builtin objects

Ping?


On 5/17/22 09:52, Nathan Chancellor wrote:
> Hi Nick,
> 
> Did you have any thoughts on this patch? It is necessary for enabling
> LTO with PowerPC:
> 
> https://lore.kernel.org/20220429064547.2334280-1-aik@ozlabs.ru/
> 
> If you don't have anything to send to Linus for the next cycle, perhaps
> this patch could be carried by the PowerPC folks with your ack?
> 
> Cheers,
> Nathan
> 
> On Fri, Apr 29, 2022 at 03:33:29PM +1000, Alexey Kardashevskiy wrote:
>> With CONFIG_ZSTD_COMPRESS=m and CONFIG_ZSTD_DECOMPRESS=y we end up in
>> a situation when files from lib/zstd/common/ are compiled once to be
>> linked later for ZSTD_DECOMPRESS (build-in) and ZSTD_COMPRESS (module)
>> even though CFLAGS are different for builtins and modules.
>> So far somehow this was not a problem but enabling LLVM LTO exposes
>> the problem as:
>>
>> ld.lld: error: linking module flags 'Code Model': IDs have conflicting values in 'lib/built-in.a(zstd_common.o at 5868)' and 'ld-temp.o'
>>
>> This particular conflict is caused by KBUILD_CFLAGS=-mcmodel=medium vs.
>> KBUILD_CFLAGS_MODULE=-mcmodel=large , modules use the large model on
>> POWERPC as explained at
>> https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/powerpc/Makefile?h=v5.18-rc4#n127
>> but the current use of common files is wrong anyway.
>>
>> This works around the issue by introducing a zstd_common module with
>> shared code.
>>
>> Cc: Masahiro Yamada <masahiroy@...nel.org>
>> Signed-off-by: Alexey Kardashevskiy <aik@...abs.ru>
>> ---
>> Changes:
>> v2:
>> * add a new module instead of inlining common bits
>>
>> ---
>>   lib/zstd/Makefile                | 18 ++++++++----------
>>   lib/zstd/common/entropy_common.c |  4 +++-
>>   lib/zstd/common/zstd_common.c    |  7 +++++++
>>   lib/Kconfig                      |  8 ++++++--
>>   4 files changed, 24 insertions(+), 13 deletions(-)
>>
>> diff --git a/lib/zstd/Makefile b/lib/zstd/Makefile
>> index fc45339fc3a3..440bd0007ae2 100644
>> --- a/lib/zstd/Makefile
>> +++ b/lib/zstd/Makefile
>> @@ -10,14 +10,10 @@
>>   # ################################################################
>>   obj-$(CONFIG_ZSTD_COMPRESS) += zstd_compress.o
>>   obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd_decompress.o
>> +obj-$(CONFIG_ZSTD_COMMON) += zstd_common.o
>>   
>>   zstd_compress-y := \
>>   		zstd_compress_module.o \
>> -		common/debug.o \
>> -		common/entropy_common.o \
>> -		common/error_private.o \
>> -		common/fse_decompress.o \
>> -		common/zstd_common.o \
>>   		compress/fse_compress.o \
>>   		compress/hist.o \
>>   		compress/huf_compress.o \
>> @@ -33,12 +29,14 @@ zstd_compress-y := \
>>   
>>   zstd_decompress-y := \
>>   		zstd_decompress_module.o \
>> -		common/debug.o \
>> -		common/entropy_common.o \
>> -		common/error_private.o \
>> -		common/fse_decompress.o \
>> -		common/zstd_common.o \
>>   		decompress/huf_decompress.o \
>>   		decompress/zstd_ddict.o \
>>   		decompress/zstd_decompress.o \
>>   		decompress/zstd_decompress_block.o \
>> +
>> +zstd_common-y := \
>> +		common/debug.o \
>> +		common/entropy_common.o \
>> +		common/error_private.o \
>> +		common/fse_decompress.o \
>> +		common/zstd_common.o \
>> diff --git a/lib/zstd/common/entropy_common.c b/lib/zstd/common/entropy_common.c
>> index 53b47a2b52ff..f84612627471 100644
>> --- a/lib/zstd/common/entropy_common.c
>> +++ b/lib/zstd/common/entropy_common.c
>> @@ -15,6 +15,7 @@
>>   /* *************************************
>>   *  Dependencies
>>   ***************************************/
>> +#include <linux/module.h>
>>   #include "mem.h"
>>   #include "error_private.h"       /* ERR_*, ERROR */
>>   #define FSE_STATIC_LINKING_ONLY  /* FSE_MIN_TABLELOG */
>> @@ -239,7 +240,7 @@ size_t FSE_readNCount(
>>   {
>>       return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0);
>>   }
>> -
>> +EXPORT_SYMBOL_GPL(FSE_readNCount);
>>   
>>   /*! HUF_readStats() :
>>       Read compact Huffman tree, saved by HUF_writeCTable().
>> @@ -255,6 +256,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
>>       U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32];
>>       return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0);
>>   }
>> +EXPORT_SYMBOL_GPL(HUF_readStats);
>>   
>>   FORCE_INLINE_TEMPLATE size_t
>>   HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats,
>> diff --git a/lib/zstd/common/zstd_common.c b/lib/zstd/common/zstd_common.c
>> index 3d7e35b309b5..06f62b2026d5 100644
>> --- a/lib/zstd/common/zstd_common.c
>> +++ b/lib/zstd/common/zstd_common.c
>> @@ -13,6 +13,7 @@
>>   /*-*************************************
>>   *  Dependencies
>>   ***************************************/
>> +#include <linux/module.h>
>>   #define ZSTD_DEPS_NEED_MALLOC
>>   #include "zstd_deps.h"   /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */
>>   #include "error_private.h"
>> @@ -59,6 +60,7 @@ void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem)
>>           return customMem.customAlloc(customMem.opaque, size);
>>       return ZSTD_malloc(size);
>>   }
>> +EXPORT_SYMBOL_GPL(ZSTD_customMalloc);
>>   
>>   void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
>>   {
>> @@ -71,6 +73,7 @@ void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem)
>>       }
>>       return ZSTD_calloc(1, size);
>>   }
>> +EXPORT_SYMBOL_GPL(ZSTD_customCalloc);
>>   
>>   void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
>>   {
>> @@ -81,3 +84,7 @@ void ZSTD_customFree(void* ptr, ZSTD_customMem customMem)
>>               ZSTD_free(ptr);
>>       }
>>   }
>> +EXPORT_SYMBOL_GPL(ZSTD_customFree);
>> +
>> +MODULE_LICENSE("Dual BSD/GPL");
>> +MODULE_DESCRIPTION("Zstd Common");
>> diff --git a/lib/Kconfig b/lib/Kconfig
>> index 087e06b4cdfd..33f3a7054cdd 100644
>> --- a/lib/Kconfig
>> +++ b/lib/Kconfig
>> @@ -333,12 +333,16 @@ config LZ4HC_COMPRESS
>>   config LZ4_DECOMPRESS
>>   	tristate
>>   
>> +config ZSTD_COMMON
>> +	select XXHASH
>> +	tristate
>> +
>>   config ZSTD_COMPRESS
>> -	select XXHASH
>> +	select ZSTD_COMMON
>>   	tristate
>>   
>>   config ZSTD_DECOMPRESS
>> -	select XXHASH
>> +	select ZSTD_COMMON
>>   	tristate
>>   
>>   source "lib/xz/Kconfig"
>> -- 
>> 2.30.2
>>
>>

-- 
Alexey