lists.openwall.net   lists  /  announce  owl-users  owl-dev  john-users  john-dev  passwdqc-users  yescrypt  popa3d-users  /  oss-security  kernel-hardening  musl  sabotage  tlsify  passwords  /  crypt-dev  xvendor  /  Bugtraq  Full-Disclosure  linux-kernel  linux-netdev  linux-ext4  linux-hardening  linux-cve-announce  PHC 
Open Source and information security mailing list archives
 
Hash Suite: Windows password security audit tool. GUI, reports in PDF.
[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Message-ID: <7a91ff31-1f56-4d0c-a4a7-a305331ba97a@gmail.com>
Date: Wed, 4 Sep 2024 12:14:41 +0100
From: Usama Arif <usamaarif642@...il.com>
To: Barry Song <21cnbao@...il.com>
Cc: akpm@...ux-foundation.org, chengming.zhou@...ux.dev, david@...hat.com,
 hannes@...xchg.org, hughd@...gle.com, kernel-team@...a.com,
 linux-kernel@...r.kernel.org, linux-mm@...ck.org, nphamcs@...il.com,
 shakeel.butt@...ux.dev, willy@...radead.org, ying.huang@...el.com,
 yosryahmed@...gle.com, hanchuanhua@...o.com
Subject: Re: [PATCH v4 1/2] mm: store zero pages to be swapped out in a bitmap



On 04/09/2024 06:55, Barry Song wrote:
> On Thu, Jun 13, 2024 at 12:48 AM Usama Arif <usamaarif642@...il.com> wrote:
>>
>> Approximately 10-20% of pages to be swapped out are zero pages [1].
>> Rather than reading/writing these pages to flash resulting
>> in increased I/O and flash wear, a bitmap can be used to mark these
>> pages as zero at write time, and the pages can be filled at
>> read time if the bit corresponding to the page is set.
>> With this patch, NVMe writes in Meta server fleet decreased
>> by almost 10% with conventional swap setup (zswap disabled).
>>
>> [1] https://lore.kernel.org/all/20171018104832epcms5p1b2232e2236258de3d03d1344dde9fce0@epcms5p1/
>>
>> Signed-off-by: Usama Arif <usamaarif642@...il.com>
>> ---
>>  include/linux/swap.h |   1 +
>>  mm/page_io.c         | 114 ++++++++++++++++++++++++++++++++++++++++++-
>>  mm/swapfile.c        |  24 ++++++++-
>>  3 files changed, 136 insertions(+), 3 deletions(-)
>>
>> diff --git a/include/linux/swap.h b/include/linux/swap.h
>> index a11c75e897ec..e88563978441 100644
>> --- a/include/linux/swap.h
>> +++ b/include/linux/swap.h
>> @@ -299,6 +299,7 @@ struct swap_info_struct {
>>         signed char     type;           /* strange name for an index */
>>         unsigned int    max;            /* extent of the swap_map */
>>         unsigned char *swap_map;        /* vmalloc'ed array of usage counts */
>> +       unsigned long *zeromap;         /* vmalloc'ed bitmap to track zero pages */
>>         struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */
>>         struct swap_cluster_list free_clusters; /* free clusters list */
>>         unsigned int lowest_bit;        /* index of first free in swap_map */
>> diff --git a/mm/page_io.c b/mm/page_io.c
>> index a360857cf75d..39fc3919ce15 100644
>> --- a/mm/page_io.c
>> +++ b/mm/page_io.c
>> @@ -172,6 +172,88 @@ int generic_swapfile_activate(struct swap_info_struct *sis,
>>         goto out;
>>  }
>>
>> +static bool is_folio_page_zero_filled(struct folio *folio, int i)
>> +{
>> +       unsigned long *data;
>> +       unsigned int pos, last_pos = PAGE_SIZE / sizeof(*data) - 1;
>> +       bool ret = false;
>> +
>> +       data = kmap_local_folio(folio, i * PAGE_SIZE);
>> +       if (data[last_pos])
>> +               goto out;
>> +       for (pos = 0; pos < PAGE_SIZE / sizeof(*data); pos++) {
>> +               if (data[pos])
>> +                       goto out;
>> +       }
>> +       ret = true;
>> +out:
>> +       kunmap_local(data);
>> +       return ret;
>> +}
>> +
>> +static bool is_folio_zero_filled(struct folio *folio)
>> +{
>> +       unsigned int i;
>> +
>> +       for (i = 0; i < folio_nr_pages(folio); i++) {
>> +               if (!is_folio_page_zero_filled(folio, i))
>> +                       return false;
>> +       }
>> +       return true;
>> +}
>> +
>> +static void folio_zero_fill(struct folio *folio)
>> +{
>> +       unsigned int i;
>> +
>> +       for (i = 0; i < folio_nr_pages(folio); i++)
>> +               clear_highpage(folio_page(folio, i));
>> +}
>> +
>> +static void swap_zeromap_folio_set(struct folio *folio)
>> +{
>> +       struct swap_info_struct *sis = swp_swap_info(folio->swap);
>> +       swp_entry_t entry;
>> +       unsigned int i;
>> +
>> +       for (i = 0; i < folio_nr_pages(folio); i++) {
>> +               entry = page_swap_entry(folio_page(folio, i));
>> +               set_bit(swp_offset(entry), sis->zeromap);
>> +       }
>> +}
>> +
>> +static void swap_zeromap_folio_clear(struct folio *folio)
>> +{
>> +       struct swap_info_struct *sis = swp_swap_info(folio->swap);
>> +       swp_entry_t entry;
>> +       unsigned int i;
>> +
>> +       for (i = 0; i < folio_nr_pages(folio); i++) {
>> +               entry = page_swap_entry(folio_page(folio, i));
>> +               clear_bit(swp_offset(entry), sis->zeromap);
>> +       }
>> +}
>> +
>> +/*
>> + * Return the index of the first subpage which is not zero-filled
>> + * according to swap_info_struct->zeromap.
>> + * If all pages are zero-filled according to zeromap, it will return
>> + * folio_nr_pages(folio).
>> + */
>> +static unsigned int swap_zeromap_folio_test(struct folio *folio)
>> +{
>> +       struct swap_info_struct *sis = swp_swap_info(folio->swap);
>> +       swp_entry_t entry;
>> +       unsigned int i;
>> +
>> +       for (i = 0; i < folio_nr_pages(folio); i++) {
>> +               entry = page_swap_entry(folio_page(folio, i));
>> +               if (!test_bit(swp_offset(entry), sis->zeromap))
>> +                       return i;
>> +       }
>> +       return i;
>> +}
>> +
>>  /*
>>   * We may have stale swap cache pages in memory: notice
>>   * them here and get rid of the unnecessary final write.
>> @@ -195,6 +277,13 @@ int swap_writepage(struct page *page, struct writeback_control *wbc)
>>                 folio_unlock(folio);
>>                 return ret;
>>         }
>> +
>> +       if (is_folio_zero_filled(folio)) {
>> +               swap_zeromap_folio_set(folio);
>> +               folio_unlock(folio);
>> +               return 0;
>> +       }
>> +       swap_zeromap_folio_clear(folio);
>>         if (zswap_store(folio)) {
>>                 folio_start_writeback(folio);
>>                 folio_unlock(folio);
>> @@ -426,6 +515,26 @@ static void sio_read_complete(struct kiocb *iocb, long ret)
>>         mempool_free(sio, sio_pool);
>>  }
>>
>> +static bool swap_read_folio_zeromap(struct folio *folio)
>> +{
>> +       unsigned int idx = swap_zeromap_folio_test(folio);
>> +
>> +       if (idx == 0)
>> +               return false;
>> +
>> +       /*
>> +        * Swapping in a large folio that is partially in the zeromap is not
>> +        * currently handled. Return true without marking the folio uptodate so
>> +        * that an IO error is emitted (e.g. do_swap_page() will sigbus).
>> +        */
>> +       if (WARN_ON_ONCE(idx < folio_nr_pages(folio)))
>> +               return true;
> 
> Hi Usama, Yosry,
> 
> I feel the warning is wrong as we could have the case where idx==0
> is not zeromap but idx=1 is zeromap. idx == 0 doesn't necessarily
> mean we should return false.
> 
> What about the below change which both fixes the warning and unblocks
> large folios swap-in?
> 
Hi Barry,

I remembered when resending the zeromap series about the comment Yosry had made earlier, but checked that the mTHP swap-in was not in mm-unstable.
I should have checked the mailing list and commented!

I have not tested the below diff yet (will do in a few hours). But there might be a small issue with it. Have commented inline.

> diff --git a/mm/page_io.c b/mm/page_io.c
> index 4bc77d1c6bfa..7d7ff7064e2b 100644
> --- a/mm/page_io.c
> +++ b/mm/page_io.c
> @@ -226,26 +226,6 @@ static void swap_zeromap_folio_clear(struct folio *folio)
>  	}
>  }
>  
> -/*
> - * Return the index of the first subpage which is not zero-filled
> - * according to swap_info_struct->zeromap.
> - * If all pages are zero-filled according to zeromap, it will return
> - * folio_nr_pages(folio).
> - */
> -static unsigned int swap_zeromap_folio_test(struct folio *folio)
> -{
> -	struct swap_info_struct *sis = swp_swap_info(folio->swap);
> -	swp_entry_t entry;
> -	unsigned int i;
> -
> -	for (i = 0; i < folio_nr_pages(folio); i++) {
> -		entry = page_swap_entry(folio_page(folio, i));
> -		if (!test_bit(swp_offset(entry), sis->zeromap))
> -			return i;
> -	}
> -	return i;
> -}
> -
>  /*
>   * We may have stale swap cache pages in memory: notice
>   * them here and get rid of the unnecessary final write.
> @@ -524,9 +504,10 @@ static void sio_read_complete(struct kiocb *iocb, long ret)
>  
>  static bool swap_read_folio_zeromap(struct folio *folio)
>  {
> -	unsigned int idx = swap_zeromap_folio_test(folio);
> +	unsigned int nr_pages = folio_nr_pages(folio);
> +	unsigned int nr = swap_zeromap_entries_count(folio->swap, nr_pages);
>  
> -	if (idx == 0)
> +	if (nr == 0)
>  		return false;
>  
>  	/*
> @@ -534,7 +515,7 @@ static bool swap_read_folio_zeromap(struct folio *folio)
>  	 * currently handled. Return true without marking the folio uptodate so
>  	 * that an IO error is emitted (e.g. do_swap_page() will sigbus).
>  	 */
> -	if (WARN_ON_ONCE(idx < folio_nr_pages(folio)))
> +	if (WARN_ON_ONCE(nr < nr_pages))
>  		return true;
>  
>  	folio_zero_range(folio, 0, folio_size(folio));
> diff --git a/mm/swap.h b/mm/swap.h
> index f8711ff82f84..2d59e9d89e95 100644
> --- a/mm/swap.h
> +++ b/mm/swap.h
> @@ -80,6 +80,32 @@ static inline unsigned int folio_swap_flags(struct folio *folio)
>  {
>  	return swp_swap_info(folio->swap)->flags;
>  }
> +
> +/*
> + * Return the number of entries which are zero-filled according to
> + * swap_info_struct->zeromap. It isn't precise if the return value
> + * is larger than 0 and smaller than nr to avoid extra iterations,
> + * In this case, it means entries haven't consistent zeromap.
> + */
> +static inline unsigned int swap_zeromap_entries_count(swp_entry_t entry, int nr)
> +{
> +	struct swap_info_struct *sis = swp_swap_info(entry);
> +	unsigned long offset = swp_offset(entry);
> +	unsigned int type = swp_type(entry);
> +	unsigned int n = 0;
> +
> +	for (int i = 0; i < nr; i++) {
> +		entry = swp_entry(type, offset + i);
> +		if (test_bit(offset + i, sis->zeromap)) {

Should this be if (test_bit(swp_offset(entry), sis->zeromap))


Also, are you going to use this in alloc_swap_folio?
You mentioned above that this unblocks large folios swap-in, but I don't see
it in the diff here. I am guessing there is some change in alloc_swap_info that
uses swap_zeromap_entries_count?

Thanks
Usama

> +			if (i != n)
> +				return i;
> +			n++;
> +		}
> +	}
> +
> +	return n;
> +}
> +
>  #else /* CONFIG_SWAP */
>  struct swap_iocb;
>  static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug)
> @@ -171,6 +197,11 @@ static inline unsigned int folio_swap_flags(struct folio *folio)
>  {
>  	return 0;
>  }
> +
> +static inline unsigned int swap_zeromap_entries_count(swp_entry_t entry, int nr)
> +{
> +	return 0;
> +}
>  #endif /* CONFIG_SWAP */
>  
>  #endif /* _MM_SWAP_H */
> 

Powered by blists - more mailing lists

Powered by Openwall GNU/*/Linux Powered by OpenVZ