mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git
synced 2025-01-04 04:02:26 +00:00
91cdcd8d62
Profiling the munmap() of a zswapped memory region shows 60% of the total
cycles currently going into updating the zswap_pool_total_size.
There are three consumers of this counter:
- store, to enforce the globally configured pool limit
- meminfo & debugfs, to report the size to the user
- shrink, to determine the batch size for each cycle
Instead of aggregating everytime an entry enters or exits the zswap
pool, aggregate the value from the zpools on-demand:
- Stores aggregate the counter anyway upon success. Aggregating to
check the limit instead is the same amount of work.
- Meminfo & debugfs might benefit somewhat from a pre-aggregated
counter, but aren't exactly hotpaths.
- Shrinking can aggregate once for every cycle instead of doing it for
every freed entry. As the shrinker might work on tens or hundreds of
objects per scan cycle, this is a large reduction in aggregations.
The paths that benefit dramatically are swapin, swapoff, and unmaps.
There could be millions of pages being processed until somebody asks for
the pool size again. This eliminates the pool size updates from those
paths entirely.
Top profile entries for a 24G range munmap(), before:
38.54% zswap-unmap [kernel.kallsyms] [k] zs_zpool_total_size
12.51% zswap-unmap [kernel.kallsyms] [k] zpool_get_total_size
9.10% zswap-unmap [kernel.kallsyms] [k] zswap_update_total_size
2.95% zswap-unmap [kernel.kallsyms] [k] obj_cgroup_uncharge_zswap
2.88% zswap-unmap [kernel.kallsyms] [k] __slab_free
2.86% zswap-unmap [kernel.kallsyms] [k] xas_store
and after:
7.70% zswap-unmap [kernel.kallsyms] [k] __slab_free
7.16% zswap-unmap [kernel.kallsyms] [k] obj_cgroup_uncharge_zswap
6.74% zswap-unmap [kernel.kallsyms] [k] xas_store
It was also briefly considered to move to a single atomic in zswap
that is updated by the backends, since zswap only cares about the sum
of all pools anyway. However, zram directly needs per-pool information
out of zsmalloc. To keep the backend from having to update two atomics
every time, I opted for the lazy aggregation instead for now.
Link: https://lkml.kernel.org/r/20240312153901.3441-1-hannes@cmpxchg.org
Signed-off-by: Johannes Weiner <hannes@cmpxchg.org>
Acked-by: Yosry Ahmed <yosryahmed@google.com>
Reviewed-by: Chengming Zhou <chengming.zhou@linux.dev>
Reviewed-by: Nhat Pham <nphamcs@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
71 lines
1.7 KiB
C
71 lines
1.7 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef _LINUX_ZSWAP_H
|
|
#define _LINUX_ZSWAP_H
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/mm_types.h>
|
|
|
|
struct lruvec;
|
|
|
|
extern atomic_t zswap_stored_pages;
|
|
|
|
#ifdef CONFIG_ZSWAP
|
|
|
|
struct zswap_lruvec_state {
|
|
/*
|
|
* Number of pages in zswap that should be protected from the shrinker.
|
|
* This number is an estimate of the following counts:
|
|
*
|
|
* a) Recent page faults.
|
|
* b) Recent insertion to the zswap LRU. This includes new zswap stores,
|
|
* as well as recent zswap LRU rotations.
|
|
*
|
|
* These pages are likely to be warm, and might incur IO if the are written
|
|
* to swap.
|
|
*/
|
|
atomic_long_t nr_zswap_protected;
|
|
};
|
|
|
|
unsigned long zswap_total_pages(void);
|
|
bool zswap_store(struct folio *folio);
|
|
bool zswap_load(struct folio *folio);
|
|
void zswap_invalidate(swp_entry_t swp);
|
|
int zswap_swapon(int type, unsigned long nr_pages);
|
|
void zswap_swapoff(int type);
|
|
void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg);
|
|
void zswap_lruvec_state_init(struct lruvec *lruvec);
|
|
void zswap_folio_swapin(struct folio *folio);
|
|
bool is_zswap_enabled(void);
|
|
#else
|
|
|
|
struct zswap_lruvec_state {};
|
|
|
|
static inline bool zswap_store(struct folio *folio)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
static inline bool zswap_load(struct folio *folio)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
static inline void zswap_invalidate(swp_entry_t swp) {}
|
|
static inline int zswap_swapon(int type, unsigned long nr_pages)
|
|
{
|
|
return 0;
|
|
}
|
|
static inline void zswap_swapoff(int type) {}
|
|
static inline void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg) {}
|
|
static inline void zswap_lruvec_state_init(struct lruvec *lruvec) {}
|
|
static inline void zswap_folio_swapin(struct folio *folio) {}
|
|
|
|
static inline bool is_zswap_enabled(void)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
#endif
|
|
|
|
#endif /* _LINUX_ZSWAP_H */
|