| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <CAPhsuW5trTZZJ7asB6j+LD7Kx7omxtqBPKGN11-wpm8V6hNCgA@mail.gmail.com>
Date: Thu, 30 May 2019 11:52:48 -0700
From: Song Liu <liu.song.a23@...il.com>
To: Roman Gushchin <guro@...com>
Cc: Alexei Starovoitov <ast@...nel.org>,
Daniel Borkmann <daniel@...earbox.net>,
bpf <bpf@...r.kernel.org>, Kernel Team <kernel-team@...com>,
open list <linux-kernel@...r.kernel.org>
Subject: Re: [PATCH bpf-next 4/5] bpf: rework memlock-based memory accounting
for maps
On Wed, May 29, 2019 at 6:05 PM Roman Gushchin <guro@...com> wrote:
>
> In order to unify the existing memlock charging code with the
> memcg-based memory accounting, which will be added later, let's
> rework the current scheme.
>
> Currently the following design is used:
> 1) .alloc() callback optionally checks if the allocation will likely
> succeed using bpf_map_precharge_memlock()
> 2) .alloc() performs actual allocations
> 3) .alloc() callback calculates map cost and sets map.memory.pages
> 4) map_create() calls bpf_map_init_memlock() which sets map.memory.user
> and performs actual charging; in case of failure the map is
> destroyed
> <map is in use>
> 1) bpf_map_free_deferred() calls bpf_map_release_memlock(), which
> performs uncharge and releases the user
> 2) .map_free() callback releases the memory
>
> The scheme can be simplified and made more robust:
> 1) .alloc() calculates map cost and calls bpf_map_charge_init()
> 2) bpf_map_charge_init() sets map.memory.user and performs actual
> charge
> 3) .alloc() performs actual allocations
> <map is in use>
> 1) .map_free() callback releases the memory
> 2) bpf_map_charge_finish() performs uncharge and releases the user
>
> The new scheme also allows to reuse bpf_map_charge_init()/finish()
> functions for memcg-based accounting. Because charges are performed
> before actual allocations and uncharges after freeing the memory,
> no bogus memory pressure can be created.
>
> In cases when the map structure is not available (e.g. it's not
> created yet, or is already destroyed), on-stack bpf_map_memory
> structure is used. The charge can be transferred with the
> bpf_map_charge_move() function.
>
> Signed-off-by: Roman Gushchin <guro@...com>
Looks good.
Acked-by: Song Liu <songliubraving@...com>
> ---
> include/linux/bpf.h | 5 ++-
> kernel/bpf/arraymap.c | 10 +++--
> kernel/bpf/cpumap.c | 8 ++--
> kernel/bpf/devmap.c | 13 ++++---
> kernel/bpf/hashtab.c | 11 +++---
> kernel/bpf/local_storage.c | 9 +++--
> kernel/bpf/lpm_trie.c | 5 +--
> kernel/bpf/queue_stack_maps.c | 9 +++--
> kernel/bpf/reuseport_array.c | 9 +++--
> kernel/bpf/stackmap.c | 30 ++++++++-------
> kernel/bpf/syscall.c | 69 +++++++++++++++++------------------
> kernel/bpf/xskmap.c | 9 +++--
> net/core/bpf_sk_storage.c | 8 ++--
> net/core/sock_map.c | 5 ++-
> 14 files changed, 112 insertions(+), 88 deletions(-)
>
> diff --git a/include/linux/bpf.h b/include/linux/bpf.h
> index 980b7a9bdd21..6187203b0414 100644
> --- a/include/linux/bpf.h
> +++ b/include/linux/bpf.h
> @@ -600,9 +600,12 @@ struct bpf_map *__bpf_map_get(struct fd f);
> struct bpf_map * __must_check bpf_map_inc(struct bpf_map *map, bool uref);
> void bpf_map_put_with_uref(struct bpf_map *map);
> void bpf_map_put(struct bpf_map *map);
> -int bpf_map_precharge_memlock(u32 pages);
> int bpf_map_charge_memlock(struct bpf_map *map, u32 pages);
> void bpf_map_uncharge_memlock(struct bpf_map *map, u32 pages);
> +int bpf_map_charge_init(struct bpf_map_memory *mem, u32 pages);
> +void bpf_map_charge_finish(struct bpf_map_memory *mem);
> +void bpf_map_charge_move(struct bpf_map_memory *dst,
> + struct bpf_map_memory *src);
> void *bpf_map_area_alloc(size_t size, int numa_node);
> void bpf_map_area_free(void *base);
> void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr);
> diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
> index 8fda24e78193..3552da4407d9 100644
> --- a/kernel/bpf/arraymap.c
> +++ b/kernel/bpf/arraymap.c
> @@ -83,6 +83,7 @@ static struct bpf_map *array_map_alloc(union bpf_attr *attr)
> u32 elem_size, index_mask, max_entries;
> bool unpriv = !capable(CAP_SYS_ADMIN);
> u64 cost, array_size, mask64;
> + struct bpf_map_memory mem;
> struct bpf_array *array;
>
> elem_size = round_up(attr->value_size, 8);
> @@ -125,23 +126,26 @@ static struct bpf_map *r(union bpf_attr *attr)
> }
> cost = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
>
> - ret = bpf_map_precharge_memlock(cost);
> + ret = bpf_map_charge_init(&mem, cost);
> if (ret < 0)
> return ERR_PTR(ret);
>
> /* allocate all map elements and zero-initialize them */
> array = bpf_map_area_alloc(array_size, numa_node);
> - if (!array)
> + if (!array) {
> + bpf_map_charge_finish(&mem);
> return ERR_PTR(-ENOMEM);
> + }
> array->index_mask = index_mask;
> array->map.unpriv_array = unpriv;
>
> /* copy mandatory map attributes */
> bpf_map_init_from_attr(&array->map, attr);
> - array->map.memory.pages = cost;
> + bpf_map_charge_move(&array->map.memory, &mem);
> array->elem_size = elem_size;
>
> if (percpu && bpf_array_alloc_percpu(array)) {
> + bpf_map_charge_finish(&array->map.memory);
> bpf_map_area_free(array);
> return ERR_PTR(-ENOMEM);
> }
> diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
> index 035268add724..c633c8d68023 100644
> --- a/kernel/bpf/cpumap.c
> +++ b/kernel/bpf/cpumap.c
> @@ -108,10 +108,10 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
> cost += cpu_map_bitmap_size(attr) * num_possible_cpus();
> if (cost >= U32_MAX - PAGE_SIZE)
> goto free_cmap;
> - cmap->map.memory.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
>
> /* Notice returns -EPERM on if map size is larger than memlock limit */
> - ret = bpf_map_precharge_memlock(cmap->map.memory.pages);
> + ret = bpf_map_charge_init(&cmap->map.memory,
> + round_up(cost, PAGE_SIZE) >> PAGE_SHIFT);
> if (ret) {
> err = ret;
> goto free_cmap;
> @@ -121,7 +121,7 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
> cmap->flush_needed = __alloc_percpu(cpu_map_bitmap_size(attr),
> __alignof__(unsigned long));
> if (!cmap->flush_needed)
> - goto free_cmap;
> + goto free_charge;
>
> /* Alloc array for possible remote "destination" CPUs */
> cmap->cpu_map = bpf_map_area_alloc(cmap->map.max_entries *
> @@ -133,6 +133,8 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
> return &cmap->map;
> free_percpu:
> free_percpu(cmap->flush_needed);
> +free_charge:
> + bpf_map_charge_finish(&cmap->map.memory);
> free_cmap:
> kfree(cmap);
> return ERR_PTR(err);
> diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c
> index f6c57efb1d0d..371bd880ed58 100644
> --- a/kernel/bpf/devmap.c
> +++ b/kernel/bpf/devmap.c
> @@ -111,10 +111,9 @@ static struct bpf_map *dev_map_alloc(union bpf_attr *attr)
> if (cost >= U32_MAX - PAGE_SIZE)
> goto free_dtab;
>
> - dtab->map.memory.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
> -
> - /* if map size is larger than memlock limit, reject it early */
> - err = bpf_map_precharge_memlock(dtab->map.memory.pages);
> + /* if map size is larger than memlock limit, reject it */
> + err = bpf_map_charge_init(&dtab->map.memory,
> + round_up(cost, PAGE_SIZE) >> PAGE_SHIFT);
> if (err)
> goto free_dtab;
>
> @@ -125,19 +124,21 @@ static struct bpf_map *dev_map_alloc(union bpf_attr *attr)
> __alignof__(unsigned long),
> GFP_KERNEL | __GFP_NOWARN);
> if (!dtab->flush_needed)
> - goto free_dtab;
> + goto free_charge;
>
> dtab->netdev_map = bpf_map_area_alloc(dtab->map.max_entries *
> sizeof(struct bpf_dtab_netdev *),
> dtab->map.numa_node);
> if (!dtab->netdev_map)
> - goto free_dtab;
> + goto free_charge;
>
> spin_lock(&dev_map_lock);
> list_add_tail_rcu(&dtab->list, &dev_map_list);
> spin_unlock(&dev_map_lock);
>
> return &dtab->map;
> +free_charge:
> + bpf_map_charge_finish(&dtab->map.memory);
> free_dtab:
> free_percpu(dtab->flush_needed);
> kfree(dtab);
> diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
> index 15bf228d2e98..b0bdc7b040ad 100644
> --- a/kernel/bpf/hashtab.c
> +++ b/kernel/bpf/hashtab.c
> @@ -364,10 +364,9 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
> /* make sure page count doesn't overflow */
> goto free_htab;
>
> - htab->map.memory.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
> -
> - /* if map size is larger than memlock limit, reject it early */
> - err = bpf_map_precharge_memlock(htab->map.memory.pages);
> + /* if map size is larger than memlock limit, reject it */
> + err = bpf_map_charge_init(&htab->map.memory,
> + round_up(cost, PAGE_SIZE) >> PAGE_SHIFT);
> if (err)
> goto free_htab;
>
> @@ -376,7 +375,7 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
> sizeof(struct bucket),
> htab->map.numa_node);
> if (!htab->buckets)
> - goto free_htab;
> + goto free_charge;
>
> if (htab->map.map_flags & BPF_F_ZERO_SEED)
> htab->hashrnd = 0;
> @@ -409,6 +408,8 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
> prealloc_destroy(htab);
> free_buckets:
> bpf_map_area_free(htab->buckets);
> +free_charge:
> + bpf_map_charge_finish(&htab->map.memory);
> free_htab:
> kfree(htab);
> return ERR_PTR(err);
> diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c
> index 574325276650..e49bfd4f4f6d 100644
> --- a/kernel/bpf/local_storage.c
> +++ b/kernel/bpf/local_storage.c
> @@ -272,6 +272,7 @@ static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr)
> {
> int numa_node = bpf_map_attr_numa_node(attr);
> struct bpf_cgroup_storage_map *map;
> + struct bpf_map_memory mem;
> u32 pages;
> int ret;
>
> @@ -294,16 +295,18 @@ static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr)
>
> pages = round_up(sizeof(struct bpf_cgroup_storage_map), PAGE_SIZE) >>
> PAGE_SHIFT;
> - ret = bpf_map_precharge_memlock(pages);
> + ret = bpf_map_charge_init(&mem, pages);
> if (ret < 0)
> return ERR_PTR(ret);
>
> map = kmalloc_node(sizeof(struct bpf_cgroup_storage_map),
> __GFP_ZERO | GFP_USER, numa_node);
> - if (!map)
> + if (!map) {
> + bpf_map_charge_finish(&mem);
> return ERR_PTR(-ENOMEM);
> + }
>
> - map->map.memory.pages = pages;
> + bpf_map_charge_move(&map->map.memory, &mem);
>
> /* copy mandatory map attributes */
> bpf_map_init_from_attr(&map->map, attr);
> diff --git a/kernel/bpf/lpm_trie.c b/kernel/bpf/lpm_trie.c
> index 8e423a582760..6345a8d2dcd0 100644
> --- a/kernel/bpf/lpm_trie.c
> +++ b/kernel/bpf/lpm_trie.c
> @@ -578,9 +578,8 @@ static struct bpf_map *trie_alloc(union bpf_attr *attr)
> goto out_err;
> }
>
> - trie->map.memory.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
> -
> - ret = bpf_map_precharge_memlock(trie->map.memory.pages);
> + ret = bpf_map_charge_init(&trie->map.memory,
> + round_up(cost, PAGE_SIZE) >> PAGE_SHIFT);
> if (ret)
> goto out_err;
>
> diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c
> index 8a510e71d486..224cb0fd8f03 100644
> --- a/kernel/bpf/queue_stack_maps.c
> +++ b/kernel/bpf/queue_stack_maps.c
> @@ -67,6 +67,7 @@ static int queue_stack_map_alloc_check(union bpf_attr *attr)
> static struct bpf_map *queue_stack_map_alloc(union bpf_attr *attr)
> {
> int ret, numa_node = bpf_map_attr_numa_node(attr);
> + struct bpf_map_memory mem = {0};
> struct bpf_queue_stack *qs;
> u64 size, queue_size, cost;
>
> @@ -77,19 +78,21 @@ static struct bpf_map *queue_stack_map_alloc(union bpf_attr *attr)
>
> cost = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
>
> - ret = bpf_map_precharge_memlock(cost);
> + ret = bpf_map_charge_init(&mem, cost);
> if (ret < 0)
> return ERR_PTR(ret);
>
> qs = bpf_map_area_alloc(queue_size, numa_node);
> - if (!qs)
> + if (!qs) {
> + bpf_map_charge_finish(&mem);
> return ERR_PTR(-ENOMEM);
> + }
>
> memset(qs, 0, sizeof(*qs));
>
> bpf_map_init_from_attr(&qs->map, attr);
>
> - qs->map.memory.pages = cost;
> + bpf_map_charge_move(&qs->map.memory, &mem);
> qs->size = size;
>
> raw_spin_lock_init(&qs->lock);
> diff --git a/kernel/bpf/reuseport_array.c b/kernel/bpf/reuseport_array.c
> index 819515242739..5c6e25b1b9b1 100644
> --- a/kernel/bpf/reuseport_array.c
> +++ b/kernel/bpf/reuseport_array.c
> @@ -151,6 +151,7 @@ static struct bpf_map *reuseport_array_alloc(union bpf_attr *attr)
> {
> int err, numa_node = bpf_map_attr_numa_node(attr);
> struct reuseport_array *array;
> + struct bpf_map_memory mem;
> u64 cost, array_size;
>
> if (!capable(CAP_SYS_ADMIN))
> @@ -165,18 +166,20 @@ static struct bpf_map *reuseport_array_alloc(union bpf_attr *attr)
> return ERR_PTR(-ENOMEM);
> cost = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
>
> - err = bpf_map_precharge_memlock(cost);
> + err = bpf_map_charge_init(&mem, cost);
> if (err)
> return ERR_PTR(err);
>
> /* allocate all map elements and zero-initialize them */
> array = bpf_map_area_alloc(array_size, numa_node);
> - if (!array)
> + if (!array) {
> + bpf_map_charge_finish(&mem);
> return ERR_PTR(-ENOMEM);
> + }
>
> /* copy mandatory map attributes */
> bpf_map_init_from_attr(&array->map, attr);
> - array->map.memory.pages = cost;
> + bpf_map_charge_move(&array->map.memory, &mem);
>
> return &array->map;
> }
> diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c
> index 08d4efff73ac..8da24ca65d97 100644
> --- a/kernel/bpf/stackmap.c
> +++ b/kernel/bpf/stackmap.c
> @@ -89,6 +89,7 @@ static struct bpf_map *stack_map_alloc(union bpf_attr *attr)
> {
> u32 value_size = attr->value_size;
> struct bpf_stack_map *smap;
> + struct bpf_map_memory mem;
> u64 cost, n_buckets;
> int err;
>
> @@ -116,40 +117,43 @@ static struct bpf_map *stack_map_alloc(union bpf_attr *attr)
> n_buckets = roundup_pow_of_two(attr->max_entries);
>
> cost = n_buckets * sizeof(struct stack_map_bucket *) + sizeof(*smap);
> + if (cost >= U32_MAX - PAGE_SIZE)
> + return ERR_PTR(-E2BIG);
> + cost += n_buckets * (value_size + sizeof(struct stack_map_bucket));
> if (cost >= U32_MAX - PAGE_SIZE)
> return ERR_PTR(-E2BIG);
>
> + err = bpf_map_charge_init(&mem,
> + round_up(cost, PAGE_SIZE) >> PAGE_SHIFT);
> + if (err)
> + return ERR_PTR(err);
> +
> smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr));
> - if (!smap)
> + if (!smap) {
> + bpf_map_charge_finish(&mem);
> return ERR_PTR(-ENOMEM);
> -
> - err = -E2BIG;
> - cost += n_buckets * (value_size + sizeof(struct stack_map_bucket));
> - if (cost >= U32_MAX - PAGE_SIZE)
> - goto free_smap;
> + }
>
> bpf_map_init_from_attr(&smap->map, attr);
> smap->map.value_size = value_size;
> smap->n_buckets = n_buckets;
> - smap->map.memory.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
> -
> - err = bpf_map_precharge_memlock(smap->map.memory.pages);
> - if (err)
> - goto free_smap;
>
> err = get_callchain_buffers(sysctl_perf_event_max_stack);
> if (err)
> - goto free_smap;
> + goto free_charge;
>
> err = prealloc_elems_and_freelist(smap);
> if (err)
> goto put_buffers;
>
> + bpf_map_charge_move(&smap->map.memory, &mem);
> +
> return &smap->map;
>
> put_buffers:
> put_callchain_buffers();
> -free_smap:
> +free_charge:
> + bpf_map_charge_finish(&mem);
> bpf_map_area_free(smap);
> return ERR_PTR(err);
> }
> diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
> index df14e63806c8..351cc434c4ad 100644
> --- a/kernel/bpf/syscall.c
> +++ b/kernel/bpf/syscall.c
> @@ -188,19 +188,6 @@ void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr)
> map->numa_node = bpf_map_attr_numa_node(attr);
> }
>
> -int bpf_map_precharge_memlock(u32 pages)
> -{
> - struct user_struct *user = get_current_user();
> - unsigned long memlock_limit, cur;
> -
> - memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
> - cur = atomic_long_read(&user->locked_vm);
> - free_uid(user);
> - if (cur + pages > memlock_limit)
> - return -EPERM;
> - return 0;
> -}
> -
> static int bpf_charge_memlock(struct user_struct *user, u32 pages)
> {
> unsigned long memlock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
> @@ -214,29 +201,40 @@ static int bpf_charge_memlock(struct user_struct *user, u32 pages)
>
> static void bpf_uncharge_memlock(struct user_struct *user, u32 pages)
> {
> - atomic_long_sub(pages, &user->locked_vm);
> + if (user)
> + atomic_long_sub(pages, &user->locked_vm);
> }
>
> -static int bpf_map_init_memlock(struct bpf_map *map)
> +int bpf_map_charge_init(struct bpf_map_memory *mem, u32 pages)
> {
> struct user_struct *user = get_current_user();
> int ret;
>
> - ret = bpf_charge_memlock(user, map->memory.pages);
> + ret = bpf_charge_memlock(user, pages);
> if (ret) {
> free_uid(user);
> return ret;
> }
> - map->memory.user = user;
> - return ret;
> +
> + mem->pages = pages;
> + mem->user = user;
> +
> + return 0;
> }
>
> -static void bpf_map_release_memlock(struct bpf_map *map)
> +void bpf_map_charge_finish(struct bpf_map_memory *mem)
> {
> - struct user_struct *user = map->memory.user;
> + bpf_uncharge_memlock(mem->user, mem->pages);
> + free_uid(mem->user);
> +}
>
> - bpf_uncharge_memlock(user, map->memory.pages);
> - free_uid(user);
> +void bpf_map_charge_move(struct bpf_map_memory *dst,
> + struct bpf_map_memory *src)
> +{
> + *dst = *src;
> +
> + /* Make sure src will not be used for the redundant uncharging. */
> + memset(src, 0, sizeof(struct bpf_map_memory));
> }
>
> int bpf_map_charge_memlock(struct bpf_map *map, u32 pages)
> @@ -304,11 +302,13 @@ void bpf_map_free_id(struct bpf_map *map, bool do_idr_lock)
> static void bpf_map_free_deferred(struct work_struct *work)
> {
> struct bpf_map *map = container_of(work, struct bpf_map, work);
> + struct bpf_map_memory mem;
>
> - bpf_map_release_memlock(map);
> + bpf_map_charge_move(&mem, &map->memory);
> security_bpf_map_free(map);
> /* implementation dependent freeing */
> map->ops->map_free(map);
> + bpf_map_charge_finish(&mem);
> }
>
> static void bpf_map_put_uref(struct bpf_map *map)
> @@ -550,6 +550,7 @@ static int map_check_btf(struct bpf_map *map, const struct btf *btf,
> static int map_create(union bpf_attr *attr)
> {
> int numa_node = bpf_map_attr_numa_node(attr);
> + struct bpf_map_memory mem;
> struct bpf_map *map;
> int f_flags;
> int err;
> @@ -574,7 +575,7 @@ static int map_create(union bpf_attr *attr)
>
> err = bpf_obj_name_cpy(map->name, attr->map_name);
> if (err)
> - goto free_map_nouncharge;
> + goto free_map;
>
> atomic_set(&map->refcnt, 1);
> atomic_set(&map->usercnt, 1);
> @@ -584,20 +585,20 @@ static int map_create(union bpf_attr *attr)
>
> if (!attr->btf_value_type_id) {
> err = -EINVAL;
> - goto free_map_nouncharge;
> + goto free_map;
> }
>
> btf = btf_get_by_fd(attr->btf_fd);
> if (IS_ERR(btf)) {
> err = PTR_ERR(btf);
> - goto free_map_nouncharge;
> + goto free_map;
> }
>
> err = map_check_btf(map, btf, attr->btf_key_type_id,
> attr->btf_value_type_id);
> if (err) {
> btf_put(btf);
> - goto free_map_nouncharge;
> + goto free_map;
> }
>
> map->btf = btf;
> @@ -609,15 +610,11 @@ static int map_create(union bpf_attr *attr)
>
> err = security_bpf_map_alloc(map);
> if (err)
> - goto free_map_nouncharge;
> -
> - err = bpf_map_init_memlock(map);
> - if (err)
> - goto free_map_sec;
> + goto free_map;
>
> err = bpf_map_alloc_id(map);
> if (err)
> - goto free_map;
> + goto free_map_sec;
>
> err = bpf_map_new_fd(map, f_flags);
> if (err < 0) {
> @@ -633,13 +630,13 @@ static int map_create(union bpf_attr *attr)
>
> return err;
>
> -free_map:
> - bpf_map_release_memlock(map);
> free_map_sec:
> security_bpf_map_free(map);
> -free_map_nouncharge:
> +free_map:
> btf_put(map->btf);
> + bpf_map_charge_move(&mem, &map->memory);
> map->ops->map_free(map);
> + bpf_map_charge_finish(&mem);
> return err;
> }
>
> diff --git a/kernel/bpf/xskmap.c b/kernel/bpf/xskmap.c
> index f816ee1a0fa0..a329dab7c7a4 100644
> --- a/kernel/bpf/xskmap.c
> +++ b/kernel/bpf/xskmap.c
> @@ -40,10 +40,9 @@ static struct bpf_map *xsk_map_alloc(union bpf_attr *attr)
> if (cost >= U32_MAX - PAGE_SIZE)
> goto free_m;
>
> - m->map.memory.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
> -
> /* Notice returns -EPERM on if map size is larger than memlock limit */
> - err = bpf_map_precharge_memlock(m->map.memory.pages);
> + err = bpf_map_charge_init(&m->map.memory,
> + round_up(cost, PAGE_SIZE) >> PAGE_SHIFT);
> if (err)
> goto free_m;
>
> @@ -51,7 +50,7 @@ static struct bpf_map *xsk_map_alloc(union bpf_attr *attr)
>
> m->flush_list = alloc_percpu(struct list_head);
> if (!m->flush_list)
> - goto free_m;
> + goto free_charge;
>
> for_each_possible_cpu(cpu)
> INIT_LIST_HEAD(per_cpu_ptr(m->flush_list, cpu));
> @@ -65,6 +64,8 @@ static struct bpf_map *xsk_map_alloc(union bpf_attr *attr)
>
> free_percpu:
> free_percpu(m->flush_list);
> +free_charge:
> + bpf_map_charge_finish(&m->map.memory);
> free_m:
> kfree(m);
> return ERR_PTR(err);
> diff --git a/net/core/bpf_sk_storage.c b/net/core/bpf_sk_storage.c
> index 92581c3ff220..621a0b07ff11 100644
> --- a/net/core/bpf_sk_storage.c
> +++ b/net/core/bpf_sk_storage.c
> @@ -640,13 +640,16 @@ static struct bpf_map *bpf_sk_storage_map_alloc(union bpf_attr *attr)
> cost = sizeof(*smap->buckets) * nbuckets + sizeof(*smap);
> pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
>
> - ret = bpf_map_precharge_memlock(pages);
> - if (ret < 0)
> + ret = bpf_map_charge_init(&smap->map.memory, pages);
> + if (ret < 0) {
> + kfree(smap);
> return ERR_PTR(ret);
> + }
>
> smap->buckets = kvcalloc(sizeof(*smap->buckets), nbuckets,
> GFP_USER | __GFP_NOWARN);
> if (!smap->buckets) {
> + bpf_map_charge_finish(&smap->map.memory);
> kfree(smap);
> return ERR_PTR(-ENOMEM);
> }
> @@ -659,7 +662,6 @@ static struct bpf_map *bpf_sk_storage_map_alloc(union bpf_attr *attr)
> smap->elem_size = sizeof(struct bpf_sk_storage_elem) + attr->value_size;
> smap->cache_idx = (unsigned int)atomic_inc_return(&cache_idx) %
> BPF_SK_STORAGE_CACHE_SIZE;
> - smap->map.memory.pages = pages;
>
> return &smap->map;
> }
> diff --git a/net/core/sock_map.c b/net/core/sock_map.c
> index 4eb5b6a1b29f..1028c922a149 100644
> --- a/net/core/sock_map.c
> +++ b/net/core/sock_map.c
> @@ -49,8 +49,8 @@ static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
> goto free_stab;
> }
>
> - stab->map.memory.pages = round_up(cost, PAGE_SIZE) >> PAGE_SHIFT;
> - err = bpf_map_precharge_memlock(stab->map.memory.pages);
> + err = bpf_map_charge_init(&stab->map.memory,
> + round_up(cost, PAGE_SIZE) >> PAGE_SHIFT);
> if (err)
> goto free_stab;
>
> @@ -60,6 +60,7 @@ static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
> if (stab->sks)
> return &stab->map;
> err = -ENOMEM;
> + bpf_map_charge_finish(&stab->map.memory);
> free_stab:
> kfree(stab);
> return ERR_PTR(err);
> --
> 2.20.1
>
Powered by blists - more mailing lists