Message-ID: <56B4B7A7.1050401@lightnvm.io>
Date:	Fri, 5 Feb 2016 15:54:31 +0100
From:	Matias Bjørling <mb@...htnvm.io>
To:	Javier González <jg@...htnvm.io>
Cc:	linux-kernel@...r.kernel.org, linux-block@...r.kernel.org,
	Javier González <javier@...xlabs.com>
Subject: Re: [RFC 3/4] lightnvm: read from rrpc write buffer if possible

On 02/04/2016 02:08 PM, Javier González wrote:
> Since writes are buffered in memory, incoming reads must retrieve
> buffered pages instead of submitting the I/O to the media.
> 
> This patch implements this logic. When a read bio arrives to rrpc, valid
> pages from the flash blocks residing in memory are copied. If there are
> any "holes" in the bio, a new bio is submitted to the media to retrieve
> the necessary pages. The original bio is updated accordingly.
> 
> Signed-off-by: Javier González <javier@...xlabs.com>
> ---
>  drivers/lightnvm/rrpc.c  | 451 ++++++++++++++++++++++++++++++++++++-----------
>  include/linux/lightnvm.h |   1 +
>  2 files changed, 346 insertions(+), 106 deletions(-)
> 
> diff --git a/drivers/lightnvm/rrpc.c b/drivers/lightnvm/rrpc.c
> index e9fb19d..6348d52 100644
> --- a/drivers/lightnvm/rrpc.c
> +++ b/drivers/lightnvm/rrpc.c
> @@ -827,10 +827,13 @@ static void rrpc_end_io(struct nvm_rq *rqd)
>  	struct rrpc *rrpc = container_of(rqd->ins, struct rrpc, instance);
>  	uint8_t nr_pages = rqd->nr_pages;
>  
> -	if (bio_data_dir(rqd->bio) == WRITE)
> +	if (bio_data_dir(rqd->bio) == WRITE) {
>  		rrpc_end_io_write(rrpc, rqd, nr_pages);
> -	else
> +	} else {
> +		if (rqd->flags & NVM_IOTYPE_SYNC)
> +			return;
>  		rrpc_end_io_read(rrpc, rqd, nr_pages);
> +	}
>  
>  	bio_put(rqd->bio);
>  
> @@ -842,83 +845,6 @@ static void rrpc_end_io(struct nvm_rq *rqd)
>  	mempool_free(rqd, rrpc->rq_pool);
>  }
>  
> -static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
> -			struct nvm_rq *rqd, struct rrpc_buf_rq *brrqd,
> -			unsigned long flags, int nr_pages)
> -{
> -	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
> -	struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rrqd);
> -	struct rrpc_addr *gp;
> -	sector_t laddr = rrpc_get_laddr(bio);
> -	int is_gc = flags & NVM_IOTYPE_GC;
> -	int i;
> -
> -	if (!is_gc && rrpc_lock_rq(rrpc, bio, rrqd)) {
> -		nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
> -		mempool_free(rrqd, rrpc->rrq_pool);
> -		mempool_free(rqd, rrpc->rq_pool);
> -		return NVM_IO_REQUEUE;
> -	}
> -
> -	for (i = 0; i < nr_pages; i++) {
> -		/* We assume that mapping occurs at 4KB granularity */
> -		BUG_ON(!(laddr + i >= 0 && laddr + i < rrpc->nr_sects));
> -		gp = &rrpc->trans_map[laddr + i];
> -
> -		if (gp->rblk) {
> -			rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev,
> -								gp->addr);
> -		} else {
> -			BUG_ON(is_gc);
> -			rrpc_unlock_laddr(rrpc, r);
> -			nvm_dev_dma_free(rrpc->dev, rqd->ppa_list,
> -							rqd->dma_ppa_list);
> -			mempool_free(rrqd, rrpc->rrq_pool);
> -			mempool_free(rqd, rrpc->rq_pool);
> -			return NVM_IO_DONE;
> -		}
> -
> -		brrqd[i].addr = gp;
> -	}
> -
> -	rqd->opcode = NVM_OP_HBREAD;
> -
> -	return NVM_IO_OK;
> -}
> -
> -static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd,
> -							unsigned long flags)
> -{
> -	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
> -	int is_gc = flags & NVM_IOTYPE_GC;
> -	sector_t laddr = rrpc_get_laddr(bio);
> -	struct rrpc_addr *gp;
> -
> -	if (!is_gc && rrpc_lock_rq(rrpc, bio, rrqd)) {
> -		mempool_free(rrqd, rrpc->rrq_pool);
> -		mempool_free(rqd, rrpc->rq_pool);
> -		return NVM_IO_REQUEUE;
> -	}
> -
> -	BUG_ON(!(laddr >= 0 && laddr < rrpc->nr_sects));
> -	gp = &rrpc->trans_map[laddr];
> -
> -	if (gp->rblk) {
> -		rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, gp->addr);
> -	} else {
> -		BUG_ON(is_gc);
> -		rrpc_unlock_rq(rrpc, rrqd);
> -		mempool_free(rrqd, rrpc->rrq_pool);
> -		mempool_free(rqd, rrpc->rq_pool);
> -		return NVM_IO_DONE;
> -	}
> -
> -	rqd->opcode = NVM_OP_HBREAD;
> -	rrqd->addr = gp;
> -
> -	return NVM_IO_OK;
> -}
> -
>  /*
>   * Copy data from current bio to block write buffer. This if necessary
>   * to guarantee durability if a flash block becomes bad before all pages
> @@ -1051,14 +977,335 @@ static int rrpc_write_rq(struct rrpc *rrpc, struct bio *bio,
>  	return NVM_IO_DONE;
>  }
>  
> +static int rrpc_buffer_write(struct rrpc *rrpc, struct bio *bio,
> +				struct rrpc_rq *rrqd, unsigned long flags)
> +{
> +	uint8_t nr_pages = rrpc_get_pages(bio);
> +
> +	rrqd->nr_pages = nr_pages;
> +
> +	if (nr_pages > 1)
> +		return rrpc_write_ppalist_rq(rrpc, bio, rrqd, flags, nr_pages);
> +	else
> +		return rrpc_write_rq(rrpc, bio, rrqd, flags);
> +}
> +
> +static int rrpc_read_ppalist_rq(struct rrpc *rrpc, struct bio *bio,
> +			struct nvm_rq *rqd, struct rrpc_buf_rq *brrqd,
> +			unsigned long flags, int nr_pages)
> +{
> +	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
> +	struct rrpc_inflight_rq *r = rrpc_get_inflight_rq(rrqd);
> +	struct rrpc_addr *gp;
> +	sector_t laddr = rrpc_get_laddr(bio);
> +	int is_gc = flags & NVM_IOTYPE_GC;
> +	int i;
> +
> +	if (!is_gc && rrpc_lock_rq(rrpc, bio, rrqd)) {
> +		nvm_dev_dma_free(rrpc->dev, rqd->ppa_list, rqd->dma_ppa_list);
> +		return NVM_IO_REQUEUE;
> +	}
> +
> +	for (i = 0; i < nr_pages; i++) {
> +		/* We assume that mapping occurs at 4KB granularity */
> +		BUG_ON(!(laddr + i >= 0 && laddr + i < rrpc->nr_sects));
> +		gp = &rrpc->trans_map[laddr + i];
> +
> +		if (gp->rblk) {
> +			rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev,
> +								gp->addr);
> +		} else {
> +			BUG_ON(is_gc);
> +			rrpc_unlock_laddr(rrpc, r);
> +			nvm_dev_dma_free(rrpc->dev, rqd->ppa_list,
> +							rqd->dma_ppa_list);
> +			return NVM_IO_DONE;
> +		}
> +
> +		brrqd[i].addr = gp;
> +	}
> +
> +	rqd->opcode = NVM_OP_HBREAD;
> +
> +	return NVM_IO_OK;
> +}
> +
> +static int rrpc_read_rq(struct rrpc *rrpc, struct bio *bio, struct nvm_rq *rqd,
> +							unsigned long flags)
> +{
> +	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
> +	int is_gc = flags & NVM_IOTYPE_GC;
> +	sector_t laddr = rrpc_get_laddr(bio);
> +	struct rrpc_addr *gp;
> +
> +	if (!is_gc && rrpc_lock_rq(rrpc, bio, rrqd))
> +		return NVM_IO_REQUEUE;
> +
> +	BUG_ON(!(laddr >= 0 && laddr < rrpc->nr_sects));
> +	gp = &rrpc->trans_map[laddr];
> +
> +	if (gp->rblk) {
> +		rqd->ppa_addr = rrpc_ppa_to_gaddr(rrpc->dev, gp->addr);
> +	} else {
> +		BUG_ON(is_gc);
> +		rrpc_unlock_rq(rrpc, rrqd);
> +		return NVM_IO_DONE;
> +	}
> +
> +	rqd->opcode = NVM_OP_HBREAD;
> +	rrqd->addr = gp;
> +
> +	return NVM_IO_OK;
> +}
> +
> +static int rrpc_read_w_buf_entry(struct bio *bio, struct rrpc_block *rblk,
> +					struct bvec_iter iter, int entry)
> +{
> +	struct buf_entry *read_entry;
> +	struct bio_vec bv;
> +	struct page *page;
> +	void *kaddr;
> +	void *data;
> +	int read = 0;
> +
> +	lockdep_assert_held(&rblk->w_buf.s_lock);
> +
> +	spin_lock(&rblk->w_buf.w_lock);
> +	if (entry >= rblk->w_buf.cur_mem) {
> +		spin_unlock(&rblk->w_buf.w_lock);
> +		goto out;
> +	}
> +	spin_unlock(&rblk->w_buf.w_lock);
> +
> +	read_entry = &rblk->w_buf.entries[entry];
> +	data = read_entry->data;
> +
> +	bv = bio_iter_iovec(bio, iter);
> +	page = bv.bv_page;
> +	kaddr = kmap_atomic(page);
> +	memcpy(kaddr + bv.bv_offset, data, RRPC_EXPOSED_PAGE_SIZE);
> +	kunmap_atomic(kaddr);
> +	read++;
> +
> +out:
> +	return read;
> +}
> +
> +static int rrpc_read_from_w_buf(struct rrpc *rrpc, struct nvm_rq *rqd,
> +			struct rrpc_buf_rq *brrqd, unsigned long *read_bitmap)
> +{
> +	struct nvm_dev *dev = rrpc->dev;
> +	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
> +	struct rrpc_addr *addr;
> +	struct bio *bio = rqd->bio;
> +	struct bvec_iter iter = bio->bi_iter;
> +	struct rrpc_block *rblk;
> +	unsigned long blk_id;
> +	int nr_pages = rqd->nr_pages;
> +	int left = nr_pages;
> +	int read = 0;
> +	int entry;
> +	int i;
> +
> +	if (nr_pages != bio->bi_vcnt)
> +		goto out;
> +
> +	if (nr_pages == 1) {
> +		rblk = rrqd->addr->rblk;
> +
> +		/* If the write buffer exists, the block is open in memory */
> +		spin_lock(&rblk->w_buf.s_lock);
> +		atomic_inc(&rblk->w_buf.refs);
> +		if (rblk->w_buf.entries) {
> +			blk_id = rblk->parent->id;
> +			entry = rrqd->addr->addr -
> +				(blk_id * dev->sec_per_pg * dev->pgs_per_blk);
> +
> +			read = rrpc_read_w_buf_entry(bio, rblk, iter, entry);
> +
> +			left -= read;
> +			WARN_ON(test_and_set_bit(0, read_bitmap));
> +		}
> +		bio_advance_iter(bio, &iter, RRPC_EXPOSED_PAGE_SIZE);
> +
> +		atomic_dec(&rblk->w_buf.refs);
> +		spin_unlock(&rblk->w_buf.s_lock);
> +
> +		goto out;
> +	}
> +
> +	/* Iterate through all pages and copy those that are found in the write
> +	 * buffer. We will complete the holes (if any) with a intermediate bio
> +	 * later on
> +	 */
> +	for (i = 0; i < nr_pages; i++) {
> +		addr = brrqd[i].addr;
> +		rblk = addr->rblk;
> +
> +		/* If the write buffer exists, the block is open in memory */
> +		spin_lock(&rblk->w_buf.s_lock);
> +		atomic_inc(&rblk->w_buf.refs);
> +		if (rblk->w_buf.entries) {
> +			blk_id = rblk->parent->id;
> +			entry = addr->addr - (blk_id * dev->sec_per_pg *
> +							dev->pgs_per_blk);
> +
> +			read = rrpc_read_w_buf_entry(bio, rblk, iter, entry);
> +
> +			left -= read;
> +			WARN_ON(test_and_set_bit(i, read_bitmap));
> +		}
> +		bio_advance_iter(bio, &iter, RRPC_EXPOSED_PAGE_SIZE);
> +
> +		atomic_dec(&rblk->w_buf.refs);
> +		spin_unlock(&rblk->w_buf.s_lock);
> +	}
> +
> +out:
> +	return left;
> +}
> +
> +static int rrpc_submit_read_io(struct rrpc *rrpc, struct bio *bio,
> +				struct nvm_rq *rqd, unsigned long flags)
> +{
> +	struct rrpc_rq *rrqd = nvm_rq_to_pdu(rqd);
> +	int err;
> +
> +	err = nvm_submit_io(rrpc->dev, rqd);
> +	if (err) {
> +		pr_err("rrpc: I/O submission failed: %d\n", err);
> +		bio_put(bio);
> +		if (!(flags & NVM_IOTYPE_GC)) {
> +			rrpc_unlock_rq(rrpc, rrqd);
> +			if (rqd->nr_pages > 1)
> +				nvm_dev_dma_free(rrpc->dev,
> +			rqd->ppa_list, rqd->dma_ppa_list);
> +		}
> +		return NVM_IO_ERR;
> +	}
> +
> +	return NVM_IO_OK;
> +}
> +
> +static int rrpc_fill_partial_read_bio(struct rrpc *rrpc, struct bio *bio,
> +				unsigned long *read_bitmap, struct nvm_rq *rqd,
> +				struct rrpc_buf_rq *brrqd, uint8_t nr_pages)
> +{
> +	struct bio *new_bio;
> +	struct page *page;
> +	struct bio_vec src_bv, dst_bv;
> +	void *src_p, *dst_p;
> +	int nr_holes = nr_pages - bitmap_weight(read_bitmap, nr_pages);
> +	int hole;
> +	int i = 0;
> +	int ret;
> +	DECLARE_COMPLETION_ONSTACK(wait);
> +
> +	new_bio = bio_alloc(GFP_KERNEL, nr_holes);
> +	if (!new_bio) {
> +		pr_err("nvm: rrpc: could not alloc read bio\n");
> +		return NVM_IO_ERR;
> +	}
> +
> +	hole = find_first_zero_bit(read_bitmap, nr_pages);
> +	do {
> +		page = mempool_alloc(rrpc->page_pool, GFP_KERNEL);
> +		if (!page) {
> +			bio_put(new_bio);
> +			pr_err("nvm: rrpc: could not alloc read page\n");
> +			goto err;
> +		}
> +
> +		ret = bio_add_page(new_bio, page, RRPC_EXPOSED_PAGE_SIZE, 0);
> +		if (ret != RRPC_EXPOSED_PAGE_SIZE) {
> +			pr_err("nvm: rrpc: could not add page to bio\n");
> +			mempool_free(page, rrpc->page_pool);
> +			goto err;
> +		}
> +
> +		rqd->ppa_list[i] = rrpc_ppa_to_gaddr(rrpc->dev,
> +							brrqd[hole].addr->addr);
> +
> +		i++;
> +		hole = find_next_zero_bit(read_bitmap, nr_pages, hole + 1);
> +	} while (hole != nr_pages);
> +
> +	if (nr_holes != new_bio->bi_vcnt) {
> +		pr_err("rrpc: malformed bio\n");
> +		goto err;
> +	}
> +
> +	new_bio->bi_iter.bi_sector = bio->bi_iter.bi_sector;
> +	new_bio->bi_rw = READ;
> +	new_bio->bi_private = &wait;
> +	new_bio->bi_end_io = rrpc_end_sync_bio;
> +
> +	rqd->flags |= NVM_IOTYPE_SYNC;
> +	rqd->bio = new_bio;
> +	rqd->nr_pages = nr_holes;
> +
> +	rrpc_submit_read_io(rrpc, new_bio, rqd, rqd->flags);
> +	wait_for_completion_io(&wait);
> +
> +	if (new_bio->bi_error)
> +		goto err;
> +
> +	/* Fill the holes in the original bio */
> +	i = 0;
> +	hole = find_first_zero_bit(read_bitmap, nr_pages);
> +	do {
> +		src_bv = new_bio->bi_io_vec[i];
> +		dst_bv = bio->bi_io_vec[hole];
> +
> +		src_p = kmap_atomic(src_bv.bv_page);
> +		dst_p = kmap_atomic(dst_bv.bv_page);
> +
> +		memcpy(dst_p + dst_bv.bv_offset,
> +			src_p + src_bv.bv_offset,
> +			RRPC_EXPOSED_PAGE_SIZE);
> +
> +		kunmap_atomic(src_p);
> +		kunmap_atomic(dst_p);
> +
> +		mempool_free(&src_bv.bv_page, rrpc->page_pool);
> +
> +		i++;
> +		hole = find_next_zero_bit(read_bitmap, nr_pages, hole + 1);
> +	} while (hole != nr_pages);
> +
> +	bio_put(new_bio);
> +
> +	/* Complete the original bio and associated request */
> +	rqd->flags &= ~NVM_IOTYPE_SYNC;
> +	rqd->bio = bio;
> +	rqd->nr_pages = nr_pages;
> +
> +	bio_endio(bio);
> +	rrpc_end_io(rqd);
> +	return NVM_IO_OK;
> +
> +err:
> +	/* Free allocated pages in new bio */
> +	for (i = 0; i < new_bio->bi_vcnt; i++) {
> +		src_bv = new_bio->bi_io_vec[i];
> +		mempool_free(&src_bv.bv_page, rrpc->page_pool);
> +	}
> +	bio_endio(new_bio);
> +	return NVM_IO_ERR;
> +}
> +
>  static int rrpc_submit_read(struct rrpc *rrpc, struct bio *bio,
>  				struct rrpc_rq *rrqd, unsigned long flags)
>  {
>  	struct nvm_rq *rqd;
>  	struct rrpc_buf_rq brrqd[rrpc->max_write_pgs];
> +	unsigned long read_bitmap; /* Max 64 ppas per request */
> +	uint8_t left;
>  	uint8_t nr_pages = rrpc_get_pages(bio);
>  	int err;
>  
> +	bitmap_zero(&read_bitmap, nr_pages);
> +
>  	rqd = mempool_alloc(rrpc->rq_pool, GFP_KERNEL);
>  	if (!rqd) {
>  		pr_err_ratelimited("rrpc: not able to queue bio.");
> @@ -1073,22 +1320,25 @@ static int rrpc_submit_read(struct rrpc *rrpc, struct bio *bio,
>  						&rqd->dma_ppa_list);
>  		if (!rqd->ppa_list) {
>  			pr_err("rrpc: not able to allocate ppa list\n");
> -			mempool_free(rrqd, rrpc->rrq_pool);
>  			mempool_free(rqd, rrpc->rq_pool);
> +			mempool_free(rrqd, rrpc->rrq_pool);
>  			return NVM_IO_ERR;
>  		}
>  
>  		err = rrpc_read_ppalist_rq(rrpc, bio, rqd, brrqd, flags,
>  								nr_pages);
>  		if (err) {
> -			mempool_free(rrqd, rrpc->rrq_pool);
>  			mempool_free(rqd, rrpc->rq_pool);
> +			mempool_free(rrqd, rrpc->rrq_pool);
>  			return err;
>  		}
>  	} else {
>  		err = rrpc_read_rq(rrpc, bio, rqd, flags);
> -		if (err)
> +		if (err) {
> +			mempool_free(rrqd, rrpc->rrq_pool);
> +			mempool_free(rqd, rrpc->rq_pool);
>  			return err;
> +		}
>  	}
>  
>  	bio_get(bio);
> @@ -1097,33 +1347,22 @@ static int rrpc_submit_read(struct rrpc *rrpc, struct bio *bio,
>  	rqd->nr_pages = rrqd->nr_pages = nr_pages;
>  	rqd->flags = flags;
>  
> -	err = nvm_submit_io(rrpc->dev, rqd);
> -	if (err) {
> -		pr_err("rrpc: I/O submission failed: %d\n", err);
> -		bio_put(bio);
> -		if (!(flags & NVM_IOTYPE_GC)) {
> -			rrpc_unlock_rq(rrpc, rrqd);
> -			if (rqd->nr_pages > 1)
> -				nvm_dev_dma_free(rrpc->dev,
> -			rqd->ppa_list, rqd->dma_ppa_list);
> -		}
> +	left = rrpc_read_from_w_buf(rrpc, rqd, brrqd, &read_bitmap);
> +	if (left == 0) {
> +		bio_endio(bio);
> +		rrpc_end_io(rqd);
> +		return NVM_IO_OK;
> +	} else if (left < 0)
>  		return NVM_IO_ERR;
> -	}
>  
> -	return NVM_IO_OK;
> -}
> +	if (bitmap_empty(&read_bitmap, nr_pages))
> +		return rrpc_submit_read_io(rrpc, bio, rqd, flags);
>  
> -static int rrpc_buffer_write(struct rrpc *rrpc, struct bio *bio,
> -				struct rrpc_rq *rrqd, unsigned long flags)
> -{
> -	uint8_t nr_pages = rrpc_get_pages(bio);
> -
> -	rrqd->nr_pages = nr_pages;
> -
> -	if (nr_pages > 1)
> -		return rrpc_write_ppalist_rq(rrpc, bio, rrqd, flags, nr_pages);
> -	else
> -		return rrpc_write_rq(rrpc, bio, rrqd, flags);
> +	/* The read bio could not be completely read from the write buffer. This
> +	 * case only occurs when several pages are sent in a single bio
> +	 */
> +	return rrpc_fill_partial_read_bio(rrpc, bio, &read_bitmap, rqd, brrqd,
> +								nr_pages);
>  }
>  
>  static int rrpc_submit_io(struct rrpc *rrpc, struct bio *bio,
> diff --git a/include/linux/lightnvm.h b/include/linux/lightnvm.h
> index eda9743..ae26ced 100644
> --- a/include/linux/lightnvm.h
> +++ b/include/linux/lightnvm.h
> @@ -11,6 +11,7 @@ enum {
>  
>  	NVM_IOTYPE_NONE = 0,
>  	NVM_IOTYPE_GC = 1,
> +	NVM_IOTYPE_SYNC = 2,
>  };
>  
>  #define NVM_BLK_BITS (16)
> 

Seems like this can be merged into the write buffer patch as well?

Powered by blists - more mailing lists