block: optionally merge discontiguous discard bios into a single request

Add a new merge strategy that merges discard bios into a request until the
maximum number of discard ranges (or the maximum discard size) is reached
from the plug merging code.  I/O scheduler merging is not wired up yet
but might also be useful, although not for fast devices like NVMe which
are the only user for now.

Note that for now we don't support limiting the size of each discard range,
but if needed that can be added later.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@fb.com>
This commit is contained in:
Christoph Hellwig 2017-02-08 14:46:49 +01:00 committed by Jens Axboe
parent 34fe7c0540
commit 1e739730c5
8 changed files with 95 additions and 1 deletions

View File

@ -1483,6 +1483,30 @@ bool bio_attempt_front_merge(struct request_queue *q, struct request *req,
return true; return true;
} }
bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
struct bio *bio)
{
unsigned short segments = blk_rq_nr_discard_segments(req);
if (segments >= queue_max_discard_segments(q))
goto no_merge;
if (blk_rq_sectors(req) + bio_sectors(bio) >
blk_rq_get_max_sectors(req, blk_rq_pos(req)))
goto no_merge;
req->biotail->bi_next = bio;
req->biotail = bio;
req->__data_len += bio->bi_iter.bi_size;
req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
req->nr_phys_segments = segments + 1;
blk_account_io_start(req, false);
return true;
no_merge:
req_set_nomerge(q, req);
return false;
}
/** /**
* blk_attempt_plug_merge - try to merge with %current's plugged list * blk_attempt_plug_merge - try to merge with %current's plugged list
* @q: request_queue new bio is being queued at * @q: request_queue new bio is being queued at
@ -1547,6 +1571,9 @@ bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
case ELEVATOR_FRONT_MERGE: case ELEVATOR_FRONT_MERGE:
merged = bio_attempt_front_merge(q, rq, bio); merged = bio_attempt_front_merge(q, rq, bio);
break; break;
case ELEVATOR_DISCARD_MERGE:
merged = bio_attempt_discard_merge(q, rq, bio);
break;
default: default:
break; break;
} }

View File

@ -803,7 +803,10 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio)
enum elv_merge blk_try_merge(struct request *rq, struct bio *bio) enum elv_merge blk_try_merge(struct request *rq, struct bio *bio)
{ {
if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector) if (req_op(rq) == REQ_OP_DISCARD &&
queue_max_discard_segments(rq->q) > 1)
return ELEVATOR_DISCARD_MERGE;
else if (blk_rq_pos(rq) + blk_rq_sectors(rq) == bio->bi_iter.bi_sector)
return ELEVATOR_BACK_MERGE; return ELEVATOR_BACK_MERGE;
else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector) else if (blk_rq_pos(rq) - bio_sectors(bio) == bio->bi_iter.bi_sector)
return ELEVATOR_FRONT_MERGE; return ELEVATOR_FRONT_MERGE;

View File

@ -780,6 +780,9 @@ static bool blk_mq_attempt_merge(struct request_queue *q,
if (blk_mq_sched_allow_merge(q, rq, bio)) if (blk_mq_sched_allow_merge(q, rq, bio))
merged = bio_attempt_front_merge(q, rq, bio); merged = bio_attempt_front_merge(q, rq, bio);
break; break;
case ELEVATOR_DISCARD_MERGE:
merged = bio_attempt_discard_merge(q, rq, bio);
break;
default: default:
continue; continue;
} }

View File

@ -88,6 +88,7 @@ EXPORT_SYMBOL_GPL(blk_queue_lld_busy);
void blk_set_default_limits(struct queue_limits *lim) void blk_set_default_limits(struct queue_limits *lim)
{ {
lim->max_segments = BLK_MAX_SEGMENTS; lim->max_segments = BLK_MAX_SEGMENTS;
lim->max_discard_segments = 1;
lim->max_integrity_segments = 0; lim->max_integrity_segments = 0;
lim->seg_boundary_mask = BLK_SEG_BOUNDARY_MASK; lim->seg_boundary_mask = BLK_SEG_BOUNDARY_MASK;
lim->virt_boundary_mask = 0; lim->virt_boundary_mask = 0;
@ -128,6 +129,7 @@ void blk_set_stacking_limits(struct queue_limits *lim)
/* Inherit limits from component devices */ /* Inherit limits from component devices */
lim->discard_zeroes_data = 1; lim->discard_zeroes_data = 1;
lim->max_segments = USHRT_MAX; lim->max_segments = USHRT_MAX;
lim->max_discard_segments = 1;
lim->max_hw_sectors = UINT_MAX; lim->max_hw_sectors = UINT_MAX;
lim->max_segment_size = UINT_MAX; lim->max_segment_size = UINT_MAX;
lim->max_sectors = UINT_MAX; lim->max_sectors = UINT_MAX;
@ -336,6 +338,22 @@ void blk_queue_max_segments(struct request_queue *q, unsigned short max_segments
} }
EXPORT_SYMBOL(blk_queue_max_segments); EXPORT_SYMBOL(blk_queue_max_segments);
/**
* blk_queue_max_discard_segments - set max segments for discard requests
* @q: the request queue for the device
* @max_segments: max number of segments
*
* Description:
* Enables a low level driver to set an upper limit on the number of
* segments in a discard request.
**/
void blk_queue_max_discard_segments(struct request_queue *q,
unsigned short max_segments)
{
q->limits.max_discard_segments = max_segments;
}
EXPORT_SYMBOL_GPL(blk_queue_max_discard_segments);
/** /**
* blk_queue_max_segment_size - set max segment size for blk_rq_map_sg * blk_queue_max_segment_size - set max segment size for blk_rq_map_sg
* @q: the request queue for the device * @q: the request queue for the device
@ -553,6 +571,8 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
b->virt_boundary_mask); b->virt_boundary_mask);
t->max_segments = min_not_zero(t->max_segments, b->max_segments); t->max_segments = min_not_zero(t->max_segments, b->max_segments);
t->max_discard_segments = min_not_zero(t->max_discard_segments,
b->max_discard_segments);
t->max_integrity_segments = min_not_zero(t->max_integrity_segments, t->max_integrity_segments = min_not_zero(t->max_integrity_segments,
b->max_integrity_segments); b->max_integrity_segments);

View File

@ -121,6 +121,12 @@ static ssize_t queue_max_segments_show(struct request_queue *q, char *page)
return queue_var_show(queue_max_segments(q), (page)); return queue_var_show(queue_max_segments(q), (page));
} }
static ssize_t queue_max_discard_segments_show(struct request_queue *q,
char *page)
{
return queue_var_show(queue_max_discard_segments(q), (page));
}
static ssize_t queue_max_integrity_segments_show(struct request_queue *q, char *page) static ssize_t queue_max_integrity_segments_show(struct request_queue *q, char *page)
{ {
return queue_var_show(q->limits.max_integrity_segments, (page)); return queue_var_show(q->limits.max_integrity_segments, (page));
@ -545,6 +551,11 @@ static struct queue_sysfs_entry queue_max_segments_entry = {
.show = queue_max_segments_show, .show = queue_max_segments_show,
}; };
static struct queue_sysfs_entry queue_max_discard_segments_entry = {
.attr = {.name = "max_discard_segments", .mode = S_IRUGO },
.show = queue_max_discard_segments_show,
};
static struct queue_sysfs_entry queue_max_integrity_segments_entry = { static struct queue_sysfs_entry queue_max_integrity_segments_entry = {
.attr = {.name = "max_integrity_segments", .mode = S_IRUGO }, .attr = {.name = "max_integrity_segments", .mode = S_IRUGO },
.show = queue_max_integrity_segments_show, .show = queue_max_integrity_segments_show,
@ -697,6 +708,7 @@ static struct attribute *default_attrs[] = {
&queue_max_hw_sectors_entry.attr, &queue_max_hw_sectors_entry.attr,
&queue_max_sectors_entry.attr, &queue_max_sectors_entry.attr,
&queue_max_segments_entry.attr, &queue_max_segments_entry.attr,
&queue_max_discard_segments_entry.attr,
&queue_max_integrity_segments_entry.attr, &queue_max_integrity_segments_entry.attr,
&queue_max_segment_size_entry.attr, &queue_max_segment_size_entry.attr,
&queue_iosched_entry.attr, &queue_iosched_entry.attr,

View File

@ -100,6 +100,8 @@ bool bio_attempt_front_merge(struct request_queue *q, struct request *req,
struct bio *bio); struct bio *bio);
bool bio_attempt_back_merge(struct request_queue *q, struct request *req, bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
struct bio *bio); struct bio *bio);
bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
struct bio *bio);
bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
unsigned int *request_count, unsigned int *request_count,
struct request **same_queue_rq); struct request **same_queue_rq);

View File

@ -331,6 +331,7 @@ struct queue_limits {
unsigned short logical_block_size; unsigned short logical_block_size;
unsigned short max_segments; unsigned short max_segments;
unsigned short max_integrity_segments; unsigned short max_integrity_segments;
unsigned short max_discard_segments;
unsigned char misaligned; unsigned char misaligned;
unsigned char discard_misaligned; unsigned char discard_misaligned;
@ -1146,6 +1147,8 @@ extern void blk_queue_bounce_limit(struct request_queue *, u64);
extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int); extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int); extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);
extern void blk_queue_max_segments(struct request_queue *, unsigned short); extern void blk_queue_max_segments(struct request_queue *, unsigned short);
extern void blk_queue_max_discard_segments(struct request_queue *,
unsigned short);
extern void blk_queue_max_segment_size(struct request_queue *, unsigned int); extern void blk_queue_max_segment_size(struct request_queue *, unsigned int);
extern void blk_queue_max_discard_sectors(struct request_queue *q, extern void blk_queue_max_discard_sectors(struct request_queue *q,
unsigned int max_discard_sectors); unsigned int max_discard_sectors);
@ -1189,6 +1192,15 @@ extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable); extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable);
extern void blk_queue_write_cache(struct request_queue *q, bool enabled, bool fua); extern void blk_queue_write_cache(struct request_queue *q, bool enabled, bool fua);
/*
* Number of physical segments as sent to the device.
*
* Normally this is the number of discontiguous data segments sent by the
* submitter. But for data-less command like discard we might have no
* actual data segments submitted, but the driver might have to add it's
* own special payload. In that case we still return 1 here so that this
* special payload will be mapped.
*/
static inline unsigned short blk_rq_nr_phys_segments(struct request *rq) static inline unsigned short blk_rq_nr_phys_segments(struct request *rq)
{ {
if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
@ -1196,6 +1208,15 @@ static inline unsigned short blk_rq_nr_phys_segments(struct request *rq)
return rq->nr_phys_segments; return rq->nr_phys_segments;
} }
/*
* Number of discard segments (or ranges) the driver needs to fill in.
* Each discard bio merged into a request is counted as one segment.
*/
static inline unsigned short blk_rq_nr_discard_segments(struct request *rq)
{
return max_t(unsigned short, rq->nr_phys_segments, 1);
}
extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *); extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
extern void blk_dump_rq_flags(struct request *, char *); extern void blk_dump_rq_flags(struct request *, char *);
extern long nr_blockdev_pages(void); extern long nr_blockdev_pages(void);
@ -1384,6 +1405,11 @@ static inline unsigned short queue_max_segments(struct request_queue *q)
return q->limits.max_segments; return q->limits.max_segments;
} }
static inline unsigned short queue_max_discard_segments(struct request_queue *q)
{
return q->limits.max_discard_segments;
}
static inline unsigned int queue_max_segment_size(struct request_queue *q) static inline unsigned int queue_max_segment_size(struct request_queue *q)
{ {
return q->limits.max_segment_size; return q->limits.max_segment_size;

View File

@ -16,6 +16,7 @@ enum elv_merge {
ELEVATOR_NO_MERGE = 0, ELEVATOR_NO_MERGE = 0,
ELEVATOR_FRONT_MERGE = 1, ELEVATOR_FRONT_MERGE = 1,
ELEVATOR_BACK_MERGE = 2, ELEVATOR_BACK_MERGE = 2,
ELEVATOR_DISCARD_MERGE = 3,
}; };
typedef enum elv_merge (elevator_merge_fn) (struct request_queue *, struct request **, typedef enum elv_merge (elevator_merge_fn) (struct request_queue *, struct request **,