block_session: server-defined payload alignment

This patch replaces the formerly fixed 2 KiB data alignment within the
packet-stream buffer by a server-defined alignment. This has two
benefits.

First, when using block servers that provide small block sizes like 512
bytes, we avoid fragmenting the packet-stream buffer, which occurs when
aligning 512-byte requests at 2 KiB boundaries. This reduces meta data
costs for the packet-stream allocator and also allows fitting more
requests into the buffer.

Second, block drivers with alignment constraints dictated by the
hardware can now pass those constraints to the client, thereby easing
the use of zero-copy DMA directly into the packet stream.

The alignment is determined by the Block::Session_client at construction
time and applied by the Block::Session_client::alloc_packet method.
Block-session clients should always use this method, not the 'alloc_packet'
method of the packet stream (tx source) directly. The latter merely
applies a default alignment of 2 KiB.

At the server side, the alignment is automatically checked by
block/component.h (old API) and block/request_stream.h (new API).

Issue #3274
This commit is contained in:
Norman Feske 2019-04-04 18:25:54 +02:00 committed by Christian Helmuth
parent 2208220c12
commit bbe3ee8dc5
35 changed files with 68 additions and 37 deletions

View File

@ -148,6 +148,7 @@ class Storage_device : public Genode::List<Storage_device>::Element,
{ {
return { .block_size = _block_size, return { .block_size = _block_size,
.block_count = _block_count, .block_count = _block_count,
.align_log2 = Genode::log2(_block_size),
.writeable = true }; .writeable = true };
} }

View File

@ -57,7 +57,7 @@ class Backend
Packet_descriptor::READ; Packet_descriptor::READ;
/* allocate packet */ /* allocate packet */
try { try {
Packet_descriptor packet( _session.dma_alloc_packet(length), Packet_descriptor packet( _session.alloc_packet(length),
opcode, offset / _info.block_size, opcode, offset / _info.block_size,
length / _info.block_size); length / _info.block_size);

View File

@ -46,6 +46,7 @@ class Driver : public Block::Driver
{ {
return { .block_size = _block_size, return { .block_size = _block_size,
.block_count = _http.file_size() / _block_size, .block_count = _http.file_size() / _block_size,
.align_log2 = log2(_block_size),
.writeable = false }; .writeable = false };
} }

View File

@ -62,6 +62,7 @@ extern "C" {
using Block::Connection::tx; using Block::Connection::tx;
using Block::Connection::sync; using Block::Connection::sync;
using Block::Connection::alloc_packet;
Drive(Platform &platform, char const *label) Drive(Platform &platform, char const *label)
: Block::Connection(platform.env, &platform.tx_alloc, 128*1024, label) : Block::Connection(platform.env, &platform.tx_alloc, 128*1024, label)
@ -125,7 +126,7 @@ extern "C" DRESULT disk_read (BYTE pdrv, BYTE* buff, DWORD sector, UINT count)
Genode::size_t const op_len = drive.info.block_size*count; Genode::size_t const op_len = drive.info.block_size*count;
/* allocate packet-descriptor for reading */ /* allocate packet-descriptor for reading */
Block::Packet_descriptor p(drive.tx()->alloc_packet(op_len), Block::Packet_descriptor p(drive.alloc_packet(op_len),
Block::Packet_descriptor::READ, sector, count); Block::Packet_descriptor::READ, sector, count);
drive.tx()->submit_packet(p); drive.tx()->submit_packet(p);
p = drive.tx()->get_acked_packet(); p = drive.tx()->get_acked_packet();
@ -155,7 +156,7 @@ extern "C" DRESULT disk_write (BYTE pdrv, const BYTE* buff, DWORD sector, UINT c
Genode::size_t const op_len = drive.info.block_size*count; Genode::size_t const op_len = drive.info.block_size*count;
/* allocate packet-descriptor for writing */ /* allocate packet-descriptor for writing */
Block::Packet_descriptor p(drive.tx()->alloc_packet(op_len), Block::Packet_descriptor p(drive.alloc_packet(op_len),
Block::Packet_descriptor::WRITE, sector, count); Block::Packet_descriptor::WRITE, sector, count);
Genode::memcpy(drive.tx()->packet_content(p), buff, op_len); Genode::memcpy(drive.tx()->packet_content(p), buff, op_len);

View File

@ -108,7 +108,8 @@ class Block::Session_component : public Block::Session_component_base,
/* ignore invalid packets */ /* ignore invalid packets */
bool const valid = packet.size() && _range_check(_p_to_handle) bool const valid = packet.size() && _range_check(_p_to_handle)
&& tx_sink()->packet_valid(packet); && tx_sink()->packet_valid(packet)
&& aligned(packet.offset(), _info.align_log2);
if (!valid) { if (!valid) {
_ack_packet(_p_to_handle); _ack_packet(_p_to_handle);
return; return;

View File

@ -26,6 +26,9 @@ class Block::Request_stream : Genode::Noncopyable
{ {
public: public:
struct Block_size { Genode::uint32_t value; };
struct Align_log2 { Genode::size_t value; };
/** /**
* Interface for accessing the content of a 'Request' * Interface for accessing the content of a 'Request'
* *
@ -59,7 +62,7 @@ class Block::Request_stream : Genode::Noncopyable
return request.count * _info.block_size; return request.count * _info.block_size;
} }
bool _valid_range(Block::Request const &request) const bool _valid_range_and_alignment(Block::Request const &request) const
{ {
/* local address of the last byte of the request */ /* local address of the last byte of the request */
Genode::addr_t const request_end = _base + request.offset Genode::addr_t const request_end = _base + request.offset
@ -73,6 +76,10 @@ class Block::Request_stream : Genode::Noncopyable
if (request_end > _base + _size - 1) if (request_end > _base + _size - 1)
return false; return false;
/* check for proper alignment */
if (!Genode::aligned(request.offset, _info.align_log2))
return false;
return true; return true;
} }
@ -94,7 +101,7 @@ class Block::Request_stream : Genode::Noncopyable
template <typename FN> template <typename FN>
void with_content(Block::Request request, FN const &fn) const void with_content(Block::Request request, FN const &fn) const
{ {
if (_valid_range(request)) if (_valid_range_and_alignment(request))
fn(_request_ptr(request), _request_size(request)); fn(_request_ptr(request), _request_size(request));
} }
}; };

View File

@ -31,7 +31,7 @@ namespace Block {
/** /**
* Represents an block-operation request * Representation of an block-operation request
* *
* The data associated with the 'Packet_descriptor' is either * The data associated with the 'Packet_descriptor' is either
* the data read from or written to the block indicated by * the data read from or written to the block indicated by
@ -42,6 +42,14 @@ class Block::Packet_descriptor : public Genode::Packet_descriptor
public: public:
enum Opcode { READ, WRITE, END }; enum Opcode { READ, WRITE, END };
/*
* Alignment used when allocating a packet directly via the 'tx'
* packet stream. This is not recommended because it does not
* apply the server's alignment constraints. Instead, the
* 'Block::Session_client::alloc_packet' should be used for
* allocating properly aligned block-request packets.
*/
enum Alignment { PACKET_ALIGNMENT = 11 }; enum Alignment { PACKET_ALIGNMENT = 11 };
private: private:
@ -82,7 +90,7 @@ class Block::Packet_descriptor : public Genode::Packet_descriptor
}; };
/* /**
* Block session interface * Block session interface
* *
* A block session corresponds to a block device that can be used to read * A block session corresponds to a block device that can be used to read
@ -109,6 +117,7 @@ struct Block::Session : public Genode::Session
{ {
Genode::size_t block_size; /* size of one block in bytes */ Genode::size_t block_size; /* size of one block in bytes */
sector_t block_count; /* number of blocks */ sector_t block_count; /* number of blocks */
Genode::size_t align_log2; /* packet alignment within payload buffer */
bool writeable; bool writeable;
}; };

View File

@ -27,6 +27,8 @@ class Block::Session_client : public Genode::Rpc_client<Session>
Packet_stream_tx::Client<Tx> _tx; Packet_stream_tx::Client<Tx> _tx;
Info const _info = info();
public: public:
/** /**
@ -59,12 +61,12 @@ class Block::Session_client : public Genode::Rpc_client<Session>
Genode::Capability<Tx> tx_cap() override { return call<Rpc_tx_cap>(); } Genode::Capability<Tx> tx_cap() override { return call<Rpc_tx_cap>(); }
/* /**
* Wrapper for alloc_packet, allocates 2KB aligned packets * Allocate packet respecting the server's alignment constraints
*/ */
Packet_descriptor dma_alloc_packet(Genode::size_t size) Packet_descriptor alloc_packet(Genode::size_t size)
{ {
return tx()->alloc_packet(size, 11); return tx()->alloc_packet(size, _info.align_log2);
} }
}; };

View File

@ -63,7 +63,7 @@ struct Test::Ping_pong : Test_base
while (_blocks < _length_in_blocks && _block->tx()->ready_to_submit()) { while (_blocks < _length_in_blocks && _block->tx()->ready_to_submit()) {
Block::Packet_descriptor tmp = Block::Packet_descriptor tmp =
_block->tx()->alloc_packet(_size_in_blocks * _info.block_size); _block->alloc_packet(_size_in_blocks * _info.block_size);
Block::sector_t const lba = _ping ? _start + _blocks Block::sector_t const lba = _ping ? _start + _blocks
: _end - _blocks; : _end - _blocks;

View File

@ -127,8 +127,7 @@ struct Test::Random : Test_base
bool next = true; bool next = true;
while (_blocks < _length_in_blocks && _block->tx()->ready_to_submit() && next) { while (_blocks < _length_in_blocks && _block->tx()->ready_to_submit() && next) {
Block::Packet_descriptor tmp = Block::Packet_descriptor tmp = _block->alloc_packet(_size);
_block->tx()->alloc_packet(_size);
Block::sector_t lba = _next_block(); Block::sector_t lba = _next_block();

View File

@ -69,7 +69,7 @@ struct Test::Replay : Test_base
more = false; more = false;
requests.dequeue([&] (Request &req) { requests.dequeue([&] (Request &req) {
Block::Packet_descriptor p( Block::Packet_descriptor p(
_block->tx()->alloc_packet(req.count * _info.block_size), _block->alloc_packet(req.count * _info.block_size),
req.op, req.nr, req.count); req.op, req.nr, req.count);
bool const write = req.op == Block::Packet_descriptor::WRITE; bool const write = req.op == Block::Packet_descriptor::WRITE;

View File

@ -66,8 +66,7 @@ struct Test::Sequential : Test_base
bool next = true; bool next = true;
while (_blocks < _length_in_blocks && _block->tx()->ready_to_submit() && next) { while (_blocks < _length_in_blocks && _block->tx()->ready_to_submit() && next) {
Block::Packet_descriptor tmp = Block::Packet_descriptor tmp = _block->alloc_packet(_size);
_block->tx()->alloc_packet(_size);
Block::Packet_descriptor p(tmp, Block::Packet_descriptor p(tmp,
_op, _start, _size_in_blocks); _op, _start, _size_in_blocks);

View File

@ -377,6 +377,7 @@ struct Ata_driver : Port_driver
{ {
return { .block_size = block_size(), return { .block_size = block_size(),
.block_count = block_count(), .block_count = block_count(),
.align_log2 = log2(block_size()),
.writeable = true }; .writeable = true };
} }

View File

@ -162,6 +162,7 @@ struct Atapi_driver : Port_driver
{ {
return { .block_size = block_size(), return { .block_size = block_size(),
.block_count = block_count(), .block_count = block_count(),
.align_log2 = 11,
.writeable = false }; .writeable = false };
} }

View File

@ -1541,6 +1541,7 @@ class Driver : public Block::Driver
_info = { .block_size = nsinfo.size, _info = { .block_size = nsinfo.size,
.block_count = nsinfo.count, .block_count = nsinfo.count,
.align_log2 = Genode::log2(nsinfo.size),
.writeable = true }; .writeable = true };
Nvme::Controller::Info const &info = _nvme_ctrlr->info(); Nvme::Controller::Info const &info = _nvme_ctrlr->info();

View File

@ -48,6 +48,7 @@ class Sd_card::Driver_base : public Block::Driver,
{ {
return { .block_size = _block_size(), return { .block_size = _block_size(),
.block_count = _block_count(), .block_count = _block_count(),
.align_log2 = Genode::log2(_block_size()),
.writeable = true }; .writeable = true };
} }
}; };

View File

@ -129,6 +129,7 @@ class Sd_card::Driver : public Block::Driver, private Attached_mmio
{ {
return { .block_size = _block_size, return { .block_size = _block_size,
.block_count = _block_count, .block_count = _block_count,
.align_log2 = log2(_block_size),
.writeable = true }; .writeable = true };
} }

View File

@ -808,6 +808,7 @@ struct Usb::Block_driver : Usb::Completion,
{ {
return { .block_size = _block_size, return { .block_size = _block_size,
.block_count = _block_count, .block_count = _block_count,
.align_log2 = Genode::log2(_block_size),
.writeable = _writeable }; .writeable = _writeable };
} }

View File

@ -106,7 +106,7 @@ class Vfs::Block_file_system : public Single_file_system
try { try {
Lock::Guard guard(_lock); Lock::Guard guard(_lock);
packet = _tx_source->alloc_packet(packet_size); packet = _block.alloc_packet(packet_size);
break; break;
} catch (Block::Session::Tx::Source::Packet_alloc_failed) { } catch (Block::Session::Tx::Source::Packet_alloc_failed) {
if (!_tx_source->ready_to_submit()) if (!_tx_source->ready_to_submit())

View File

@ -244,7 +244,7 @@ class Driver : public Block::Driver
/* construct and send the packet */ /* construct and send the packet */
p_to_dev = p_to_dev =
Block::Packet_descriptor(_blk.dma_alloc_packet(_info.block_size*cnt), Block::Packet_descriptor(_blk.alloc_packet(_info.block_size*cnt),
Block::Packet_descriptor::READ, Block::Packet_descriptor::READ,
nr, cnt); nr, cnt);
_r_list.insert(new (&_r_slab) Request(p_to_dev, packet, buffer)); _r_list.insert(new (&_r_slab) Request(p_to_dev, packet, buffer));

View File

@ -35,7 +35,7 @@ void Driver<POLICY>::Policy::sync(const typename POLICY::Element *e, char *dst)
throw Write_failed(off); throw Write_failed(off);
try { try {
Block::Packet_descriptor Block::Packet_descriptor
p(driver->blk()->dma_alloc_packet(Driver::CACHE_BLK_SIZE), p(driver->blk()->alloc_packet(Driver::CACHE_BLK_SIZE),
Block::Packet_descriptor::WRITE, off / driver->blk_sz(), Block::Packet_descriptor::WRITE, off / driver->blk_sz(),
Driver::CACHE_BLK_SIZE / driver->blk_sz()); Driver::CACHE_BLK_SIZE / driver->blk_sz());
driver->blk()->tx()->submit_packet(p); driver->blk()->tx()->submit_packet(p);

View File

@ -56,7 +56,7 @@ class Iso::Sector {
{ {
try { try {
_p = Block::Packet_descriptor( _p = Block::Packet_descriptor(
block.dma_alloc_packet(blk_size() * count), block.alloc_packet(blk_size() * count),
Block::Packet_descriptor::READ, Block::Packet_descriptor::READ,
blk_nr * ((float)blk_size() / BLOCK_SIZE), blk_nr * ((float)blk_size() / BLOCK_SIZE),
count * ((float)blk_size() / BLOCK_SIZE)); count * ((float)blk_size() / BLOCK_SIZE));

View File

@ -73,6 +73,7 @@ class Lx_block_driver : public Block::Driver
return { return {
.block_size = block_size, .block_size = block_size,
.block_count = st.st_size / block_size, .block_count = st.st_size / block_size,
.align_log2 = Genode::log2(block_size),
.writeable = xml_attr_ok(config, "writeable") .writeable = xml_attr_ok(config, "writeable")
}; };
} }

View File

@ -220,6 +220,7 @@ class Block::Session_component : public Block::Session_rpc_object,
{ {
return Info { .block_size = _driver.blk_size(), return Info { .block_size = _driver.blk_size(),
.block_count = _partition->sectors, .block_count = _partition->sectors,
.align_log2 = Genode::log2(_driver.blk_size()),
.writeable = _writeable && _driver.writeable() }; .writeable = _writeable && _driver.writeable() };
} }

View File

@ -139,7 +139,7 @@ class Block::Driver
? Block::Packet_descriptor::WRITE ? Block::Packet_descriptor::WRITE
: Block::Packet_descriptor::READ; : Block::Packet_descriptor::READ;
Genode::size_t const size = _info.block_size * cnt; Genode::size_t const size = _info.block_size * cnt;
Packet_descriptor p(_session.dma_alloc_packet(size), Packet_descriptor p(_session.alloc_packet(size),
op, nr, cnt); op, nr, cnt);
Request *r = new (&_r_slab) Request(dispatcher, cli, p); Request *r = new (&_r_slab) Request(dispatcher, cli, p);
_r_list.insert(r); _r_list.insert(r);

View File

@ -54,7 +54,7 @@ struct Block::Partition_table : Genode::Interface
unsigned long count, unsigned long count,
bool write = false) bool write = false)
: _session(driver.session()), : _session(driver.session()),
_p(_session.dma_alloc_packet(driver.blk_size() * count), _p(_session.alloc_packet(driver.blk_size() * count),
write ? Packet_descriptor::WRITE : Packet_descriptor::READ, write ? Packet_descriptor::WRITE : Packet_descriptor::READ,
blk_nr, count) blk_nr, count)
{ {

View File

@ -116,6 +116,7 @@ class Ram_block : public Block::Driver
{ {
return { .block_size = _block_size, return { .block_size = _block_size,
.block_count = _block_count, .block_count = _block_count,
.align_log2 = log2(_block_size),
.writeable = true }; .writeable = true };
} }

View File

@ -48,6 +48,7 @@ class Rom_block : public Block::Driver
{ {
return { .block_size = _blk_sz, return { .block_size = _blk_sz,
.block_count = _blk_cnt, .block_count = _blk_cnt,
.align_log2 = log2(_blk_sz),
.writeable = false }; .writeable = false };
} }

View File

@ -175,7 +175,7 @@ void Block_driver::_new_request(Vm_base &vm)
size_t const size = vm.smc_arg_3(); size_t const size = vm.smc_arg_3();
void *const req = (void*)vm.smc_arg_4(); void *const req = (void*)vm.smc_arg_4();
Packet_descriptor pkt = dev.session().tx()->alloc_packet(size); Packet_descriptor pkt = dev.session().alloc_packet(size);
void *addr = dev.session().tx()->packet_content(pkt); void *addr = dev.session().tx()->packet_content(pkt);
dev.cache().insert(addr, req); dev.cache().insert(addr, req);
vm.smc_ret((long)addr, pkt.offset()); vm.smc_ret((long)addr, pkt.offset());

View File

@ -67,7 +67,7 @@ class Throughput
try { try {
while (_session.tx()->ready_to_submit()) { while (_session.tx()->ready_to_submit()) {
Block::Packet_descriptor p( Block::Packet_descriptor p(
_session.tx()->alloc_packet(REQUEST_SIZE), _session.alloc_packet(REQUEST_SIZE),
!_read_done ? Block::Packet_descriptor::READ : Block::Packet_descriptor::WRITE, !_read_done ? Block::Packet_descriptor::READ : Block::Packet_descriptor::WRITE,
_current, count); _current, count);

View File

@ -140,7 +140,7 @@ struct Read_test : Test
try { try {
Block::Packet_descriptor p( Block::Packet_descriptor p(
_session.dma_alloc_packet(cnt*blk_sz), _session.alloc_packet(cnt*blk_sz),
Block::Packet_descriptor::READ, nr, cnt); Block::Packet_descriptor::READ, nr, cnt);
_session.tx()->submit_packet(p); _session.tx()->submit_packet(p);
} catch(Block::Session::Tx::Source::Packet_alloc_failed) { } catch(Block::Session::Tx::Source::Packet_alloc_failed) {
@ -241,8 +241,7 @@ struct Write_test : Test
{ {
while (!read_packets.empty()) { while (!read_packets.empty()) {
Block::Packet_descriptor r = read_packets.get(); Block::Packet_descriptor r = read_packets.get();
Block::Packet_descriptor w(_session.dma_alloc_packet(r.block_count() Block::Packet_descriptor w(_session.alloc_packet(r.block_count()*blk_sz),
*blk_sz),
Block::Packet_descriptor::WRITE, Block::Packet_descriptor::WRITE,
r.block_number(), r.block_count()); r.block_number(), r.block_count());
signed char *dst = (signed char*)_session.tx()->packet_content(w), signed char *dst = (signed char*)_session.tx()->packet_content(w),
@ -263,7 +262,7 @@ struct Write_test : Test
for (sector_t nr = start, cnt = Genode::min(NR_PER_REQ, end - start); nr < end; for (sector_t nr = start, cnt = Genode::min(NR_PER_REQ, end - start); nr < end;
nr += cnt, nr += cnt,
cnt = Genode::min<sector_t>(NR_PER_REQ, end-nr)) { cnt = Genode::min<sector_t>(NR_PER_REQ, end-nr)) {
Block::Packet_descriptor p(_session.dma_alloc_packet(cnt*blk_sz), Block::Packet_descriptor p(_session.alloc_packet(cnt*blk_sz),
Block::Packet_descriptor::READ, nr, cnt); Block::Packet_descriptor::READ, nr, cnt);
_session.tx()->submit_packet(p); _session.tx()->submit_packet(p);
} }
@ -338,7 +337,7 @@ struct Violation_test : Test
void req(Block::sector_t nr, Genode::size_t cnt, bool write) void req(Block::sector_t nr, Genode::size_t cnt, bool write)
{ {
Block::Packet_descriptor p(_session.dma_alloc_packet(blk_sz), Block::Packet_descriptor p(_session.alloc_packet(blk_sz),
write ? Block::Packet_descriptor::WRITE write ? Block::Packet_descriptor::WRITE
: Block::Packet_descriptor::READ, : Block::Packet_descriptor::READ,
nr, cnt); nr, cnt);

View File

@ -67,6 +67,7 @@ class Driver : public Block::Driver
{ {
return { .block_size = _size, return { .block_size = _size,
.block_count = _number, .block_count = _number,
.align_log2 = Genode::log2(_size),
.writeable = true }; .writeable = true };
} }

View File

@ -44,6 +44,7 @@ struct Test::Block_session_component : Rpc_object<Block::Session>,
Request_stream(rm, ds, ep, sigh, Request_stream(rm, ds, ep, sigh,
Info { .block_size = BLOCK_SIZE, Info { .block_size = BLOCK_SIZE,
.block_count = NUM_BLOCKS, .block_count = NUM_BLOCKS,
.align_log2 = log2(BLOCK_SIZE),
.writeable = true }), .writeable = true }),
_ep(ep) _ep(ep)
{ {

View File

@ -56,7 +56,7 @@ struct Main
size_t const cnt = (info.block_count - i > REQ_PARALLEL) size_t const cnt = (info.block_count - i > REQ_PARALLEL)
? REQ_PARALLEL : info.block_count - i; ? REQ_PARALLEL : info.block_count - i;
Packet_descriptor pkt(src.alloc_packet(cnt * info.block_size), Packet_descriptor pkt(block.alloc_packet(cnt * info.block_size),
Packet_descriptor::READ, i, cnt); Packet_descriptor::READ, i, cnt);
log("Check blocks ", i, "..", i + cnt - 1); log("Check blocks ", i, "..", i + cnt - 1);

View File

@ -260,7 +260,7 @@ bool Seoul::Disk::restart(struct disk_session const &disk,
Genode::Lock::Guard lock_guard(_alloc_lock); Genode::Lock::Guard lock_guard(_alloc_lock);
packet = Block::Packet_descriptor( packet = Block::Packet_descriptor(
source->alloc_packet(blocks * blk_size), disk.blk_con->alloc_packet(blocks * blk_size),
(write) ? Block::Packet_descriptor::WRITE (write) ? Block::Packet_descriptor::WRITE
: Block::Packet_descriptor::READ, : Block::Packet_descriptor::READ,
msg->sector, blocks); msg->sector, blocks);
@ -317,7 +317,7 @@ bool Seoul::Disk::execute(bool const write, struct disk_session const &disk,
Genode::Lock::Guard lock_guard(_alloc_lock); Genode::Lock::Guard lock_guard(_alloc_lock);
packet = Block::Packet_descriptor( packet = Block::Packet_descriptor(
source->alloc_packet(blocks * blk_size), disk.blk_con->alloc_packet(blocks * blk_size),
(write) ? Block::Packet_descriptor::WRITE (write) ? Block::Packet_descriptor::WRITE
: Block::Packet_descriptor::READ, : Block::Packet_descriptor::READ,
sector, blocks); sector, blocks);