Skip to content

Commit

Permalink
iotune: keep track of actual file_size, and bail on ENOSPC
Browse files Browse the repository at this point in the history 
We may not be able to write the whole file, and in cases like that, we would like to
dump a warning and move forward. So let's make file_size an instance member instead
of a static property of the iotune_manager. We can also use that to cut short our execution
in case of ENOSPC instead of using an early check.

Signed-off-by: Glauber Costa <[email protected]>
  • Loading branch information
Glauber Costa committed Mar 21, 2016
1 parent 69d37f5 commit e2a691f
Showing 1 changed file with 45 additions and 28 deletions.
73 changes: 45 additions & 28 deletions apps/iotune/iotune.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
using namespace std::chrono_literals;

bool filesystem_has_good_aio_support(sstring directory, bool verbose);
class iotune_manager;

struct directory {
sstring name;
Expand All @@ -58,7 +59,7 @@ struct test_file {
file_desc file;

test_file(const directory& dir);
void generate();
void generate(iotune_manager& iotune_manager);
};

struct run_stats {
Expand All @@ -78,7 +79,7 @@ class iotune_manager {
public:
enum class test_done { no, yes };
using clock = std::chrono::steady_clock;
static constexpr uint64_t file_size = 10ull << 30;
uint64_t file_size = 10ull << 30;
static constexpr uint64_t wbuffer_size = 128ul << 10;
static constexpr uint64_t rbuffer_size = 4ul << 10;
private:
Expand Down Expand Up @@ -251,7 +252,7 @@ class iotune_manager {
, _time_run_atomic(n)
, _test_file(directory(dirname))
, _run_start_time(iotune_manager::clock::now()) {
_test_file.generate();
_test_file.generate(*this);
// Initial exploratory run
for (auto initial: boost::irange<unsigned, unsigned>(4, 512, 4)) {
_concurrency_queue.push(initial);
Expand Down Expand Up @@ -335,24 +336,13 @@ class iotune_manager {
}
};

constexpr uint64_t iotune_manager::file_size;
constexpr uint64_t iotune_manager::wbuffer_size;
constexpr uint64_t iotune_manager::rbuffer_size;

test_file::test_file(const directory& dir)
: name(dir.name + "/ioqueue-discovery")
, file(file_desc::open(name.c_str(), O_DIRECT | O_CLOEXEC | O_RDWR | O_CREAT, S_IRWXU)) {

auto gb = [] (auto b) {
return float(b) / (1ull << 30);
};

auto si = boost::filesystem::space(boost::filesystem::path(dir.name));
::unlink(name.c_str());

if (si.available < iotune_manager::file_size) {
throw std::runtime_error(sprint("iotune requires at least %.2f GB available. Filesystem contains only %.2f GB available\n", gb(iotune_manager::file_size), gb(si.available)));
}
unlink(name.c_str());
}

static thread_local std::default_random_engine random_generator(std::chrono::duration_cast<std::chrono::nanoseconds>(iotune_manager::clock::now().time_since_epoch()).count());
Expand All @@ -367,9 +357,9 @@ class reader {
iotune_manager::clock::time_point _end_time;
std::unique_ptr<char[], free_deleter> _buf;
public:
reader(file_desc f, iotune_manager::clock::time_point start_time, iotune_manager::clock::time_point end_time)
reader(file_desc f, uint64_t file_size, iotune_manager::clock::time_point start_time, iotune_manager::clock::time_point end_time)
: _file(std::move(f))
, _pos_distribution(0, (iotune_manager::file_size/ iotune_manager::rbuffer_size) - 1)
, _pos_distribution(0, (file_size/ iotune_manager::rbuffer_size) - 1)
, _start_time(start_time)
, _tstamp(iotune_manager::clock::now())
, _end_time(end_time)
Expand Down Expand Up @@ -402,8 +392,14 @@ class reader {
}
};

void sanity_check_ev(const io_event& ev) {
if (long(ev.res) < 0) {
throw_kernel_error(long(ev.res));
}
}

void sanity_check_ev(const io_event& ev, size_t size) {
throw_kernel_error(long(ev.res));
sanity_check_ev(ev);
if (size_t(ev.res) != size) {
throw std::runtime_error(sprint("Expected %ld bytes I/O, found %ld\n", size, size_t(ev.res)));
}
Expand All @@ -427,7 +423,7 @@ run_stats iotune_manager::issue_reads(size_t cpu_id, unsigned concurrency) {

auto fds = std::vector<reader>();
for (unsigned i = 0u; i < concurrency; ++i) {
fds.emplace_back(_test_file.file.dup(), start_time, start_time + total_time);
fds.emplace_back(_test_file.file.dup(), file_size, start_time, start_time + total_time);
}

for (auto& r: fds) {
Expand Down Expand Up @@ -462,8 +458,12 @@ run_stats iotune_manager::issue_reads(size_t cpu_id, unsigned concurrency) {
return result;
}

void test_file::generate() {
std::cout << "Generating evaluation file..." << std::flush;
void test_file::generate(iotune_manager& iotune_manager) {
auto to_gb = [] (auto b) {
return float(b) / (1ull << 30);
};

std::cout << "Generating evaluation file sized " << to_gb(iotune_manager.file_size) << "GB..." << std::flush;
io_context_t io_context = {0};
auto max_aio = 128;
auto r = ::io_setup(max_aio, &io_context);
Expand All @@ -472,7 +472,7 @@ void test_file::generate() {

auto buf = allocate_aligned_buffer<char>(iotune_manager::wbuffer_size, 4096);
memset(buf.get(), 0, iotune_manager::wbuffer_size);
auto ft = ftruncate(file.get(), iotune_manager::file_size);
auto ft = ftruncate(file.get(), iotune_manager.file_size);
throw_kernel_error(ft);

std::vector<iocb*> iocb_vecptr;
Expand All @@ -484,12 +484,14 @@ void test_file::generate() {
iocb_vecptr.resize(max_aio);
std::iota(iocb_vecptr.begin(), iocb_vecptr.end(), iocbs.data());
uint64_t pos = 0;
uint64_t bytes_written = 0;
unsigned aio_outstanding = 0;
bool stopped_on_error = false;

while (pos < iotune_manager::file_size || aio_outstanding) {
while ((pos < iotune_manager.file_size && !stopped_on_error) || aio_outstanding) {
unsigned i = 0;
while (i < max_aio - aio_outstanding && pos < iotune_manager::file_size) {
auto now = std::min(iotune_manager::file_size - pos, iotune_manager::wbuffer_size);
while (i < max_aio - aio_outstanding && pos < iotune_manager.file_size) {
auto now = std::min(iotune_manager.file_size - pos, iotune_manager::wbuffer_size);
auto& iocb = iocbs[i++];
iocb.data = buf.get();
io_prep_pwrite(&iocb, file.get(), buf.get(), now, pos);
Expand All @@ -504,13 +506,28 @@ void test_file::generate() {
struct timespec timeout = {0, 0};
int n = ::io_getevents(io_context, 1, ev.size(), ev.data(), &timeout);
throw_kernel_error(n);
aio_outstanding -= n;
for (auto i = 0ul; i < size_t(n); ++i) {
sanity_check_ev(ev[i], iotune_manager::wbuffer_size);
if (stopped_on_error) {
// We have given up already, just loop through so we will
// flush all outstanding I/O.
break;
} else if ((long(ev[i].res) == -ENOSPC) || (ev[i].res < iotune_manager::wbuffer_size)) {
// FIXME: The buffer size can be cut short due to other conditions that are unrelated
// to ENOSPC. We should be testing it separately.
std::cout << " stopped early due to disk space issues. Will continue but accuracy may suffer." << std::endl;
iotune_manager.file_size = bytes_written;
stopped_on_error = true;
break;
} else {
sanity_check_ev(ev[i], iotune_manager::wbuffer_size);
bytes_written += iotune_manager::wbuffer_size;
}
}
aio_outstanding -= n;
}
}
std::cout << " done." << std::endl;
iotune_manager.file_size = bytes_written;
std::cout << to_gb(iotune_manager.file_size) << "GB written" << std::endl;
}

uint32_t io_queue_discovery(sstring dir, std::vector<unsigned> cpus) {
Expand Down

0 comments on commit e2a691f

Please sign in to comment.