Skip to content

Commit

Permalink
[breakchange][feat] async_upload support offset
Browse files Browse the repository at this point in the history
  • Loading branch information
@poor-circle
poor-circle committed Aug 8, 2024
1 parent 03a5cf8 commit 31c6191
Show file tree
Hide file tree
Showing 2 changed files with 111 additions and 19 deletions.
32 changes: 24 additions & 8 deletions include/cinatra/coro_http_client.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
: coro_http_client(executor->get_asio_executor()) {}

bool init_config(const config &conf) {
config_ = conf;
if (conf.conn_timeout_duration.has_value()) {
set_conn_timeout(*conf.conn_timeout_duration);
}
Expand Down Expand Up @@ -207,6 +208,8 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {

coro_io::ExecutorWrapper<> &get_executor() { return executor_wrapper_; }

const config &get_config() { return config_; }

#ifdef CINATRA_ENABLE_SSL
bool init_ssl(int verify_mode, const std::string &base_path,
const std::string &cert_file, const std::string &sni_hostname) {
Expand Down Expand Up @@ -875,14 +878,16 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
}

async_simple::coro::Lazy<void> send_file_no_chunked_with_copy(
std::string_view source, std::error_code &ec, std::size_t length) {
std::string_view source, std::error_code &ec, std::size_t length,
std::size_t offset) {
if (length <= 0) {
co_return;
}
std::string file_data;
detail::resize(file_data, std::min(max_single_part_size_, length));
coro_io::coro_file file{};
file.open(source, std::ios::in);
file.seek(offset, std::ios::cur);
if (!file.is_open()) {
ec = std::make_error_code(std::errc::bad_file_descriptor);
co_return;
Expand Down Expand Up @@ -920,15 +925,15 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
};
async_simple::coro::Lazy<void> send_file_without_copy(
const std::filesystem::path &source, std::error_code &ec,
std::size_t length) {
std::size_t length, std::size_t offset) {
fd_guard guard(source.c_str());
if (guard.fd < 0) [[unlikely]] {
ec = std::make_error_code(std::errc::bad_file_descriptor);
co_return;
}
std::size_t actual_len = 0;
std::tie(ec, actual_len) =
co_await coro_io::async_sendfile(socket_->impl_, guard.fd, 0, length);
std::tie(ec, actual_len) = co_await coro_io::async_sendfile(
socket_->impl_, guard.fd, offset, length);
if (ec) [[unlikely]] {
co_return;
}
Expand Down Expand Up @@ -1006,7 +1011,8 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
template <typename S, typename Source>
async_simple::coro::Lazy<resp_data> async_upload(
S uri, http_method method, Source source /* file */,
int64_t content_length = -1,
uint64_t offset = 0 /*file offset*/,
int64_t content_length = -1 /*upload size*/,
req_content_type content_type = req_content_type::text,
std::unordered_map<std::string, std::string> headers = {}) {
std::error_code ec{};
Expand Down Expand Up @@ -1050,6 +1056,12 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
co_return resp_data{std::make_error_code(std::errc::invalid_argument),
404};
}
content_length -= offset;
if (content_length < 0) {
CINATRA_LOG_ERROR << "the offset is larger than the end of file";
co_return resp_data{std::make_error_code(std::errc::invalid_argument),
404};
}
}

assert(content_length >= 0);
Expand Down Expand Up @@ -1088,6 +1100,7 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
}

if constexpr (is_stream_file) {
source->seekg(offset, std::ios::cur);
std::string file_data;
detail::resize(file_data, std::min<std::size_t>(max_single_part_size_,
content_length));
Expand Down Expand Up @@ -1116,15 +1129,17 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
if (!has_init_ssl_) {
#endif
co_await send_file_without_copy(std::filesystem::path{source}, ec,
content_length);
content_length, offset);
#ifdef CINATRA_ENABLE_SSL
}
else {
co_await send_file_no_chunked_with_copy(source, ec, content_length);
co_await send_file_no_chunked_with_copy(source, ec, content_length,
offset);
}
#endif
#else
co_await send_file_no_chunked_with_copy(source, ec, content_length);
co_await send_file_no_chunked_with_copy(source, ec, content_length,
offset);
#endif
}
else {
Expand Down Expand Up @@ -2432,6 +2447,7 @@ class coro_http_client : public std::enable_shared_from_this<coro_http_client> {
std::string resp_chunk_str_;
std::span<char> out_buf_;
bool should_reset_ = false;
config config_;

#ifdef CINATRA_ENABLE_GZIP
bool enable_ws_deflate_ = false;
Expand Down
98 changes: 87 additions & 11 deletions tests/test_cinatra.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,10 @@ bool create_file(std::string_view filename, size_t file_size = 1024) {
return false;
}

std::string str(file_size, 'A');
std::string str;
for (int i = 0; i < file_size; ++i) {
str.push_back(rand() % 26 + 'A');
}
out.write(str.data(), str.size());
return true;
}
Expand Down Expand Up @@ -1146,40 +1149,45 @@ TEST_CASE("test coro_http_client multipart upload") {

TEST_CASE("test coro_http_client upload") {
auto test_upload_by_file_path = [](std::string filename,
std::size_t offset = 0,
std::size_t r_size = SIZE_MAX,
bool should_failed = false) {
coro_http_client client{};
client.add_header("filename", filename);
client.add_header("offset", std::to_string(offset));
if (r_size != SIZE_MAX)
client.add_header("filesize", std::to_string(r_size));
std::string uri = "http://127.0.0.1:8090/upload";
cinatra::resp_data result;
if (r_size != SIZE_MAX) {
auto lazy = client.async_upload(uri, http_method::PUT, filename, r_size);
auto lazy =
client.async_upload(uri, http_method::PUT, filename, offset, r_size);
result = async_simple::coro::syncAwait(lazy);
}
else {
auto lazy = client.async_upload(uri, http_method::PUT, filename);
auto lazy = client.async_upload(uri, http_method::PUT, filename, offset);
result = async_simple::coro::syncAwait(lazy);
}
CHECK(((result.status == 200) ^ should_failed));
};
auto test_upload_by_stream = [](std::string filename,
auto test_upload_by_stream = [](std::string filename, std::size_t offset = 0,
std::size_t r_size = SIZE_MAX,
bool should_failed = false) {
coro_http_client client{};
client.add_header("filename", filename);
client.add_header("offset", std::to_string(offset));
if (r_size != SIZE_MAX)
client.add_header("filesize", std::to_string(r_size));
std::string uri = "http://127.0.0.1:8090/upload";
std::ifstream ifs(filename, std::ios::binary);
cinatra::resp_data result;
if (r_size != SIZE_MAX) {
auto lazy = client.async_upload(uri, http_method::PUT, filename, r_size);
auto lazy =
client.async_upload(uri, http_method::PUT, filename, offset, r_size);
result = async_simple::coro::syncAwait(lazy);
}
else {
auto lazy = client.async_upload(uri, http_method::PUT, filename);
auto lazy = client.async_upload(uri, http_method::PUT, filename, offset);
result = async_simple::coro::syncAwait(lazy);
}
CHECK(((result.status == 200) ^ should_failed));
Expand All @@ -1189,6 +1197,7 @@ TEST_CASE("test coro_http_client upload") {
bool should_failed = false) {
coro_http_client client{};
client.add_header("filename", filename);
client.add_header("offset", "0");
if (r_size != SIZE_MAX)
client.add_header("filesize", std::to_string(r_size));
std::string uri = "http://127.0.0.1:8090/upload";
Expand All @@ -1210,7 +1219,7 @@ TEST_CASE("test coro_http_client upload") {
}
else {
auto lazy =
client.async_upload(uri, http_method::PUT, async_read, r_size);
client.async_upload(uri, http_method::PUT, async_read, 0, r_size);
result = async_simple::coro::syncAwait(lazy);
CHECK(((result.status == 200) ^ should_failed));
}
Expand All @@ -1231,15 +1240,31 @@ TEST_CASE("test coro_http_client upload") {
file.write(req.get_body().data(), req.get_body().size());
file.flush();
file.close();

size_t offset = 0;
std::string offset_s = std::string{req.get_header_value("offset")};
if (!offset_s.empty()) {
offset = stoull(offset_s);
}

std::string filesize = std::string{req.get_header_value("filesize")};

if (!filesize.empty()) {
sz = stoull(filesize);
}
else {
sz = std::filesystem::file_size(oldpath);
sz -= offset;
}

CHECK(!filename.empty());
CHECK(sz == std::filesystem::file_size(newpath));
std::ifstream ifs(oldpath);
ifs.seekg(offset, std::ios::cur);
std::string str;
str.resize(sz);
ifs.read(str.data(), sz);
CHECK(str == req.get_body());
resp.set_status_and_content(status_type::ok, std::string(filename));
co_return;
});
Expand Down Expand Up @@ -1274,8 +1299,8 @@ TEST_CASE("test coro_http_client upload") {
}
bool r = create_file(filename, size);
CHECK(r);
test_upload_by_file_path(filename, r_size);
test_upload_by_stream(filename, r_size);
test_upload_by_file_path(filename, 0, r_size);
test_upload_by_stream(filename, 0, r_size);
test_upload_by_coro(filename, r_size);
}
}
Expand All @@ -1292,11 +1317,62 @@ TEST_CASE("test coro_http_client upload") {
}
bool r = create_file(filename, size);
CHECK(r);
test_upload_by_file_path(filename, r_size, true);
test_upload_by_stream(filename, r_size, true);
test_upload_by_file_path(filename, 0, r_size, true);
test_upload_by_stream(filename, 0, r_size, true);
test_upload_by_coro(filename, r_size, true);
}
}
// upload with offset
{
auto sizes = {std::pair{1024 * 1024, 1'000'000},
std::pair{2'000'000, 1'999'999}, std::pair{200, 1},
std::pair{100, 0}, std::pair{0, 0}};
for (auto [size, offset] : sizes) {
std::error_code ec{};
fs::remove(filename, ec);
if (ec) {
std::cout << ec << "\n";
}
bool r = create_file(filename, size);
CHECK(r);
test_upload_by_file_path(filename, offset);
test_upload_by_stream(filename, offset);
}
}
// upload with size & offset
{
auto sizes = {std::tuple{1024 * 1024, 500'000, 500'000},
std::tuple{2'000'000, 1'999'999, 1}, std::tuple{200, 1, 199},
std::tuple{100, 100, 0}};
for (auto [size, offset, r_size] : sizes) {
std::error_code ec{};
fs::remove(filename, ec);
if (ec) {
std::cout << ec << "\n";
}
bool r = create_file(filename, size);
CHECK(r);
test_upload_by_file_path(filename, offset, r_size);
test_upload_by_stream(filename, offset, r_size);
}
}
// upload with too large size & offset
{
auto sizes = {std::tuple{1024 * 1024, 1'000'000, 50'000},
std::tuple{2'000'000, 1'999'999, 2}, std::tuple{200, 1, 200},
std::tuple{100, 100, 1}};
for (auto [size, offset, r_size] : sizes) {
std::error_code ec{};
fs::remove(filename, ec);
if (ec) {
std::cout << ec << "\n";
}
bool r = create_file(filename, size);
CHECK(r);
test_upload_by_file_path(filename, offset, r_size, true);
test_upload_by_stream(filename, offset, r_size, true);
}
}
}

TEST_CASE("test coro_http_client chunked upload and download") {
Expand Down

0 comments on commit 31c6191

Please sign in to comment.