Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit03f6d3b

Browse files
authored
Merge pull request#720 from umennel/master-integration
Introduced new chunked transfer encoding parser to remove chunk markers
2 parents44f2089 +192d520 commit03f6d3b

File tree

10 files changed

+169
-64
lines changed

10 files changed

+169
-64
lines changed

‎boost/network/protocol/http/client/async_impl.hpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,15 +38,16 @@ struct async_client
3838
typedef std::function<bool(string_type&)> body_generator_function_type;
3939

4040
async_client(bool cache_resolved,bool follow_redirect,
41-
bool always_verify_peer,int timeout,
41+
bool always_verify_peer,int timeout,bool remove_chunk_markers,
4242
std::shared_ptr<boost::asio::io_service> service,
4343
optional<string_type> certificate_filename,
4444
optional<string_type> verify_path,
4545
optional<string_type> certificate_file,
4646
optional<string_type> private_key_file,
4747
optional<string_type> ciphers,
4848
optional<string_type> sni_hostname,long ssl_options)
49-
: connection_base(cache_resolved, follow_redirect, timeout),
49+
: connection_base(cache_resolved, follow_redirect, timeout,
50+
remove_chunk_markers),
5051
service_ptr(service.get() ? service
5152
: std::make_shared<boost::asio::io_service>()),
5253
service_(*service_ptr),

‎boost/network/protocol/http/client/connection/async_base.hpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ struct async_connection_base {
4444
static connection_ptrnew_connection(
4545
resolve_function resolve, resolver_type &resolver,bool follow_redirect,
4646
bool always_verify_peer,bool https,int timeout,
47+
bool remove_chunk_markers,
4748
optional<string_type> certificate_filename = optional<string_type>(),
4849
optional<string_type>const &verify_path = optional<string_type>(),
4950
optional<string_type> certificate_file = optional<string_type>(),
@@ -59,7 +60,8 @@ struct async_connection_base {
5960
certificate_filename, verify_path, certificate_file, private_key_file,
6061
ciphers, sni_hostname, ssl_options);
6162
auto temp = std::make_shared<async_connection>(
62-
resolver, resolve, follow_redirect, timeout,std::move(delegate));
63+
resolver, resolve, follow_redirect, timeout, remove_chunk_markers,
64+
std::move(delegate));
6365
BOOST_ASSERT(temp !=nullptr);
6466
return temp;
6567
}

‎boost/network/protocol/http/client/connection/async_normal.hpp

Lines changed: 112 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,82 @@ namespace network {
3737
namespacehttp {
3838
namespaceimpl {
3939

40+
template<classTag>
41+
structchunk_encoding_parser {
42+
chunk_encoding_parser() : state(state_t::header), chunk_size(0) {}
43+
44+
enumclassstate_t { header, header_end, data, data_end };
45+
46+
state_t state;
47+
size_t chunk_size;
48+
std::array<typename char_<Tag>::type,1024> buffer;
49+
50+
voidupdate_chunk_size(
51+
boost::iterator_range<typename std::array<
52+
typename char_<Tag>::type,1024>::const_iterator>const &range) {
53+
if (range.empty())return;
54+
std::stringstream ss;
55+
ss << std::hex << range;
56+
size_t size;
57+
ss >> size;
58+
// New digits are appended as LSBs
59+
chunk_size = (chunk_size << (range.size() *4)) | size;
60+
}
61+
62+
boost::iterator_range<
63+
typename std::array<typename char_<Tag>::type,1024>::const_iterator>
64+
operator()(
65+
boost::iterator_range<typename std::array<
66+
typename char_<Tag>::type,1024>::const_iterator>const &range) {
67+
auto iter =boost::begin(range);
68+
auto begin = iter;
69+
auto pos =boost::begin(buffer);
70+
71+
while (iter !=boost::end(range))switch (state) {
72+
casestate_t::header:
73+
iter =std::find(iter,boost::end(range),'\r');
74+
update_chunk_size(boost::make_iterator_range(begin, iter));
75+
if (iter !=boost::end(range)) {
76+
state =state_t::header_end;
77+
++iter;
78+
}
79+
break;
80+
81+
casestate_t::header_end:
82+
BOOST_ASSERT(*iter =='\n');
83+
++iter;
84+
state =state_t::data;
85+
break;
86+
87+
casestate_t::data:
88+
if (chunk_size ==0) {
89+
BOOST_ASSERT(*iter =='\r');
90+
++iter;
91+
state =state_t::data_end;
92+
}else {
93+
auto len =std::min(chunk_size,
94+
(size_t)std::distance(iter,boost::end(range)));
95+
begin = iter;
96+
iter =std::next(iter, len);
97+
pos =std::copy(begin, iter, pos);
98+
chunk_size -= len;
99+
}
100+
break;
101+
102+
casestate_t::data_end:
103+
BOOST_ASSERT(*iter =='\n');
104+
++iter;
105+
begin = iter;
106+
state =state_t::header;
107+
break;
108+
109+
default:
110+
BOOST_ASSERT(false &&"Bug, report this to the developers!");
111+
}
112+
returnboost::make_iterator_range(boost::begin(buffer), pos);
113+
}
114+
};
115+
40116
template<classTag,unsigned version_major,unsigned version_minor>
41117
structasync_connection_base;
42118

@@ -72,8 +148,10 @@ struct http_async_connection
72148

73149
http_async_connection(resolver_type& resolver, resolve_function resolve,
74150
bool follow_redirect,int timeout,
151+
bool remove_chunk_markers,
75152
connection_delegate_ptr delegate)
76153
: timeout_(timeout),
154+
remove_chunk_markers_(remove_chunk_markers),
77155
timer_(resolver.get_io_service()),
78156
is_timedout_(false),
79157
follow_redirect_(follow_redirect),
@@ -348,8 +426,11 @@ struct http_async_connection
348426

349427
// The invocation of the callback is synchronous to allow us to
350428
// wait before scheduling another read.
351-
callback(make_iterator_range(begin, end), ec);
352-
429+
if (this->is_chunk_encoding && remove_chunk_markers_) {
430+
callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec);
431+
}else {
432+
callback(make_iterator_range(begin, end), ec);
433+
}
353434
auto self =this->shared_from_this();
354435
delegate_->read_some(
355436
boost::asio::mutable_buffers_1(this->part.data(),
@@ -388,14 +469,31 @@ struct http_async_connection
388469
// We call the callback function synchronously passing the error
389470
// condition (in this case, end of file) so that it can handle it
390471
// appropriately.
391-
callback(make_iterator_range(begin, end), ec);
472+
if (this->is_chunk_encoding && remove_chunk_markers_) {
473+
callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec);
474+
}else {
475+
callback(make_iterator_range(begin, end), ec);
476+
}
392477
}else {
393478
string_type body_string;
394-
std::swap(body_string,this->partial_parsed);
395-
body_string.append(this->part.begin(),this->part.begin() + bytes_transferred);
396-
if (this->is_chunk_encoding) {
397-
this->body_promise.set_value(parse_chunk_encoding(body_string));
479+
if (this->is_chunk_encoding && remove_chunk_markers_) {
480+
for (size_t i =0; i <this->partial_parsed.size(); i +=1024) {
481+
auto range =parse_chunk_encoding(boost::make_iterator_range(
482+
this->partial_parsed.data() + i,
483+
this->partial_parsed.data() +
484+
std::min(i +1024,this->partial_parsed.size())));
485+
body_string.append(boost::begin(range),boost::end(range));
486+
}
487+
this->partial_parsed.clear();
488+
auto range =parse_chunk_encoding(boost::make_iterator_range(
489+
this->part.begin(),
490+
this->part.begin() + bytes_transferred));
491+
body_string.append(boost::begin(range),boost::end(range));
492+
this->body_promise.set_value(body_string);
398493
}else {
494+
std::swap(body_string,this->partial_parsed);
495+
body_string.append(this->part.begin(),
496+
this->part.begin() + bytes_transferred);
399497
this->body_promise.set_value(body_string);
400498
}
401499
}
@@ -417,7 +515,11 @@ struct http_async_connection
417515
this->part.begin();
418516
typename protocol_base::buffer_type::const_iterator end = begin;
419517
std::advance(end, bytes_transferred);
420-
callback(make_iterator_range(begin, end), ec);
518+
if (this->is_chunk_encoding && remove_chunk_markers_) {
519+
callback(parse_chunk_encoding(make_iterator_range(begin, end)), ec);
520+
}else {
521+
callback(make_iterator_range(begin, end), ec);
522+
}
421523
auto self =this->shared_from_this();
422524
delegate_->read_some(
423525
boost::asio::mutable_buffers_1(this->part.data(),
@@ -476,38 +578,8 @@ struct http_async_connection
476578
}
477579
}
478580

479-
string_typeparse_chunk_encoding(string_type& body_string) {
480-
string_type body;
481-
string_type crlf ="\r\n";
482-
483-
typename string_type::iterator begin = body_string.begin();
484-
for (typename string_type::iterator iter =
485-
std::search(begin, body_string.end(), crlf.begin(), crlf.end());
486-
iter != body_string.end();
487-
iter =
488-
std::search(begin, body_string.end(), crlf.begin(), crlf.end())) {
489-
string_typeline(begin, iter);
490-
if (line.empty()) {
491-
break;
492-
}
493-
std::stringstreamstream(line);
494-
int len;
495-
stream >> std::hex >> len;
496-
std::advance(iter,2);
497-
if (len ==0) {
498-
break;
499-
}
500-
if (len <= body_string.end() - iter) {
501-
body.insert(body.end(), iter, iter + len);
502-
std::advance(iter, len +2);
503-
}
504-
begin = iter;
505-
}
506-
507-
return body;
508-
}
509-
510581
int timeout_;
582+
bool remove_chunk_markers_;
511583
boost::asio::steady_timer timer_;
512584
bool is_timedout_;
513585
bool follow_redirect_;
@@ -517,6 +589,7 @@ struct http_async_connection
517589
connection_delegate_ptr delegate_;
518590
boost::asio::streambuf command_streambuf;
519591
string_type method;
592+
chunk_encoding_parser<Tag> parse_chunk_encoding;
520593
};
521594

522595
}// namespace impl

‎boost/network/protocol/http/client/facade.hpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,8 @@ class basic_client_facade {
303303
options.openssl_verify_path(), options.openssl_certificate_file(),
304304
options.openssl_private_key_file(), options.openssl_ciphers(),
305305
options.openssl_sni_hostname(), options.openssl_options(),
306-
options.io_service(), options.timeout()));
306+
options.io_service(), options.timeout(),
307+
options.remove_chunk_markers()));
307308
}
308309
};
309310

‎boost/network/protocol/http/client/options.hpp

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ class client_options {
3434
openssl_options_(0),
3535
io_service_(),
3636
always_verify_peer_(true),
37-
timeout_(0) {}
37+
timeout_(0),
38+
remove_chunk_markers_(false) {}
3839

3940
client_options(client_optionsconst& other)
4041
: cache_resolved_(other.cache_resolved_),
@@ -48,7 +49,8 @@ class client_options {
4849
openssl_options_(other.openssl_options_),
4950
io_service_(other.io_service_),
5051
always_verify_peer_(other.always_verify_peer_),
51-
timeout_(other.timeout_) {}
52+
timeout_(other.timeout_),
53+
remove_chunk_markers_(other.remove_chunk_markers) {}
5254

5355
client_options&operator=(client_options other) {
5456
other.swap(*this);
@@ -69,6 +71,7 @@ class client_options {
6971
swap(io_service_, other.io_service_);
7072
swap(always_verify_peer_, other.always_verify_peer_);
7173
swap(timeout_, other.timeout_);
74+
swap(remove_chunk_markers_, other.remove_chunk_markers_);
7275
}
7376

7477
/// Specify whether the client should cache resolved endpoints.
@@ -154,6 +157,12 @@ class client_options {
154157
return *this;
155158
}
156159

160+
/// Set an overall timeout for HTTP requests.
161+
client_options&remove_chunk_markers(bool v) {
162+
remove_chunk_markers_ = v;
163+
return *this;
164+
}
165+
157166
boolcache_resolved()const {return cache_resolved_; }
158167

159168
boolfollow_redirects()const {return follow_redirects_; }
@@ -190,6 +199,8 @@ class client_options {
190199

191200
inttimeout()const {return timeout_; }
192201

202+
boolremove_chunk_markers()const {return remove_chunk_markers_; }
203+
193204
private:
194205
bool cache_resolved_;
195206
bool follow_redirects_;
@@ -203,6 +214,7 @@ class client_options {
203214
std::shared_ptr<boost::asio::io_service> io_service_;
204215
bool always_verify_peer_;
205216
int timeout_;
217+
bool remove_chunk_markers_;
206218
};
207219

208220
template<classTag>

‎boost/network/protocol/http/client/pimpl.hpp

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,12 @@ struct basic_client_impl
7474
optional<string_type>const& private_key_file,
7575
optional<string_type>const& ciphers,
7676
optional<string_type>const& sni_hostname,long ssl_options,
77-
std::shared_ptr<boost::asio::io_service> service,int timeout)
78-
: base_type(cache_resolved, follow_redirect, always_verify_peer, timeout,
79-
service, certificate_filename, verify_path, certificate_file,
80-
private_key_file, ciphers, sni_hostname, ssl_options) {}
77+
std::shared_ptr<boost::asio::io_service> service,int timeout,
78+
bool remove_chunk_markers)
79+
: base_type(cache_resolved, follow_redirect, always_verify_peer, timeout,
80+
remove_chunk_markers, service, certificate_filename, verify_path,
81+
certificate_file, private_key_file, ciphers, sni_hostname,
82+
ssl_options) {}
8183

8284
~basic_client_impl() =default;
8385
};

‎boost/network/protocol/http/policies/async_connection.hpp

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ struct async_connection_policy : resolver_policy<Tag>::type {
3838
structconnection_impl {
3939
connection_impl(
4040
bool follow_redirect,bool always_verify_peer, resolve_function resolve,
41-
resolver_type& resolver,bool https,int timeout,
41+
resolver_type& resolver,bool https,int timeout,bool remove_chunk_markers,
4242
optional<string_type>/*unused*/const& certificate_filename,
4343
optional<string_type>const& verify_path,
4444
optional<string_type>const& certificate_file,
@@ -47,9 +47,9 @@ struct async_connection_policy : resolver_policy<Tag>::type {
4747
optional<string_type>const& sni_hostname,long ssl_options) {
4848
pimpl = impl::async_connection_base<Tag, version_major, version_minor>::
4949
new_connection(resolve, resolver, follow_redirect, always_verify_peer,
50-
https, timeout,certificate_filename, verify_path,
51-
certificate_file, private_key_file, ciphers,
52-
sni_hostname, ssl_options);
50+
https, timeout,remove_chunk_markers,
51+
certificate_filename, verify_path, certificate_file,
52+
private_key_file, ciphers,sni_hostname, ssl_options);
5353
}
5454

5555
basic_response<Tag>send_request(string_type/*unused*/const& method,
@@ -86,20 +86,22 @@ struct async_connection_policy : resolver_policy<Tag>::type {
8686
this->resolve(resolver, host, port, once_resolved);
8787
},
8888
resolver,boost::iequals(protocol_,string_type("https")), timeout_,
89-
certificate_filename, verify_path, certificate_file, private_key_file,
90-
ciphers, sni_hostname, ssl_options);
89+
remove_chunk_markers_, certificate_filename, verify_path,
90+
certificate_file, private_key_file,ciphers, sni_hostname, ssl_options);
9191
}
9292

9393
voidcleanup() {}
9494

9595
async_connection_policy(bool cache_resolved,bool follow_redirect,
96-
int timeout)
96+
int timeout,bool remove_chunk_markers)
9797
: resolver_base(cache_resolved),
9898
follow_redirect_(follow_redirect),
99-
timeout_(timeout) {}
99+
timeout_(timeout),
100+
remove_chunk_markers_(remove_chunk_markers) {}
100101

101102
bool follow_redirect_;
102103
int timeout_;
104+
bool remove_chunk_markers_;
103105
};
104106

105107
}// namespace http

‎libs/network/test/http/client_get_different_port_test.cpp

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,12 @@ namespace http = boost::network::http;
1515
TYPED_TEST_CASE(HTTPClientTest, ClientTypes);
1616

1717
TYPED_TEST(HTTPClientTest, GetDifferentPort) {
18-
TypeParam client;
19-
typename TypeParam::requestr("http://www.boost.org:80/");
20-
auto response_ = client.get(r);
18+
using client = TypeParam;
19+
typename client::options options;
20+
options.remove_chunk_markers(true);
21+
client client_;
22+
typename TypeParam::requestrequest("http://www.boost.org:80/");
23+
auto response_ = client_.get(request);
2124
auto range =headers(response_)["Content-Type"];
2225
EXPECT_TRUE(std::begin(range) !=std::end(range));
2326
EXPECT_NE(0,body(response_).size());

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp