Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit9ab1643

Browse files
umenneldeanberris
authored andcommitted
Introduced new chunked transfer encoding parser to remove chunk markers in streaming clients.
1 parent73d4024 commit9ab1643

File tree

7 files changed

+126
-22
lines changed

7 files changed

+126
-22
lines changed

‎boost/network/protocol/http/client/async_impl.hpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,14 +42,15 @@ struct async_client
4242

4343
async_client(bool cache_resolved,bool follow_redirect,
4444
bool always_verify_peer,int timeout,
45+
bool remove_chunk_markers,
4546
std::shared_ptr<boost::asio::io_service> service,
4647
optional<string_type> certificate_filename,
4748
optional<string_type> verify_path,
4849
optional<string_type> certificate_file,
4950
optional<string_type> private_key_file,
5051
optional<string_type> ciphers,
5152
optional<string_type> sni_hostname,long ssl_options)
52-
: connection_base(cache_resolved, follow_redirect, timeout),
53+
: connection_base(cache_resolved, follow_redirect, timeout, remove_chunk_markers),
5354
service_ptr(service.get() ? service
5455
: std::make_shared<boost::asio::io_service>()),
5556
service_(*service_ptr),

‎boost/network/protocol/http/client/connection/async_base.hpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ struct async_connection_base {
4343
// tag.
4444
static connection_ptrnew_connection(
4545
resolve_function resolve, resolver_type &resolver,bool follow_redirect,
46-
bool always_verify_peer,bool https,int timeout,
46+
bool always_verify_peer,bool https,int timeout,bool remove_chunk_markers,
4747
optional<string_type> certificate_filename = optional<string_type>(),
4848
optional<string_type>const &verify_path = optional<string_type>(),
4949
optional<string_type> certificate_file = optional<string_type>(),
@@ -59,7 +59,7 @@ struct async_connection_base {
5959
certificate_filename, verify_path, certificate_file, private_key_file,
6060
ciphers, sni_hostname, ssl_options);
6161
auto temp = std::make_shared<async_connection>(
62-
resolver, resolve, follow_redirect, timeout,std::move(delegate));
62+
resolver, resolve, follow_redirect, timeout,remove_chunk_markers,std::move(delegate));
6363
BOOST_ASSERT(temp !=nullptr);
6464
return temp;
6565
}

‎boost/network/protocol/http/client/connection/async_normal.hpp‎

Lines changed: 92 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
#include<iterator>
1313
#include<cstdint>
14+
#include<iostream>
1415
#include<boost/algorithm/string/trim.hpp>
1516
#include<boost/asio/steady_timer.hpp>
1617
#include<boost/asio/placeholders.hpp>
@@ -37,6 +38,77 @@ namespace network {
3738
namespacehttp {
3839
namespaceimpl {
3940

41+
template<classTag>
42+
structchunk_encoding_parser {
43+
44+
chunk_encoding_parser() : state(state_t::header), chunk_size(0) {}
45+
46+
enumstate_t { header, header_end, data, data_end };
47+
48+
state_t state;
49+
size_t chunk_size;
50+
std::array<typename char_<Tag>::type,1024> buffer;
51+
52+
voidupdate_chunk_size(boost::iterator_range<typename std::array<typename char_<Tag>::type,1024>::const_iterator>const& range) {
53+
if (range.empty())
54+
return;
55+
std::stringstream ss;
56+
ss << std::hex << range;
57+
size_t size;
58+
ss >> size;
59+
chunk_size = (chunk_size << (range.size()*4)) + size;
60+
}
61+
62+
boost::iterator_range<typename std::array<typename char_<Tag>::type,1024>::const_iterator>operator()(boost::iterator_range<typename std::array<typename char_<Tag>::type,1024>::const_iterator>const& range) {
63+
auto iter =boost::begin(range);
64+
auto begin = iter;
65+
auto pos =boost::begin(buffer);
66+
67+
while (iter !=boost::end(range))
68+
switch(state) {
69+
casestate_t::header:
70+
iter =std::find(iter,boost::end(range),'\r');
71+
update_chunk_size(boost::make_iterator_range(begin, iter));
72+
if (iter !=boost::end(range)) {
73+
state =state_t::header_end;
74+
++iter;
75+
}
76+
break;
77+
78+
casestate_t::header_end:
79+
BOOST_ASSERT(*iter =='\n');
80+
++iter;
81+
state =state_t::data;
82+
break;
83+
84+
casestate_t::data:
85+
if (chunk_size ==0) {
86+
BOOST_ASSERT(*iter =='\r');
87+
++iter;
88+
state =state_t::data_end;
89+
}else {
90+
auto len =std::min(chunk_size, (size_t)std::distance(iter,boost::end(range)));
91+
begin = iter;
92+
iter =std::next(iter, len);
93+
pos =std::copy(begin, iter, pos);
94+
chunk_size -= len;
95+
}
96+
break;
97+
98+
casestate_t::data_end:
99+
BOOST_ASSERT (*iter =='\n');
100+
++iter;
101+
begin = iter;
102+
state =state_t::header;
103+
break;
104+
105+
default:
106+
BOOST_ASSERT(false &&"Bug, report this to the developers!");
107+
}
108+
returnboost::make_iterator_range(boost::begin(buffer), pos);
109+
}
110+
};
111+
40112
template<classTag,unsigned version_major,unsigned version_minor>
41113
structasync_connection_base;
42114

@@ -73,8 +145,10 @@ struct http_async_connection
73145

74146
http_async_connection(resolver_type& resolver, resolve_function resolve,
75147
bool follow_redirect,int timeout,
148+
bool remove_chunk_markers,
76149
connection_delegate_ptr delegate)
77150
: timeout_(timeout),
151+
remove_chunk_markers_(remove_chunk_markers),
78152
timer_(resolver.get_io_service()),
79153
is_timedout_(false),
80154
follow_redirect_(follow_redirect),
@@ -397,13 +471,23 @@ struct http_async_connection
397471
callback(make_iterator_range(begin, end), ec);
398472
}else {
399473
string_type body_string;
400-
std::swap(body_string,this->partial_parsed);
401-
auto it =this->part.begin();
402-
std::advance(it, bytes_transferred);
403-
body_string.append(this->part.begin(), it);
404-
if (this->is_chunk_encoding) {
405-
this->body_promise.set_value(parse_chunk_encoding(body_string));
474+
if (this->is_chunk_encoding && remove_chunk_markers_) {
475+
for (size_t i =0; i <this->partial_parsed.size(); i +=1024) {
476+
auto range =parse_chunk_encoding(
477+
boost::make_iterator_range(this->partial_parsed.data() + i,
478+
this->partial_parsed.data() +std::min(i+1024,this->partial_parsed.size())));
479+
body_string.append(boost::begin(range),boost::end(range));
480+
}
481+
this->partial_parsed.clear();
482+
auto range =parse_chunk_encoding(boost::make_iterator_range(this->part.begin(),
483+
this->part.begin() + bytes_transferred));
484+
body_string.append(boost::begin(range),boost::end(range));
485+
this->body_promise.set_value(body_string);
406486
}else {
487+
std::swap(body_string,this->partial_parsed);
488+
auto it =this->part.begin();
489+
std::advance(it, bytes_transferred);
490+
body_string.append(this->part.begin(), it);
407491
this->body_promise.set_value(body_string);
408492
}
409493
}
@@ -520,6 +604,7 @@ struct http_async_connection
520604
}
521605

522606
int timeout_;
607+
bool remove_chunk_markers_;
523608
boost::asio::steady_timer timer_;
524609
bool is_timedout_;
525610
bool follow_redirect_;
@@ -529,6 +614,7 @@ struct http_async_connection
529614
connection_delegate_ptr delegate_;
530615
boost::asio::streambuf command_streambuf;
531616
string_type method;
617+
chunk_encoding_parser<Tag> parse_chunk_encoding;
532618
};
533619

534620
}// namespace impl

‎boost/network/protocol/http/client/facade.hpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,8 @@ class basic_client_facade {
312312
options.openssl_verify_path(), options.openssl_certificate_file(),
313313
options.openssl_private_key_file(), options.openssl_ciphers(),
314314
options.openssl_sni_hostname(), options.openssl_options(),
315-
options.io_service(), options.timeout()));
315+
options.io_service(), options.timeout(),
316+
options.remove_chunk_markers()));
316317
}
317318
};
318319

‎boost/network/protocol/http/client/options.hpp‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ class client_options {
3434
openssl_options_(0),
3535
io_service_(),
3636
always_verify_peer_(true),
37-
timeout_(0) {}
37+
timeout_(0),
38+
remove_chunk_markers_(false) {}
3839

3940
client_options(client_optionsconst& other)
4041
: cache_resolved_(other.cache_resolved_),
@@ -48,7 +49,8 @@ class client_options {
4849
openssl_options_(other.openssl_options_),
4950
io_service_(other.io_service_),
5051
always_verify_peer_(other.always_verify_peer_),
51-
timeout_(other.timeout_) {}
52+
timeout_(other.timeout_),
53+
remove_chunk_markers_(other.remove_chunk_markers) {}
5254

5355
client_options&operator=(client_options other) {
5456
other.swap(*this);
@@ -69,6 +71,7 @@ class client_options {
6971
swap(io_service_, other.io_service_);
7072
swap(always_verify_peer_, other.always_verify_peer_);
7173
swap(timeout_, other.timeout_);
74+
swap(remove_chunk_markers_, other.remove_chunk_markers_);
7275
}
7376

7477
/// Specify whether the client should cache resolved endpoints.
@@ -154,6 +157,12 @@ class client_options {
154157
return *this;
155158
}
156159

160+
/// Set an overall timeout for HTTP requests.
161+
client_options&remove_chunk_markers(bool v) {
162+
remove_chunk_markers_ = v;
163+
return *this;
164+
}
165+
157166
boolcache_resolved()const {return cache_resolved_; }
158167

159168
boolfollow_redirects()const {return follow_redirects_; }
@@ -190,6 +199,8 @@ class client_options {
190199

191200
inttimeout()const {return timeout_; }
192201

202+
boolremove_chunk_markers()const {return remove_chunk_markers_; }
203+
193204
private:
194205
bool cache_resolved_;
195206
bool follow_redirects_;
@@ -203,6 +214,7 @@ class client_options {
203214
std::shared_ptr<boost::asio::io_service> io_service_;
204215
bool always_verify_peer_;
205216
int timeout_;
217+
bool remove_chunk_markers_;
206218
};
207219

208220
template<classTag>

‎boost/network/protocol/http/client/pimpl.hpp‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,12 @@ struct basic_client_impl
7474
optional<string_type>const& private_key_file,
7575
optional<string_type>const& ciphers,
7676
optional<string_type>const& sni_hostname,long ssl_options,
77-
std::shared_ptr<boost::asio::io_service> service,int timeout)
78-
: base_type(cache_resolved, follow_redirect, always_verify_peer, timeout,
79-
service, certificate_filename, verify_path, certificate_file,
80-
private_key_file, ciphers, sni_hostname, ssl_options) {}
77+
std::shared_ptr<boost::asio::io_service> service,int timeout,
78+
bool remove_chunk_markers)
79+
: base_type(cache_resolved, follow_redirect, always_verify_peer, timeout,
80+
remove_chunk_markers, service, certificate_filename, verify_path,
81+
certificate_file, private_key_file, ciphers, sni_hostname,
82+
ssl_options) {}
8183

8284
~basic_client_impl() =default;
8385
};

‎boost/network/protocol/http/policies/async_connection.hpp‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ struct async_connection_policy : resolver_policy<Tag>::type {
4040
structconnection_impl {
4141
connection_impl(
4242
bool follow_redirect,bool always_verify_peer, resolve_function resolve,
43-
resolver_type& resolver,bool https,int timeout,
43+
resolver_type& resolver,bool https,int timeout,bool remove_chunk_markers,
4444
optional<string_type>/*unused*/const& certificate_filename,
4545
optional<string_type>const& verify_path,
4646
optional<string_type>const& certificate_file,
@@ -49,8 +49,8 @@ struct async_connection_policy : resolver_policy<Tag>::type {
4949
optional<string_type>const& sni_hostname,long ssl_options) {
5050
pimpl = impl::async_connection_base<Tag, version_major, version_minor>::
5151
new_connection(resolve, resolver, follow_redirect, always_verify_peer,
52-
https, timeout,certificate_filename, verify_path,
53-
certificate_file, private_key_file, ciphers,
52+
https, timeout,remove_chunk_markers, certificate_filename,
53+
verify_path,certificate_file, private_key_file, ciphers,
5454
sni_hostname, ssl_options);
5555
}
5656

@@ -87,21 +87,23 @@ struct async_connection_policy : resolver_policy<Tag>::type {
8787
std::uint16_t port, resolve_completion_function once_resolved) {
8888
this->resolve(resolver, host, port, once_resolved);
8989
},
90-
resolver,boost::iequals(protocol_,string_type("https")), timeout_,
90+
resolver,boost::iequals(protocol_,string_type("https")), timeout_, remove_chunk_markers_,
9191
certificate_filename, verify_path, certificate_file, private_key_file,
9292
ciphers, sni_hostname, ssl_options);
9393
}
9494

9595
voidcleanup() {}
9696

9797
async_connection_policy(bool cache_resolved,bool follow_redirect,
98-
int timeout)
98+
int timeout,bool remove_chunk_markers)
9999
: resolver_base(cache_resolved),
100100
follow_redirect_(follow_redirect),
101-
timeout_(timeout) {}
101+
timeout_(timeout),
102+
remove_chunk_markers_(remove_chunk_markers) {}
102103

103104
bool follow_redirect_;
104105
int timeout_;
106+
bool remove_chunk_markers_;
105107
};
106108

107109
}// namespace http

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp