From 9f163418fabbe6219ab04cfe9bf81d2f33bd54d7 Mon Sep 17 00:00:00 2001 From: Richy Kim Date: Tue, 20 Dec 2016 05:48:15 -0500 Subject: [PATCH 1/7] CURLOPT_BUFFERSIZE: support enlarging receive buffer Replace use of fixed macro BUFSIZE to define the size of the receive buffer. Reappropriate CURLOPT_BUFFERSIZE to include enlarging receive buffer size. Upon setting, resize buffer if larger than the current default size up to a MAX_BUFSIZE (512KB). This can benefit protocols like SFTP. Closes #1222 Upstream-commit: 6b7616690e5370c21e3a760321af6bf4edbabfb6 Signed-off-by: Kamil Dudka --- docs/libcurl/curl_easy_setopt.3 | 12 ++++++------ docs/libcurl/symbols-in-versions | 1 + include/curl/curl.h | 5 +++++ lib/easy.c | 6 ++++++ lib/file.c | 2 +- lib/ftp.c | 4 ++-- lib/http.c | 3 ++- lib/telnet.c | 5 +++-- lib/url.c | 28 +++++++++++++++++++++++----- lib/urldata.h | 5 ++++- 10 files changed, 53 insertions(+), 18 deletions(-) diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3 index cbebfba..17b632f 100644 --- a/docs/libcurl/curl_easy_setopt.3 +++ b/docs/libcurl/curl_easy_setopt.3 @@ -938,12 +938,12 @@ to using the share interface instead! See \fICURLOPT_SHARE\fP and .IP CURLOPT_BUFFERSIZE Pass a long specifying your preferred size (in bytes) for the receive buffer in libcurl. The main point of this would be that the write callback gets -called more often and with smaller chunks. This is just treated as a request, -not an order. You cannot be guaranteed to actually get the given size. (Added -in 7.10) - -This size is by default set as big as possible (CURL_MAX_WRITE_SIZE), so it -only makes sense to use this option if you want it smaller. +called more often and with smaller chunks. Secondly, for some protocols, +there's a benefit of having a larger buffer for performance. This is just +treated as a request, not an order. You cannot be guaranteed to actually get +the given size. This buffer size is by default \fICURL_MAX_WRITE_SIZE\fP +(16kB). The maximum buffer size allowed to set is \fICURL_MAX_READ_SIZE\fP +(512kB). (Added in 7.10) .IP CURLOPT_PORT Pass a long specifying what remote port number to connect to, instead of the one specified in the URL or the default port for the used protocol. diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions index b0b6232..e2cce4c 100644 --- a/docs/libcurl/symbols-in-versions +++ b/docs/libcurl/symbols-in-versions @@ -639,6 +639,7 @@ CURL_LOCK_TYPE_DNS 7.10 - 7.10.2 CURL_LOCK_TYPE_NONE 7.10 - 7.10.2 CURL_LOCK_TYPE_SSL_SESSION 7.10 - 7.10.2 CURL_MAX_HTTP_HEADER 7.19.7 +CURL_MAX_READ_SIZE 7.53.0 CURL_MAX_WRITE_SIZE 7.9.7 CURL_NETRC_IGNORED 7.9.8 CURL_NETRC_OPTIONAL 7.9.8 diff --git a/include/curl/curl.h b/include/curl/curl.h index 0375a64..8b639fa 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h @@ -170,6 +170,11 @@ typedef int (*curl_progress_callback)(void *clientp, double ultotal, double ulnow); +#ifndef CURL_MAX_READ_SIZE + /* The maximum receive buffer size configurable via CURLOPT_BUFFERSIZE. */ +#define CURL_MAX_READ_SIZE 524288 +#endif + #ifndef CURL_MAX_WRITE_SIZE /* Tests have proven that 20K is a very bad buffer size for uploads on Windows, while 16K for some odd reason performed a lot better. diff --git a/lib/easy.c b/lib/easy.c index 0e9ba18..5d4d5ae 100644 --- a/lib/easy.c +++ b/lib/easy.c @@ -563,6 +563,11 @@ CURL *curl_easy_duphandle(CURL *incurl) * get setup on-demand in the code, as that would probably decrease * the likeliness of us forgetting to init a buffer here in the future. */ + outcurl->set.buffer_size = data->set.buffer_size; + outcurl->state.buffer = malloc(CURL_BUFSIZE(outcurl->set.buffer_size) + 1); + if(!outcurl->state.buffer) + goto fail; + outcurl->state.headerbuff = malloc(HEADERSIZE); if(!outcurl->state.headerbuff) goto fail; @@ -633,6 +638,7 @@ CURL *curl_easy_duphandle(CURL *incurl) if(outcurl) { curl_slist_free_all(outcurl->change.cookielist); outcurl->change.cookielist = NULL; + Curl_safefree(outcurl->state.buffer); Curl_safefree(outcurl->state.headerbuff); Curl_safefree(outcurl->change.url); Curl_safefree(outcurl->change.referer); diff --git a/lib/file.c b/lib/file.c index 038bf42..1ad4758 100644 --- a/lib/file.c +++ b/lib/file.c @@ -473,7 +473,7 @@ static CURLcode file_do(struct connectdata *conn, bool *done) date. */ if(data->set.opt_no_body && data->set.include_header && fstated) { CURLcode result; - snprintf(buf, sizeof(data->state.buffer), + snprintf(buf, CURL_BUFSIZE(data->set.buffer_size), "Content-Length: %" FORMAT_OFF_T "\r\n", expected_size); result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0); if(result) diff --git a/lib/ftp.c b/lib/ftp.c index a9826ce..730b695 100644 --- a/lib/ftp.c +++ b/lib/ftp.c @@ -2136,7 +2136,7 @@ static CURLcode ftp_state_mdtm_resp(struct connectdata *conn, /* we have a time, reformat it */ time_t secs=time(NULL); /* using the good old yacc/bison yuck */ - snprintf(buf, sizeof(conn->data->state.buffer), + snprintf(buf, CURL_BUFSIZE(conn->data->set.buffer_size), "%04d%02d%02d %02d:%02d:%02d GMT", year, month, day, hour, minute, second); /* now, convert this into a time() value: */ @@ -2347,7 +2347,7 @@ static CURLcode ftp_state_size_resp(struct connectdata *conn, if(instate == FTP_SIZE) { #ifdef CURL_FTP_HTTPSTYLE_HEAD if(-1 != filesize) { - snprintf(buf, sizeof(data->state.buffer), + snprintf(buf, CURL_BUFSIZE(data->set.buffer_size), "Content-Length: %" FORMAT_OFF_T "\r\n", filesize); result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0); if(result) diff --git a/lib/http.c b/lib/http.c index 1487fb2..f4368c4 100644 --- a/lib/http.c +++ b/lib/http.c @@ -247,7 +247,8 @@ static CURLcode http_output_basic(struct connectdata *conn, bool proxy) pwd = conn->passwd; } - snprintf(data->state.buffer, sizeof(data->state.buffer), "%s:%s", user, pwd); + snprintf(data->state.buffer, CURL_BUFSIZE(data->set.buffer_size), + "%s:%s", user, pwd); error = Curl_base64_encode(data, data->state.buffer, strlen(data->state.buffer), diff --git a/lib/telnet.c b/lib/telnet.c index 77d8b7b..89452dd 100644 --- a/lib/telnet.c +++ b/lib/telnet.c @@ -1421,6 +1421,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done) /* Keep on listening and act on events */ while(keepon) { + const size_t buf_size = CURL_BUFSIZE(data->set.buffer_size); waitret = WaitForMultipleObjects(obj_count, objs, FALSE, wait_timeout); switch(waitret) { case WAIT_TIMEOUT: @@ -1455,7 +1456,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done) if(!readfile_read) break; - if(!ReadFile(stdin_handle, buf, sizeof(data->state.buffer), + if(!ReadFile(stdin_handle, buf, buf_size, &readfile_read, NULL)) { keepon = FALSE; code = CURLE_READ_ERROR; @@ -1474,7 +1475,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done) case WAIT_OBJECT_0 + 1: { - if(!ReadFile(stdin_handle, buf, sizeof(data->state.buffer), + if(!ReadFile(stdin_handle, buf, buf_size, &readfile_read, NULL)) { keepon = FALSE; code = CURLE_READ_ERROR; diff --git a/lib/url.c b/lib/url.c index 89958a7..32e7e2e 100644 --- a/lib/url.c +++ b/lib/url.c @@ -441,6 +441,7 @@ CURLcode Curl_close(struct SessionHandle *data) } data->change.url = NULL; + Curl_safefree(data->state.buffer); Curl_safefree(data->state.headerbuff); Curl_flush_cookies(data, 1); @@ -612,6 +613,12 @@ CURLcode Curl_open(struct SessionHandle **curl) /* We do some initial setup here, all those fields that can't be just 0 */ + data->state.buffer = malloc(BUFSIZE + 1); + if(!data->state.buffer) { + DEBUGF(fprintf(stderr, "Error: malloc of buffer failed\n")); + res = CURLE_OUT_OF_MEMORY; + } + data->state.headerbuff = malloc(HEADERSIZE); if(!data->state.headerbuff) { DEBUGF(fprintf(stderr, "Error: malloc of headerbuff failed\n")); @@ -642,8 +649,8 @@ CURLcode Curl_open(struct SessionHandle **curl) if(res) { Curl_resolver_cleanup(data->state.resolver); - if(data->state.headerbuff) - free(data->state.headerbuff); + free(data->state.buffer); + free(data->state.headerbuff); Curl_freeset(data); free(data); data = NULL; @@ -1960,9 +1967,20 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option, */ data->set.buffer_size = va_arg(param, long); - if((data->set.buffer_size> (BUFSIZE -1 )) || - (data->set.buffer_size < 1)) - data->set.buffer_size = 0; /* huge internal default */ + if(data->set.buffer_size > MAX_BUFSIZE) + data->set.buffer_size = MAX_BUFSIZE; /* huge internal default */ + else if(data->set.buffer_size < 1) + data->set.buffer_size = BUFSIZE; + + /* Resize only if larger than default buffer size. */ + if(data->set.buffer_size > BUFSIZE) { + data->state.buffer = realloc(data->state.buffer, + data->set.buffer_size + 1); + if(!data->state.buffer) { + DEBUGF(fprintf(stderr, "Error: realloc of buffer failed\n")); + result = CURLE_OUT_OF_MEMORY; + } + } break; diff --git a/lib/urldata.h b/lib/urldata.h index 7431825..a7807cf 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -196,6 +196,9 @@ /* Download buffer size, keep it fairly big for speed reasons */ #undef BUFSIZE #define BUFSIZE CURL_MAX_WRITE_SIZE +#undef MAX_BUFSIZE +#define MAX_BUFSIZE CURL_MAX_READ_SIZE +#define CURL_BUFSIZE(x) ((x)?(x):(BUFSIZE)) /* Initial size of the buffer to store headers in, it'll be enlarged in case of need. */ @@ -1174,7 +1177,7 @@ struct UrlState { char *headerbuff; /* allocated buffer to store headers in */ size_t headersize; /* size of the allocation */ - char buffer[BUFSIZE+1]; /* download buffer */ + char *buffer; /* download buffer */ char uploadbuffer[BUFSIZE+1]; /* upload buffer */ curl_off_t current_speed; /* the ProgressShow() funcion sets this, bytes / second */ -- 2.14.3 From f175a713c964d351012baaf8c78c1b468cc6aba0 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Mon, 24 Apr 2017 15:33:57 +0200 Subject: [PATCH 2/7] http: use private user:password output buffer Don't clobber the receive buffer. Upstream-commit: 94460878cc634b590a7282e3fe60ceafb62d141a Signed-off-by: Kamil Dudka --- lib/http.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/lib/http.c b/lib/http.c index f4368c4..12e7dc3 100644 --- a/lib/http.c +++ b/lib/http.c @@ -234,7 +234,8 @@ static CURLcode http_output_basic(struct connectdata *conn, bool proxy) char **userp; const char *user; const char *pwd; - CURLcode error; + CURLcode result; + char *out; if(proxy) { userp = &conn->allocptr.proxyuserpwd; @@ -247,27 +248,32 @@ static CURLcode http_output_basic(struct connectdata *conn, bool proxy) pwd = conn->passwd; } - snprintf(data->state.buffer, CURL_BUFSIZE(data->set.buffer_size), - "%s:%s", user, pwd); + out = aprintf("%s:%s", user, pwd); + if(!out) + return CURLE_OUT_OF_MEMORY; - error = Curl_base64_encode(data, - data->state.buffer, strlen(data->state.buffer), - &authorization, &size); - if(error) - return error; + result = Curl_base64_encode(data, out, strlen(out), &authorization, &size); + if(result) + goto fail; - if(!authorization) - return CURLE_REMOTE_ACCESS_DENIED; + if(!authorization) { + result = CURLE_REMOTE_ACCESS_DENIED; + goto fail; + } Curl_safefree(*userp); *userp = aprintf("%sAuthorization: Basic %s\r\n", proxy?"Proxy-":"", authorization); free(authorization); - if(!*userp) - return CURLE_OUT_OF_MEMORY; + if(!*userp) { + result = CURLE_OUT_OF_MEMORY; + goto fail; + } - return CURLE_OK; + fail: + free(out); + return result; } /* pickoneauth() selects the most favourable authentication method from the -- 2.14.3 From 6ff175806c338223a2a9a69f6ae8ae2b91dc2b56 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Mon, 24 Apr 2017 16:05:46 +0200 Subject: [PATCH 3/7] ftp: use private buffer for temp storage, not receive buffer Upstream-commit: 349789e645a306a6ee467ef90a57f6cc306ca92e Signed-off-by: Kamil Dudka --- lib/ftp.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/ftp.c b/lib/ftp.c index 730b695..10a21ce 100644 --- a/lib/ftp.c +++ b/lib/ftp.c @@ -2130,17 +2130,17 @@ static CURLcode ftp_state_mdtm_resp(struct connectdata *conn, /* we got a time. Format should be: "YYYYMMDDHHMMSS[.sss]" where the last .sss part is optional and means fractions of a second */ int year, month, day, hour, minute, second; - char *buf = data->state.buffer; - if(6 == sscanf(buf+4, "%04d%02d%02d%02d%02d%02d", + if(6 == sscanf(&data->state.buffer[4], "%04d%02d%02d%02d%02d%02d", &year, &month, &day, &hour, &minute, &second)) { /* we have a time, reformat it */ + char timebuf[24]; time_t secs=time(NULL); - /* using the good old yacc/bison yuck */ - snprintf(buf, CURL_BUFSIZE(conn->data->set.buffer_size), + + snprintf(timebuf, sizeof(timebuf), "%04d%02d%02d %02d:%02d:%02d GMT", year, month, day, hour, minute, second); /* now, convert this into a time() value: */ - data->info.filetime = (long)curl_getdate(buf, &secs); + data->info.filetime = (long)curl_getdate(timebuf, &secs); } #ifdef CURL_FTP_HTTPSTYLE_HEAD @@ -2151,6 +2151,7 @@ static CURLcode ftp_state_mdtm_resp(struct connectdata *conn, ftpc->file && data->set.get_filetime && (data->info.filetime>=0) ) { + char headerbuf[128]; time_t filetime = (time_t)data->info.filetime; struct tm buffer; const struct tm *tm = &buffer; @@ -2160,7 +2161,7 @@ static CURLcode ftp_state_mdtm_resp(struct connectdata *conn, return result; /* format: "Tue, 15 Nov 1994 12:45:26" */ - snprintf(buf, BUFSIZE-1, + snprintf(headerbuf, sizeof(headerbuf), "Last-Modified: %s, %02d %s %4d %02d:%02d:%02d GMT\r\n", Curl_wkday[tm->tm_wday?tm->tm_wday-1:6], tm->tm_mday, @@ -2169,7 +2170,7 @@ static CURLcode ftp_state_mdtm_resp(struct connectdata *conn, tm->tm_hour, tm->tm_min, tm->tm_sec); - result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0); + result = Curl_client_write(conn, CLIENTWRITE_BOTH, headerbuf, 0); if(result) return result; } /* end of a ridiculous amount of conditionals */ @@ -2347,9 +2348,10 @@ static CURLcode ftp_state_size_resp(struct connectdata *conn, if(instate == FTP_SIZE) { #ifdef CURL_FTP_HTTPSTYLE_HEAD if(-1 != filesize) { - snprintf(buf, CURL_BUFSIZE(data->set.buffer_size), + char clbuf[128]; + snprintf(clbuf, sizeof(clbuf), "Content-Length: %" FORMAT_OFF_T "\r\n", filesize); - result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0); + result = Curl_client_write(conn, CLIENTWRITE_BOTH, clbuf, 0); if(result) return result; } @@ -2450,7 +2452,6 @@ static CURLcode ftp_state_get_resp(struct connectdata *conn, CURLcode result = CURLE_OK; struct SessionHandle *data = conn->data; struct FTP *ftp = data->state.proto.ftp; - char *buf = data->state.buffer; if((ftpcode == 150) || (ftpcode == 125)) { @@ -2494,6 +2495,7 @@ static CURLcode ftp_state_get_resp(struct connectdata *conn, * * Example D above makes this parsing a little tricky */ char *bytes; + char *buf = data->state.buffer; bytes=strstr(buf, " bytes"); if(bytes--) { long in=(long)(bytes-buf); -- 2.14.3 From b67324919089fc4f9bb7a38a6a31174883a4bc24 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Tue, 25 Apr 2017 00:09:22 +0200 Subject: [PATCH 4/7] CURLOPT_BUFFERSIZE: 1024 bytes is now the minimum size The buffer is needed to receive FTP, HTTP CONNECT responses etc so already at this size things risk breaking and smaller is certainly not wise. Upstream-commit: c2ddc12d6086b522703c8b80a72ab791680f1a28 Signed-off-by: Kamil Dudka --- lib/url.c | 15 +++++++++------ lib/urldata.h | 1 + 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/url.c b/lib/url.c index 32e7e2e..f87dca4 100644 --- a/lib/url.c +++ b/lib/url.c @@ -1965,15 +1965,17 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option, * The application kindly asks for a differently sized receive buffer. * If it seems reasonable, we'll use it. */ - data->set.buffer_size = va_arg(param, long); + arg = va_arg(param, long); - if(data->set.buffer_size > MAX_BUFSIZE) - data->set.buffer_size = MAX_BUFSIZE; /* huge internal default */ - else if(data->set.buffer_size < 1) - data->set.buffer_size = BUFSIZE; + if(arg > MAX_BUFSIZE) + arg = MAX_BUFSIZE; /* huge internal default */ + else if(arg < 1) + arg = BUFSIZE; + else if(arg < MIN_BUFSIZE) + arg = BUFSIZE; /* Resize only if larger than default buffer size. */ - if(data->set.buffer_size > BUFSIZE) { + if(arg > BUFSIZE) { data->state.buffer = realloc(data->state.buffer, data->set.buffer_size + 1); if(!data->state.buffer) { @@ -1981,6 +1983,7 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option, result = CURLE_OUT_OF_MEMORY; } } + data->set.buffer_size = arg; break; diff --git a/lib/urldata.h b/lib/urldata.h index a7807cf..cd96e8f 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -198,6 +198,7 @@ #define BUFSIZE CURL_MAX_WRITE_SIZE #undef MAX_BUFSIZE #define MAX_BUFSIZE CURL_MAX_READ_SIZE +#define MIN_BUFSIZE 1024 #define CURL_BUFSIZE(x) ((x)?(x):(BUFSIZE)) /* Initial size of the buffer to store headers in, it'll be enlarged in case -- 2.14.3 From 9798012315c087168c5a4a1dc56eacfe82c69626 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Tue, 25 Apr 2017 00:15:28 +0200 Subject: [PATCH 5/7] file: use private buffer for C-L output ... instead of clobbering the download buffer. Upstream-commit: 7c312f84ea930d89c0f0f774b50032c4f9ae30e4 Signed-off-by: Kamil Dudka --- lib/file.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/file.c b/lib/file.c index 1ad4758..b6bf18e 100644 --- a/lib/file.c +++ b/lib/file.c @@ -473,9 +473,10 @@ static CURLcode file_do(struct connectdata *conn, bool *done) date. */ if(data->set.opt_no_body && data->set.include_header && fstated) { CURLcode result; - snprintf(buf, CURL_BUFSIZE(data->set.buffer_size), + char header[80]; + snprintf(header, sizeof(header), "Content-Length: %" FORMAT_OFF_T "\r\n", expected_size); - result = Curl_client_write(conn, CLIENTWRITE_BOTH, buf, 0); + result = Curl_client_write(conn, CLIENTWRITE_BOTH, header, 0); if(result) return result; @@ -493,7 +494,7 @@ static CURLcode file_do(struct connectdata *conn, bool *done) return result; /* format: "Tue, 15 Nov 1994 12:45:26 GMT" */ - snprintf(buf, BUFSIZE-1, + snprintf(header, sizeof(header), "Last-Modified: %s, %02d %s %4d %02d:%02d:%02d GMT\r\n", Curl_wkday[tm->tm_wday?tm->tm_wday-1:6], tm->tm_mday, -- 2.14.3 From f4868e737e9f8d719cb9897506da2c7f92dfd87d Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Tue, 25 Apr 2017 00:16:10 +0200 Subject: [PATCH 6/7] buffer_size: make sure it always has the correct size Removes the need for CURL_BUFSIZE Upstream-commit: f535f4f5fc6cbdce1aec5a3481cec37369dca468 Signed-off-by: Kamil Dudka --- lib/easy.c | 2 +- lib/telnet.c | 2 +- lib/url.c | 2 ++ lib/urldata.h | 1 - 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/easy.c b/lib/easy.c index 5d4d5ae..9cad5f1 100644 --- a/lib/easy.c +++ b/lib/easy.c @@ -564,7 +564,7 @@ CURL *curl_easy_duphandle(CURL *incurl) * the likeliness of us forgetting to init a buffer here in the future. */ outcurl->set.buffer_size = data->set.buffer_size; - outcurl->state.buffer = malloc(CURL_BUFSIZE(outcurl->set.buffer_size) + 1); + outcurl->state.buffer = malloc(outcurl->set.buffer_size + 1); if(!outcurl->state.buffer) goto fail; diff --git a/lib/telnet.c b/lib/telnet.c index 89452dd..e43b423 100644 --- a/lib/telnet.c +++ b/lib/telnet.c @@ -1421,7 +1421,7 @@ static CURLcode telnet_do(struct connectdata *conn, bool *done) /* Keep on listening and act on events */ while(keepon) { - const size_t buf_size = CURL_BUFSIZE(data->set.buffer_size); + const size_t buf_size = (DWORD)data->set.buffer_size; waitret = WaitForMultipleObjects(obj_count, objs, FALSE, wait_timeout); switch(waitret) { case WAIT_TIMEOUT: diff --git a/lib/url.c b/lib/url.c index f87dca4..81de7c2 100644 --- a/lib/url.c +++ b/lib/url.c @@ -577,6 +577,8 @@ CURLcode Curl_init_userdefined(struct UserDefined *set) set->tcp_keepintvl = 60; set->tcp_keepidle = 60; + set->buffer_size = BUFSIZE; + return res; } diff --git a/lib/urldata.h b/lib/urldata.h index cd96e8f..fbe69c2 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -199,7 +199,6 @@ #undef MAX_BUFSIZE #define MAX_BUFSIZE CURL_MAX_READ_SIZE #define MIN_BUFSIZE 1024 -#define CURL_BUFSIZE(x) ((x)?(x):(BUFSIZE)) /* Initial size of the buffer to store headers in, it'll be enlarged in case of need. */ -- 2.14.3 From 9f3810bae5fad685e848a39750863557e17a0163 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Thu, 8 Mar 2018 10:33:16 +0100 Subject: [PATCH 7/7] readwrite: make sure excess reads don't go beyond buffer end CVE-2018-1000122 Bug: https://curl.haxx.se/docs/adv_2018-b047.html Detected by OSS-fuzz Upstream-commit: d52dc4760f6d9ca1937eefa2093058a952465128 Signed-off-by: Kamil Dudka --- lib/transfer.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/transfer.c b/lib/transfer.c index dff6838..7ad6e3c 100644 --- a/lib/transfer.c +++ b/lib/transfer.c @@ -738,10 +738,15 @@ static CURLcode readwrite_data(struct SessionHandle *data, } /* if(! header and data to read ) */ - if(conn->handler->readwrite && - (excess > 0 && !conn->bits.stream_was_rewound)) { + if(conn->handler->readwrite && excess && !conn->bits.stream_was_rewound) { /* Parse the excess data */ k->str += nread; + + if(&k->str[excess] > &k->buf[data->set.buffer_size]) { + /* the excess amount was too excessive(!), make sure + it doesn't read out of buffer */ + excess = &k->buf[data->set.buffer_size] - k->str; + } nread = (ssize_t)excess; result = conn->handler->readwrite(data, conn, &nread, &readmore); -- 2.14.3