Browse Source
Move shared HTTP request functionality out of http-fetch and http-push, and replace the two fwrite_buffer/fwrite_buffer_dynamic functions with one fwrite_buffer function that does dynamic buffering. Use slot callbacks to process responses to fetch object transfer requests and push transfer requests, and put all of http-push into an #ifdef check for curl multi support. Signed-off-by: Nick Hengeveld <nickh@reactrix.com> Signed-off-by: Junio C Hamano <junkio@cox.net>maint
Nick Hengeveld
19 years ago
committed by
Junio C Hamano
5 changed files with 635 additions and 976 deletions
@ -0,0 +1,443 @@
@@ -0,0 +1,443 @@
|
||||
#include "http.h" |
||||
|
||||
int data_received; |
||||
int active_requests = 0; |
||||
|
||||
#ifdef USE_CURL_MULTI |
||||
int max_requests = -1; |
||||
CURLM *curlm; |
||||
#endif |
||||
#ifndef NO_CURL_EASY_DUPHANDLE |
||||
CURL *curl_default; |
||||
#endif |
||||
char curl_errorstr[CURL_ERROR_SIZE]; |
||||
|
||||
int curl_ssl_verify = -1; |
||||
char *ssl_cert = NULL; |
||||
#if LIBCURL_VERSION_NUM >= 0x070902 |
||||
char *ssl_key = NULL; |
||||
#endif |
||||
#if LIBCURL_VERSION_NUM >= 0x070908 |
||||
char *ssl_capath = NULL; |
||||
#endif |
||||
char *ssl_cainfo = NULL; |
||||
long curl_low_speed_limit = -1; |
||||
long curl_low_speed_time = -1; |
||||
|
||||
struct curl_slist *pragma_header; |
||||
struct curl_slist *no_range_header; |
||||
|
||||
struct active_request_slot *active_queue_head = NULL; |
||||
|
||||
size_t fread_buffer(void *ptr, size_t eltsize, size_t nmemb, |
||||
struct buffer *buffer) |
||||
{ |
||||
size_t size = eltsize * nmemb; |
||||
if (size > buffer->size - buffer->posn) |
||||
size = buffer->size - buffer->posn; |
||||
memcpy(ptr, buffer->buffer + buffer->posn, size); |
||||
buffer->posn += size; |
||||
return size; |
||||
} |
||||
|
||||
size_t fwrite_buffer(const void *ptr, size_t eltsize, |
||||
size_t nmemb, struct buffer *buffer) |
||||
{ |
||||
size_t size = eltsize * nmemb; |
||||
if (size > buffer->size - buffer->posn) { |
||||
buffer->size = buffer->size * 3 / 2; |
||||
if (buffer->size < buffer->posn + size) |
||||
buffer->size = buffer->posn + size; |
||||
buffer->buffer = xrealloc(buffer->buffer, buffer->size); |
||||
} |
||||
memcpy(buffer->buffer + buffer->posn, ptr, size); |
||||
buffer->posn += size; |
||||
data_received++; |
||||
return size; |
||||
} |
||||
|
||||
size_t fwrite_null(const void *ptr, size_t eltsize, |
||||
size_t nmemb, struct buffer *buffer) |
||||
{ |
||||
data_received++; |
||||
return eltsize * nmemb; |
||||
} |
||||
|
||||
static void finish_active_slot(struct active_request_slot *slot); |
||||
|
||||
#ifdef USE_CURL_MULTI |
||||
static void process_curl_messages(void) |
||||
{ |
||||
int num_messages; |
||||
struct active_request_slot *slot; |
||||
CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages); |
||||
|
||||
while (curl_message != NULL) { |
||||
if (curl_message->msg == CURLMSG_DONE) { |
||||
int curl_result = curl_message->data.result; |
||||
slot = active_queue_head; |
||||
while (slot != NULL && |
||||
slot->curl != curl_message->easy_handle) |
||||
slot = slot->next; |
||||
if (slot != NULL) { |
||||
curl_multi_remove_handle(curlm, slot->curl); |
||||
slot->curl_result = curl_result; |
||||
finish_active_slot(slot); |
||||
} else { |
||||
fprintf(stderr, "Received DONE message for unknown request!\n"); |
||||
} |
||||
} else { |
||||
fprintf(stderr, "Unknown CURL message received: %d\n", |
||||
(int)curl_message->msg); |
||||
} |
||||
curl_message = curl_multi_info_read(curlm, &num_messages); |
||||
} |
||||
} |
||||
#endif |
||||
|
||||
static int http_options(const char *var, const char *value) |
||||
{ |
||||
if (!strcmp("http.sslverify", var)) { |
||||
if (curl_ssl_verify == -1) { |
||||
curl_ssl_verify = git_config_bool(var, value); |
||||
} |
||||
return 0; |
||||
} |
||||
|
||||
if (!strcmp("http.sslcert", var)) { |
||||
if (ssl_cert == NULL) { |
||||
ssl_cert = xmalloc(strlen(value)+1); |
||||
strcpy(ssl_cert, value); |
||||
} |
||||
return 0; |
||||
} |
||||
#if LIBCURL_VERSION_NUM >= 0x070902 |
||||
if (!strcmp("http.sslkey", var)) { |
||||
if (ssl_key == NULL) { |
||||
ssl_key = xmalloc(strlen(value)+1); |
||||
strcpy(ssl_key, value); |
||||
} |
||||
return 0; |
||||
} |
||||
#endif |
||||
#if LIBCURL_VERSION_NUM >= 0x070908 |
||||
if (!strcmp("http.sslcapath", var)) { |
||||
if (ssl_capath == NULL) { |
||||
ssl_capath = xmalloc(strlen(value)+1); |
||||
strcpy(ssl_capath, value); |
||||
} |
||||
return 0; |
||||
} |
||||
#endif |
||||
if (!strcmp("http.sslcainfo", var)) { |
||||
if (ssl_cainfo == NULL) { |
||||
ssl_cainfo = xmalloc(strlen(value)+1); |
||||
strcpy(ssl_cainfo, value); |
||||
} |
||||
return 0; |
||||
} |
||||
|
||||
#ifdef USE_CURL_MULTI |
||||
if (!strcmp("http.maxrequests", var)) { |
||||
if (max_requests == -1) |
||||
max_requests = git_config_int(var, value); |
||||
return 0; |
||||
} |
||||
#endif |
||||
|
||||
if (!strcmp("http.lowspeedlimit", var)) { |
||||
if (curl_low_speed_limit == -1) |
||||
curl_low_speed_limit = (long)git_config_int(var, value); |
||||
return 0; |
||||
} |
||||
if (!strcmp("http.lowspeedtime", var)) { |
||||
if (curl_low_speed_time == -1) |
||||
curl_low_speed_time = (long)git_config_int(var, value); |
||||
return 0; |
||||
} |
||||
|
||||
/* Fall back on the default ones */ |
||||
return git_default_config(var, value); |
||||
} |
||||
|
||||
void http_init(void) |
||||
{ |
||||
char *low_speed_limit; |
||||
char *low_speed_time; |
||||
|
||||
curl_global_init(CURL_GLOBAL_ALL); |
||||
|
||||
pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache"); |
||||
no_range_header = curl_slist_append(no_range_header, "Range:"); |
||||
|
||||
#ifdef USE_CURL_MULTI |
||||
{ |
||||
char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS"); |
||||
if (http_max_requests != NULL) |
||||
max_requests = atoi(http_max_requests); |
||||
} |
||||
|
||||
curlm = curl_multi_init(); |
||||
if (curlm == NULL) { |
||||
fprintf(stderr, "Error creating curl multi handle.\n"); |
||||
exit(1); |
||||
} |
||||
#endif |
||||
|
||||
if (getenv("GIT_SSL_NO_VERIFY")) |
||||
curl_ssl_verify = 0; |
||||
|
||||
ssl_cert = getenv("GIT_SSL_CERT"); |
||||
#if LIBCURL_VERSION_NUM >= 0x070902 |
||||
ssl_key = getenv("GIT_SSL_KEY"); |
||||
#endif |
||||
#if LIBCURL_VERSION_NUM >= 0x070908 |
||||
ssl_capath = getenv("GIT_SSL_CAPATH"); |
||||
#endif |
||||
ssl_cainfo = getenv("GIT_SSL_CAINFO"); |
||||
|
||||
low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT"); |
||||
if (low_speed_limit != NULL) |
||||
curl_low_speed_limit = strtol(low_speed_limit, NULL, 10); |
||||
low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME"); |
||||
if (low_speed_time != NULL) |
||||
curl_low_speed_time = strtol(low_speed_time, NULL, 10); |
||||
|
||||
git_config(http_options); |
||||
|
||||
if (curl_ssl_verify == -1) |
||||
curl_ssl_verify = 1; |
||||
|
||||
#ifdef USE_CURL_MULTI |
||||
if (max_requests < 1) |
||||
max_requests = DEFAULT_MAX_REQUESTS; |
||||
#endif |
||||
|
||||
#ifndef NO_CURL_EASY_DUPHANDLE |
||||
curl_default = get_curl_handle(); |
||||
#endif |
||||
} |
||||
|
||||
void http_cleanup(void) |
||||
{ |
||||
struct active_request_slot *slot = active_queue_head; |
||||
#ifdef USE_CURL_MULTI |
||||
char *wait_url; |
||||
CURLMcode curlm_result; |
||||
#endif |
||||
|
||||
while (slot != NULL) { |
||||
#ifdef USE_CURL_MULTI |
||||
if (slot->in_use) { |
||||
curl_easy_getinfo(slot->curl, |
||||
CURLINFO_EFFECTIVE_URL, |
||||
&wait_url); |
||||
fprintf(stderr, "Waiting for %s\n", wait_url); |
||||
run_active_slot(slot); |
||||
} |
||||
#endif |
||||
if (slot->curl != NULL) |
||||
curl_easy_cleanup(slot->curl); |
||||
slot = slot->next; |
||||
} |
||||
|
||||
#ifndef NO_CURL_EASY_DUPHANDLE |
||||
curl_easy_cleanup(curl_default); |
||||
#endif |
||||
|
||||
#ifdef USE_CURL_MULTI |
||||
curl_multi_cleanup(curlm); |
||||
#endif |
||||
curl_global_cleanup(); |
||||
|
||||
} |
||||
|
||||
static CURL* get_curl_handle(void) |
||||
{ |
||||
CURL* result = curl_easy_init(); |
||||
|
||||
curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify); |
||||
#if LIBCURL_VERSION_NUM >= 0x070907 |
||||
curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); |
||||
#endif |
||||
|
||||
if (ssl_cert != NULL) |
||||
curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert); |
||||
#if LIBCURL_VERSION_NUM >= 0x070902 |
||||
if (ssl_key != NULL) |
||||
curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key); |
||||
#endif |
||||
#if LIBCURL_VERSION_NUM >= 0x070908 |
||||
if (ssl_capath != NULL) |
||||
curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath); |
||||
#endif |
||||
if (ssl_cainfo != NULL) |
||||
curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo); |
||||
curl_easy_setopt(result, CURLOPT_FAILONERROR, 1); |
||||
|
||||
if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) { |
||||
curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT, |
||||
curl_low_speed_limit); |
||||
curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME, |
||||
curl_low_speed_time); |
||||
} |
||||
|
||||
curl_easy_setopt(result, CURLOPT_FOLLOWLOCATION, 1); |
||||
|
||||
return result; |
||||
} |
||||
|
||||
struct active_request_slot *get_active_slot(void) |
||||
{ |
||||
struct active_request_slot *slot = active_queue_head; |
||||
struct active_request_slot *newslot; |
||||
|
||||
#ifdef USE_CURL_MULTI |
||||
int num_transfers; |
||||
|
||||
/* Wait for a slot to open up if the queue is full */ |
||||
while (active_requests >= max_requests) { |
||||
curl_multi_perform(curlm, &num_transfers); |
||||
if (num_transfers < active_requests) { |
||||
process_curl_messages(); |
||||
} |
||||
} |
||||
#endif |
||||
|
||||
while (slot != NULL && slot->in_use) { |
||||
slot = slot->next; |
||||
} |
||||
if (slot == NULL) { |
||||
newslot = xmalloc(sizeof(*newslot)); |
||||
newslot->curl = NULL; |
||||
newslot->in_use = 0; |
||||
newslot->next = NULL; |
||||
|
||||
slot = active_queue_head; |
||||
if (slot == NULL) { |
||||
active_queue_head = newslot; |
||||
} else { |
||||
while (slot->next != NULL) { |
||||
slot = slot->next; |
||||
} |
||||
slot->next = newslot; |
||||
} |
||||
slot = newslot; |
||||
} |
||||
|
||||
if (slot->curl == NULL) { |
||||
#ifdef NO_CURL_EASY_DUPHANDLE |
||||
slot->curl = get_curl_handle(); |
||||
#else |
||||
slot->curl = curl_easy_duphandle(curl_default); |
||||
#endif |
||||
} |
||||
|
||||
active_requests++; |
||||
slot->in_use = 1; |
||||
slot->local = NULL; |
||||
slot->callback_data = NULL; |
||||
slot->callback_func = NULL; |
||||
curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header); |
||||
curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_range_header); |
||||
curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr); |
||||
|
||||
return slot; |
||||
} |
||||
|
||||
int start_active_slot(struct active_request_slot *slot) |
||||
{ |
||||
#ifdef USE_CURL_MULTI |
||||
CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl); |
||||
|
||||
if (curlm_result != CURLM_OK && |
||||
curlm_result != CURLM_CALL_MULTI_PERFORM) { |
||||
active_requests--; |
||||
slot->in_use = 0; |
||||
return 0; |
||||
} |
||||
#endif |
||||
return 1; |
||||
} |
||||
|
||||
#ifdef USE_CURL_MULTI |
||||
void step_active_slots(void) |
||||
{ |
||||
int num_transfers; |
||||
CURLMcode curlm_result; |
||||
|
||||
do { |
||||
curlm_result = curl_multi_perform(curlm, &num_transfers); |
||||
} while (curlm_result == CURLM_CALL_MULTI_PERFORM); |
||||
if (num_transfers < active_requests) { |
||||
process_curl_messages(); |
||||
fill_active_slots(); |
||||
} |
||||
} |
||||
#endif |
||||
|
||||
void run_active_slot(struct active_request_slot *slot) |
||||
{ |
||||
#ifdef USE_CURL_MULTI |
||||
long last_pos = 0; |
||||
long current_pos; |
||||
fd_set readfds; |
||||
fd_set writefds; |
||||
fd_set excfds; |
||||
int max_fd; |
||||
struct timeval select_timeout; |
||||
|
||||
while (slot->in_use) { |
||||
data_received = 0; |
||||
step_active_slots(); |
||||
|
||||
if (!data_received && slot->local != NULL) { |
||||
current_pos = ftell(slot->local); |
||||
if (current_pos > last_pos) |
||||
data_received++; |
||||
last_pos = current_pos; |
||||
} |
||||
|
||||
if (slot->in_use && !data_received) { |
||||
max_fd = 0; |
||||
FD_ZERO(&readfds); |
||||
FD_ZERO(&writefds); |
||||
FD_ZERO(&excfds); |
||||
select_timeout.tv_sec = 0; |
||||
select_timeout.tv_usec = 50000; |
||||
select(max_fd, &readfds, &writefds, |
||||
&excfds, &select_timeout); |
||||
} |
||||
} |
||||
#else |
||||
while (slot->in_use) { |
||||
slot->curl_result = curl_easy_perform(slot->curl); |
||||
finish_active_slot(slot); |
||||
} |
||||
#endif |
||||
} |
||||
|
||||
static void finish_active_slot(struct active_request_slot *slot) |
||||
{ |
||||
active_requests--; |
||||
slot->in_use = 0; |
||||
curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code); |
||||
|
||||
/* Run callback if appropriate */ |
||||
if (slot->callback_func != NULL) { |
||||
slot->callback_func(slot->callback_data); |
||||
} |
||||
} |
||||
|
||||
void finish_all_active_slots(void) |
||||
{ |
||||
struct active_request_slot *slot = active_queue_head; |
||||
|
||||
while (slot != NULL) |
||||
if (slot->in_use) { |
||||
run_active_slot(slot); |
||||
slot = active_queue_head; |
||||
} else { |
||||
slot = slot->next; |
||||
} |
||||
} |
||||
@ -0,0 +1,95 @@
@@ -0,0 +1,95 @@
|
||||
#ifndef HTTP_H |
||||
#define HTTP_H |
||||
|
||||
#include "cache.h" |
||||
|
||||
#include <curl/curl.h> |
||||
#include <curl/easy.h> |
||||
|
||||
#if LIBCURL_VERSION_NUM >= 0x070908 |
||||
#define USE_CURL_MULTI |
||||
#define DEFAULT_MAX_REQUESTS 5 |
||||
#endif |
||||
|
||||
#if LIBCURL_VERSION_NUM < 0x070704 |
||||
#define curl_global_cleanup() do { /* nothing */ } while(0) |
||||
#endif |
||||
#if LIBCURL_VERSION_NUM < 0x070800 |
||||
#define curl_global_init(a) do { /* nothing */ } while(0) |
||||
#endif |
||||
|
||||
#if LIBCURL_VERSION_NUM < 0x070c04 |
||||
#define NO_CURL_EASY_DUPHANDLE |
||||
#endif |
||||
|
||||
struct active_request_slot |
||||
{ |
||||
CURL *curl; |
||||
FILE *local; |
||||
int in_use; |
||||
CURLcode curl_result; |
||||
long http_code; |
||||
void *callback_data; |
||||
void (*callback_func)(void *data); |
||||
struct active_request_slot *next; |
||||
}; |
||||
|
||||
struct buffer |
||||
{ |
||||
size_t posn; |
||||
size_t size; |
||||
void *buffer; |
||||
}; |
||||
|
||||
/* Curl request read/write callbacks */ |
||||
extern size_t fread_buffer(void *ptr, size_t eltsize, size_t nmemb, |
||||
struct buffer *buffer); |
||||
extern size_t fwrite_buffer(const void *ptr, size_t eltsize, |
||||
size_t nmemb, struct buffer *buffer); |
||||
extern size_t fwrite_null(const void *ptr, size_t eltsize, |
||||
size_t nmemb, struct buffer *buffer); |
||||
|
||||
/* Slot lifecycle functions */ |
||||
extern struct active_request_slot *get_active_slot(void); |
||||
extern int start_active_slot(struct active_request_slot *slot); |
||||
extern void run_active_slot(struct active_request_slot *slot); |
||||
extern void finish_all_active_slots(void); |
||||
|
||||
#ifdef USE_CURL_MULTI |
||||
extern void fill_active_slots(void); |
||||
extern void step_active_slots(void); |
||||
#endif |
||||
|
||||
extern void http_init(void); |
||||
extern void http_cleanup(void); |
||||
|
||||
extern int data_received; |
||||
extern int active_requests; |
||||
|
||||
#ifdef USE_CURL_MULTI |
||||
extern int max_requests; |
||||
extern CURLM *curlm; |
||||
#endif |
||||
#ifndef NO_CURL_EASY_DUPHANDLE |
||||
extern CURL *curl_default; |
||||
#endif |
||||
extern char curl_errorstr[CURL_ERROR_SIZE]; |
||||
|
||||
extern int curl_ssl_verify; |
||||
extern char *ssl_cert; |
||||
#if LIBCURL_VERSION_NUM >= 0x070902 |
||||
extern char *ssl_key; |
||||
#endif |
||||
#if LIBCURL_VERSION_NUM >= 0x070908 |
||||
extern char *ssl_capath; |
||||
#endif |
||||
extern char *ssl_cainfo; |
||||
extern long curl_low_speed_limit; |
||||
extern long curl_low_speed_time; |
||||
|
||||
extern struct curl_slist *pragma_header; |
||||
extern struct curl_slist *no_range_header; |
||||
|
||||
extern struct active_request_slot *active_queue_head; |
||||
|
||||
#endif /* HTTP_H */ |
||||
Loading…
Reference in new issue