From a0b59783a2becd2a059aa4338445a1dd8e197858 Mon Sep 17 00:00:00 2001 From: joe-conigliaro Date: Sat, 10 Aug 2019 18:05:59 +1000 Subject: [PATCH] vlib.http: fix http schannel & follow redirects & cleanup --- thirdparty/vschannel/vschannel.c | 194 +++++++++++++++++-------------- thirdparty/vschannel/vschannel.h | 9 +- vlib/http/backend_nix.v | 13 +-- vlib/http/backend_win.v | 36 ++++-- vlib/http/http.v | 29 +++-- 5 files changed, 164 insertions(+), 117 deletions(-) diff --git a/thirdparty/vschannel/vschannel.c b/thirdparty/vschannel/vschannel.c index 6d532b6bd7..b3d496f422 100644 --- a/thirdparty/vschannel/vschannel.c +++ b/thirdparty/vschannel/vschannel.c @@ -24,6 +24,26 @@ HMODULE g_hsecurity = NULL; // SSPI PSecurityFunctionTable sspi; +struct TlsContext tls_ctx; + +struct TlsContext { + SOCKET Socket; + WSADATA WsaData; + CredHandle hClientCreds; + CtxtHandle hContext; + BOOL fCredsInitialized; + BOOL fContextInitialized; + PCCERT_CONTEXT pRemoteCertContext; +}; + +struct TlsContext new_tls_context() { + return (struct TlsContext) { + .Socket = INVALID_SOCKET, + .fCredsInitialized = FALSE, + .fContextInitialized = FALSE, + .pRemoteCertContext = NULL + }; +}; BOOL load_security_library(void) { INIT_SECURITY_INTERFACE pInitSecurityInterface; @@ -58,79 +78,105 @@ void unload_security_library(void) { g_hsecurity = NULL; } +void vschannel_cleanup() { + // Free the server certificate context. + if(tls_ctx.pRemoteCertContext) { + CertFreeCertificateContext(tls_ctx.pRemoteCertContext); + tls_ctx.pRemoteCertContext = NULL; + } + + // Free SSPI context handle. + if(tls_ctx.fContextInitialized) { + sspi->DeleteSecurityContext(&tls_ctx.hContext); + tls_ctx.fContextInitialized = FALSE; + } + + // Free SSPI credentials handle. + if(tls_ctx.fCredsInitialized) { + sspi->FreeCredentialsHandle(&tls_ctx.hClientCreds); + tls_ctx.fCredsInitialized = FALSE; + } + + // Close socket. + if(tls_ctx.Socket != INVALID_SOCKET) { + closesocket(tls_ctx.Socket); + } + + // Shutdown WinSock subsystem. + WSACleanup(); + + // Close "MY" certificate store. + if(cert_store) { + CertCloseStore(cert_store, 0); + } + + unload_security_library(); +} + +void vschannel_init() { + tls_ctx = new_tls_context(); + + if(!load_security_library()) { + printf("Error initializing the security library\n"); + vschannel_cleanup(); + } + + // Initialize the WinSock subsystem. + if(WSAStartup(0x0101, &tls_ctx.WsaData) == SOCKET_ERROR) { + printf("Error %d returned by WSAStartup\n", GetLastError()); + vschannel_cleanup(); + } + + // Create credentials. + if(create_credentials(&tls_ctx.hClientCreds)) { + printf("Error creating credentials\n"); + vschannel_cleanup(); + } + tls_ctx.fCredsInitialized = TRUE; +} INT request(CHAR *host, CHAR *req, CHAR *out) { - WSADATA WsaData; - SOCKET Socket = INVALID_SOCKET; - - CredHandle hClientCreds; - CtxtHandle hContext; - BOOL fCredsInitialized = FALSE; - BOOL fContextInitialized = FALSE; - SecBuffer ExtraData; SECURITY_STATUS Status; - PCCERT_CONTEXT pRemoteCertContext = NULL; - INT i; INT iOption; PCHAR pszOption; INT resp_length = 0; - // protocol = SP_PROT_PCT1; - // protocol = SP_PROT_SSL2; - // protocol = SP_PROT_SSL3; protocol = SP_PROT_TLS1_2_CLIENT; - - if(!load_security_library()) { - printf("Error initializing the security library\n"); - goto cleanup; - } - - // Initialize the WinSock subsystem. - if(WSAStartup(0x0101, &WsaData) == SOCKET_ERROR) { - printf("Error %d returned by WSAStartup\n", GetLastError()); - goto cleanup; - } - - // Create credentials. - if(create_credentials(&hClientCreds)) { - printf("Error creating credentials\n"); - goto cleanup; - } - fCredsInitialized = TRUE; - // Connect to server. - if(connect_to_server(host, port_number, &Socket)) { + if(connect_to_server(host, port_number, &tls_ctx.Socket)) { printf("Error connecting to server\n"); - goto cleanup; + vschannel_cleanup(); + return resp_length; } - // Perform handshake - if(perform_client_handshake(Socket, &hClientCreds, host, &hContext, &ExtraData)) { + if(perform_client_handshake(tls_ctx.Socket, &tls_ctx.hClientCreds, host, &tls_ctx.hContext, &ExtraData)) { printf("Error performing handshake\n"); - goto cleanup; + vschannel_cleanup(); + return resp_length; } - fContextInitialized = TRUE; + tls_ctx.fContextInitialized = TRUE; // Authenticate server's credentials. // Get server's certificate. - Status = sspi->QueryContextAttributes(&hContext, + Status = sspi->QueryContextAttributes(&tls_ctx.hContext, SECPKG_ATTR_REMOTE_CERT_CONTEXT, - (PVOID)&pRemoteCertContext); + (PVOID)&tls_ctx.pRemoteCertContext); if(Status != SEC_E_OK) { printf("Error 0x%x querying remote certificate\n", Status); - goto cleanup; + vschannel_cleanup(); + return resp_length; } // Attempt to validate server certificate. - Status = verify_server_certificate(pRemoteCertContext, host,0); + Status = verify_server_certificate(tls_ctx.pRemoteCertContext, host,0); if(Status) { // The server certificate did not validate correctly. At this // point, we cannot tell if we are connecting to the correct @@ -140,62 +186,30 @@ INT request(CHAR *host, CHAR *req, CHAR *out) // It is therefore best if we abort the connection. printf("Error 0x%x authenticating server credentials!\n", Status); - // goto cleanup; + vschannel_cleanup(); + return resp_length; } // Free the server certificate context. - CertFreeCertificateContext(pRemoteCertContext); - pRemoteCertContext = NULL; + CertFreeCertificateContext(tls_ctx.pRemoteCertContext); + tls_ctx.pRemoteCertContext = NULL; // Request from server - if(https_make_request(Socket, &hClientCreds, &hContext, req, out, &resp_length)) { - goto cleanup; - } else - + if(https_make_request(tls_ctx.Socket, &tls_ctx.hClientCreds, &tls_ctx.hContext, req, out, &resp_length)) { + vschannel_cleanup(); + return resp_length; + } + + // Send a close_notify alert to the server and // close down the connection. - if(disconnect_from_server(Socket, &hClientCreds, &hContext)) { + if(disconnect_from_server(tls_ctx.Socket, &tls_ctx.hClientCreds, &tls_ctx.hContext)) { printf("Error disconnecting from server\n"); - goto cleanup; + vschannel_cleanup(); + return resp_length; } - fContextInitialized = FALSE; - Socket = INVALID_SOCKET; - - -cleanup: - - // Free the server certificate context. - if(pRemoteCertContext) { - CertFreeCertificateContext(pRemoteCertContext); - pRemoteCertContext = NULL; - } - - // Free SSPI context handle. - if(fContextInitialized) { - sspi->DeleteSecurityContext(&hContext); - fContextInitialized = FALSE; - } - - // Free SSPI credentials handle. - if(fCredsInitialized) { - sspi->FreeCredentialsHandle(&hClientCreds); - fCredsInitialized = FALSE; - } - - // Close socket. - if(Socket != INVALID_SOCKET) { - closesocket(Socket); - } - - // Shutdown WinSock subsystem. - WSACleanup(); - - // Close "MY" certificate store. - if(cert_store) { - CertCloseStore(cert_store, 0); - } - - unload_security_library(); + tls_ctx.fContextInitialized = FALSE; + tls_ctx.Socket = INVALID_SOCKET; return resp_length; } @@ -936,8 +950,8 @@ static SECURITY_STATUS https_make_request(SOCKET Socket, PCredHandle phCreds, Ct } // Copy the decrypted data to our output buffer - memcpy(out, pDataBuffer->pvBuffer, (int)pDataBuffer->cbBuffer); *length += (int)pDataBuffer->cbBuffer; + memcpy(out, pDataBuffer->pvBuffer, (int)pDataBuffer->cbBuffer); out += (int)pDataBuffer->cbBuffer; // Move any "extra" data to the input buffer. diff --git a/thirdparty/vschannel/vschannel.h b/thirdparty/vschannel/vschannel.h index 7dbb8dc2cd..3faf2bae0c 100644 --- a/thirdparty/vschannel/vschannel.h +++ b/thirdparty/vschannel/vschannel.h @@ -19,7 +19,13 @@ #define SP_PROT_TLS1_2_CLIENT 0x00000800 -INT request(CHAR *host, CHAR *req, CHAR *out); +static struct TlsContext new_tls_context(); + +static void vschannel_init(); + +static void vschannel_cleanup(); + +static INT request(CHAR *host, CHAR *req, CHAR *out); static SECURITY_STATUS create_credentials(PCredHandle phCreds); @@ -36,7 +42,6 @@ static SECURITY_STATUS client_handshake_loop( static SECURITY_STATUS https_make_request( SOCKET Socket, PCredHandle phCreds, CtxtHandle *phContext, CHAR *req, CHAR *out, int *length); - // CtxtHandle *phContext, CHAR *path); static DWORD verify_server_certificate( PCCERT_CONTEXT pServerCert, PSTR host, DWORD dwCertFlags); diff --git a/vlib/http/backend_nix.v b/vlib/http/backend_nix.v index 9e3a834d3b..e951ae48e8 100644 --- a/vlib/http/backend_nix.v +++ b/vlib/http/backend_nix.v @@ -31,7 +31,7 @@ fn init_module() { //C.OPENSSL_config(0) } -fn ssl_do(method, host_name, path string) string { +fn ssl_do(method, host_name, path string) Response { //ssl_method := C.SSLv23_method() ssl_method := C.TLSv1_2_method() if isnil(method) { @@ -66,10 +66,8 @@ fn ssl_do(method, host_name, path string) string { res = C.BIO_do_handshake(web) cert := C.SSL_get_peer_certificate(ssl) res = C.SSL_get_verify_result(ssl) - /////// - s := '$method $path HTTP/1.1\r\n' + - 'Host: $host_name\r\n' + - 'Connection: close\r\n\r\n' + /////// + s := build_request_headers('', method, host_name, path) C.BIO_puts(web, s.str) C.BIO_puts(out, '\n') mut sb := strings.new_builder(100) @@ -91,6 +89,7 @@ fn ssl_do(method, host_name, path string) string { } if !isnil(ctx) { C.SSL_CTX_free(ctx) - } - return sb.str() + } + + return parse_response(sb.str() ) } diff --git a/vlib/http/backend_win.v b/vlib/http/backend_win.v index 5f8afcd9f5..5ec18340e6 100644 --- a/vlib/http/backend_win.v +++ b/vlib/http/backend_win.v @@ -4,28 +4,44 @@ module http -import strings +import strings +import net.urllib #flag windows -I @VROOT/thirdparty/vschannel -#flag -lws2_32 -lcrypt32 +#flag -l ws2_32 +#flag -l crypt32 #include "vschannel.c" +const ( + max_redirects = 4 +) fn init_module() {} -fn ssl_do(method, host_name, path string) string { - mut buff := malloc(10000) +fn ssl_do(method, host_name, path string) Response { + C.vschannel_init() + // TODO: joe-c + // dynamically increase in vschannel.c if needed + mut buff := malloc(44000) - req := '$method $path HTTP/1.0\r\nUser-Agent: v\r\nAccept:*/*\r\n\r\n' - length := int(C.request(host_name.str, req.str, buff)) + mut p := if path == '' { '/' } else { path } + mut req := build_request_headers('', method, host_name, p) + mut length := int(C.request(host_name.str, req.str, buff)) + mut resp := parse_response(string(buff, length)) - if length == 0 { - return '' + mut no_redirects := 0 + for resp.status_code == 301 && no_redirects <= max_redirects { + u := urllib.parse(resp.headers['Location']) or { break } + p = if u.path == '' { '/' } else { u.path } + req = build_request_headers('', method, u.hostname(), p) + length = int(C.request(u.hostname().str, req.str, buff)) + resp = parse_response(string(buff, length)) + no_redirects++ } - resp := tos(buff, length) - + free(buff) + C.vschannel_cleanup() return resp } diff --git a/vlib/http/http.v b/vlib/http/http.v index 3431caeaed..0424b2465c 100644 --- a/vlib/http/http.v +++ b/vlib/http/http.v @@ -93,7 +93,6 @@ pub fn (req mut Request) add_header(key, val string) { } pub fn (req &Request) do() Response { - mut headers := map[string]string{} if req.typ == 'POST' { // req.headers << 'Content-Type: application/x-www-form-urlencoded' } @@ -108,9 +107,13 @@ pub fn (req &Request) do() Response { if !is_ssl { panic('non https requests are not supported right now') } - s := ssl_do(req.typ, url.host, url.path) - // s := ssl_do(req.typ, url.host, url.path) - first_header := s.all_before('\n') + + return ssl_do(req.typ, url.hostname(), url.path) +} + +fn parse_response(resp string) Response { + mut headers := map[string]string{} + first_header := resp.all_before('\n') mut status_code := 0 if first_header.contains('HTTP/') { val := first_header.find_between(' ', ' ') @@ -122,14 +125,14 @@ pub fn (req &Request) do() Response { mut i := 1 for { old_pos := nl_pos - nl_pos = s.index_after('\n', nl_pos+1) + nl_pos = resp.index_after('\n', nl_pos+1) if nl_pos == -1 { break } - h := s.substr(old_pos + 1, nl_pos) + h := resp.substr(old_pos + 1, nl_pos) // End of headers if h.len <= 1 { - text = s.right(nl_pos + 1) + text = resp.right(nl_pos + 1) break } i++ @@ -144,14 +147,24 @@ pub fn (req &Request) do() Response { val := h.right(pos + 2) headers[key] = val.trim_space() } + if headers['Transfer-Encoding'] == 'chunked' { text = chunked.decode( text ) } + return Response { status_code: status_code headers: headers text: text - } + } +} + +fn build_request_headers(user_agent, method, host_name, path string) string { + ua := if user_agent == '' { 'v' } else { user_agent } + return '$method $path HTTP/1.1\r\n' + + 'Host: $host_name\r\n' + + 'User-Agent: $ua\r\n' + + 'Connection: close\r\n\r\n' } pub fn unescape_url(s string) string {