vlib.http: fix http schannel & follow redirects & cleanup

thirdparty/vschannel/vschannel.c

@@ -24,6 +24,26 @@ HMODULE g_hsecurity = NULL;
 // SSPI
 PSecurityFunctionTable sspi;
 
+struct TlsContext tls_ctx;
+
+struct TlsContext {
+	SOCKET  Socket;
+	WSADATA WsaData;
+	CredHandle hClientCreds;
+	CtxtHandle hContext;
+	BOOL fCredsInitialized;
+	BOOL fContextInitialized;
+	PCCERT_CONTEXT pRemoteCertContext;
+};
+
+struct TlsContext new_tls_context() {
+	return (struct TlsContext) {
+		.Socket = INVALID_SOCKET,
+		.fCredsInitialized = FALSE,
+		.fContextInitialized = FALSE,
+		.pRemoteCertContext = NULL
+	};
+};
 
 BOOL load_security_library(void) {
 	INIT_SECURITY_INTERFACE pInitSecurityInterface;
@@ -58,79 +78,105 @@ void unload_security_library(void) {
 	g_hsecurity = NULL;
 }
 
+void vschannel_cleanup() {
+	// Free the server certificate context.
+	if(tls_ctx.pRemoteCertContext) {
+		CertFreeCertificateContext(tls_ctx.pRemoteCertContext);
+		tls_ctx.pRemoteCertContext = NULL;
+	}
+
+	// Free SSPI context handle.
+	if(tls_ctx.fContextInitialized) {
+		sspi->DeleteSecurityContext(&tls_ctx.hContext);
+		tls_ctx.fContextInitialized = FALSE;
+	}
+
+	// Free SSPI credentials handle.
+	if(tls_ctx.fCredsInitialized) {
+		sspi->FreeCredentialsHandle(&tls_ctx.hClientCreds);
+		tls_ctx.fCredsInitialized = FALSE;
+	}
+	
+	// Close socket.
+	if(tls_ctx.Socket != INVALID_SOCKET) {
+		closesocket(tls_ctx.Socket);
+	}
+
+	// Shutdown WinSock subsystem.
+	WSACleanup();
+	
+	// Close "MY" certificate store.
+	if(cert_store) {
+		CertCloseStore(cert_store, 0);
+	}
+
+	unload_security_library();
+}
+
+void vschannel_init() {
+	tls_ctx = new_tls_context();
+
+	if(!load_security_library()) {
+		printf("Error initializing the security library\n");
+		vschannel_cleanup();
+	}
+
+	// Initialize the WinSock subsystem.
+	if(WSAStartup(0x0101, &tls_ctx.WsaData) == SOCKET_ERROR) {
+		printf("Error %d returned by WSAStartup\n", GetLastError());
+		vschannel_cleanup();
+	}
+
+	// Create credentials.
+	if(create_credentials(&tls_ctx.hClientCreds)) {
+		printf("Error creating credentials\n");
+		vschannel_cleanup();
+	}
+	tls_ctx.fCredsInitialized = TRUE;
+}
 
 INT request(CHAR *host, CHAR *req, CHAR *out)
 {
-	WSADATA WsaData;
-	SOCKET  Socket = INVALID_SOCKET;
-
-	CredHandle hClientCreds;
-	CtxtHandle hContext;
-	BOOL fCredsInitialized = FALSE;
-	BOOL fContextInitialized = FALSE;
-
 	SecBuffer  ExtraData;
 	SECURITY_STATUS Status;
 
-	PCCERT_CONTEXT pRemoteCertContext = NULL;
-
 	INT i;
 	INT iOption;
 	PCHAR pszOption;
 
 	INT resp_length = 0;
 
-	// protocol = SP_PROT_PCT1;
-	// protocol = SP_PROT_SSL2;
-	// protocol = SP_PROT_SSL3;
 	protocol = SP_PROT_TLS1_2_CLIENT;
 
-
-	if(!load_security_library()) {
-		printf("Error initializing the security library\n");
-		goto cleanup;
-	}
-
-	// Initialize the WinSock subsystem.
-	if(WSAStartup(0x0101, &WsaData) == SOCKET_ERROR) {
-		printf("Error %d returned by WSAStartup\n", GetLastError());
-		goto cleanup;
-	}
-
-	// Create credentials.
-	if(create_credentials(&hClientCreds)) {
-		printf("Error creating credentials\n");
-		goto cleanup;
-	}
-	fCredsInitialized = TRUE;
-
 	// Connect to server.
-	if(connect_to_server(host, port_number, &Socket)) {
+	if(connect_to_server(host, port_number, &tls_ctx.Socket)) {
 		printf("Error connecting to server\n");
-		goto cleanup;
+		vschannel_cleanup();
+		return resp_length;
 	}
 
-
 	// Perform handshake
-	if(perform_client_handshake(Socket, &hClientCreds, host, &hContext, &ExtraData)) {
+	if(perform_client_handshake(tls_ctx.Socket, &tls_ctx.hClientCreds, host, &tls_ctx.hContext, &ExtraData)) {
 		printf("Error performing handshake\n");
-		goto cleanup;
+		vschannel_cleanup();
+		return resp_length;
 	}
-	fContextInitialized = TRUE;
+	tls_ctx.fContextInitialized = TRUE;
 
 	// Authenticate server's credentials.
 
 	// Get server's certificate.
-	Status = sspi->QueryContextAttributes(&hContext,
+	Status = sspi->QueryContextAttributes(&tls_ctx.hContext,
 											 SECPKG_ATTR_REMOTE_CERT_CONTEXT,
-											 (PVOID)&pRemoteCertContext);
+											 (PVOID)&tls_ctx.pRemoteCertContext);
 	if(Status != SEC_E_OK) {
 		printf("Error 0x%x querying remote certificate\n", Status);
-		goto cleanup;
+		vschannel_cleanup();
+		return resp_length;
 	}
 
 	// Attempt to validate server certificate.
-	Status = verify_server_certificate(pRemoteCertContext, host,0);
+	Status = verify_server_certificate(tls_ctx.pRemoteCertContext, host,0);
 	if(Status) {
 		// The server certificate did not validate correctly. At this
 		// point, we cannot tell if we are connecting to the correct 
@@ -140,62 +186,30 @@ INT request(CHAR *host, CHAR *req, CHAR *out)
 		// It is therefore best if we abort the connection.
 
 		printf("Error 0x%x authenticating server credentials!\n", Status);
-		// goto cleanup;
+		vschannel_cleanup();
+		return resp_length;
 	}
 
 	// Free the server certificate context.
-	CertFreeCertificateContext(pRemoteCertContext);
-	pRemoteCertContext = NULL;
+	CertFreeCertificateContext(tls_ctx.pRemoteCertContext);
+	tls_ctx.pRemoteCertContext = NULL;
 
 	// Request from server
-	if(https_make_request(Socket, &hClientCreds, &hContext,  req, out, &resp_length)) {
-		goto cleanup;
-	} else 
-
+	if(https_make_request(tls_ctx.Socket, &tls_ctx.hClientCreds, &tls_ctx.hContext,  req, out, &resp_length)) {
+		vschannel_cleanup();
+		return resp_length;
+	}
+	
+	
 	// Send a close_notify alert to the server and
 	// close down the connection.
-	if(disconnect_from_server(Socket, &hClientCreds, &hContext)) {
+	if(disconnect_from_server(tls_ctx.Socket, &tls_ctx.hClientCreds, &tls_ctx.hContext)) {
 		printf("Error disconnecting from server\n");
-		goto cleanup;
+		vschannel_cleanup();
+		return resp_length;
 	}
-	fContextInitialized = FALSE;
-	Socket = INVALID_SOCKET;
-
-
-cleanup:
-
-	// Free the server certificate context.
-	if(pRemoteCertContext) {
-		CertFreeCertificateContext(pRemoteCertContext);
-		pRemoteCertContext = NULL;
-	}
-
-	// Free SSPI context handle.
-	if(fContextInitialized) {
-		sspi->DeleteSecurityContext(&hContext);
-		fContextInitialized = FALSE;
-	}
-
-	// Free SSPI credentials handle.
-	if(fCredsInitialized) {
-		sspi->FreeCredentialsHandle(&hClientCreds);
-		fCredsInitialized = FALSE;
-	}
-
-	// Close socket.
-	if(Socket != INVALID_SOCKET) {
-		closesocket(Socket);
-	}
-
-	// Shutdown WinSock subsystem.
-	WSACleanup();
-
-	// Close "MY" certificate store.
-	if(cert_store) {
-		CertCloseStore(cert_store, 0);
-	}
-
-	unload_security_library();
+	tls_ctx.fContextInitialized = FALSE;
+	tls_ctx.Socket = INVALID_SOCKET;
 
 	return resp_length;
 }
@@ -936,8 +950,8 @@ static SECURITY_STATUS https_make_request(SOCKET Socket, PCredHandle phCreds, Ct
 		}
 
 		// Copy the decrypted data to our output buffer
-		memcpy(out, pDataBuffer->pvBuffer, (int)pDataBuffer->cbBuffer);
 		*length += (int)pDataBuffer->cbBuffer;
+		memcpy(out, pDataBuffer->pvBuffer, (int)pDataBuffer->cbBuffer);
 		out += (int)pDataBuffer->cbBuffer;
 		
 		// Move any "extra" data to the input buffer.

thirdparty/vschannel/vschannel.h

@@ -19,7 +19,13 @@
 #define SP_PROT_TLS1_2_CLIENT 0x00000800
 
 
-INT request(CHAR *host, CHAR *req, CHAR *out);
+static struct TlsContext new_tls_context();
+
+static void vschannel_init();
+
+static void vschannel_cleanup();
+
+static INT request(CHAR *host, CHAR *req, CHAR *out);
 
 static SECURITY_STATUS create_credentials(PCredHandle phCreds);
 
@@ -36,7 +42,6 @@ static SECURITY_STATUS client_handshake_loop(
 static SECURITY_STATUS https_make_request(
 	SOCKET Socket, PCredHandle phCreds,
 	CtxtHandle *phContext, CHAR *req, CHAR *out, int *length);
-	// CtxtHandle *phContext, CHAR *path);
 
 static DWORD verify_server_certificate(
 	PCCERT_CONTEXT  pServerCert, PSTR host, DWORD dwCertFlags);

vlib/http/backend_nix.v

@@ -31,7 +31,7 @@ fn init_module() {
 	//C.OPENSSL_config(0) 
 }
 
-fn ssl_do(method, host_name, path string) string { 
+fn ssl_do(method, host_name, path string) Response { 
 	//ssl_method := C.SSLv23_method() 
 	ssl_method := C.TLSv1_2_method() 
 	if isnil(method) { 
@@ -66,10 +66,8 @@ fn ssl_do(method, host_name, path string) string {
 	res = C.BIO_do_handshake(web) 
 	cert := C.SSL_get_peer_certificate(ssl) 
 	res = C.SSL_get_verify_result(ssl) 
-	/////// 
-	s := '$method $path HTTP/1.1\r\n' + 
-	     'Host: $host_name\r\n' + 
-	     'Connection: close\r\n\r\n' 
+	///////
+	s := build_request_headers('', method, host_name, path)
 	C.BIO_puts(web, s.str) 
 	C.BIO_puts(out, '\n') 
 	mut sb := strings.new_builder(100) 
@@ -91,6 +89,7 @@ fn ssl_do(method, host_name, path string) string {
 	} 
 	if !isnil(ctx) { 
 		C.SSL_CTX_free(ctx) 
-	} 
-	return sb.str() 
+	}
+
+	return parse_response(sb.str() )
 }

vlib/http/backend_win.v

@@ -4,28 +4,44 @@
 
 module http
 
-import strings 
+import strings
+import net.urllib
 
 
 #flag windows -I @VROOT/thirdparty/vschannel
-#flag -lws2_32 -lcrypt32
+#flag -l ws2_32
+#flag -l crypt32
  
 #include "vschannel.c"
 
+const (
+	max_redirects = 4
+)
 
 fn init_module() {}
 
-fn ssl_do(method, host_name, path string) string { 
-	mut buff := malloc(10000)
+fn ssl_do(method, host_name, path string) Response { 
+	C.vschannel_init()
+	// TODO: joe-c
+	// dynamically increase in vschannel.c if needed
+	mut buff := malloc(44000)
 	
-	req := '$method $path HTTP/1.0\r\nUser-Agent: v\r\nAccept:*/*\r\n\r\n'
-	length := int(C.request(host_name.str, req.str, buff))
+	mut p := if path == '' { '/' } else { path }
+	mut req := build_request_headers('', method, host_name, p)
+	mut length := int(C.request(host_name.str, req.str, buff))
+	mut resp := parse_response(string(buff, length))
 	
-	if length == 0 {
-		return ''
+	mut no_redirects := 0
+	for resp.status_code == 301 && no_redirects <= max_redirects {
+		u := urllib.parse(resp.headers['Location']) or { break }
+		p = if u.path == '' { '/' } else { u.path }
+		req = build_request_headers('', method, u.hostname(), p)
+		length = int(C.request(u.hostname().str, req.str, buff))
+		resp = parse_response(string(buff, length))
+		no_redirects++
 	}
 
-	resp := tos(buff, length)
-	
+	free(buff)
+	C.vschannel_cleanup()
 	return resp
 }

vlib/http/http.v

@@ -93,7 +93,6 @@ pub fn (req mut Request) add_header(key, val string) {
 }
 
 pub fn (req &Request) do() Response {
-	mut headers := map[string]string{}
 	if req.typ == 'POST' {
 		// req.headers << 'Content-Type: application/x-www-form-urlencoded'
 	}
@@ -108,9 +107,13 @@ pub fn (req &Request) do() Response {
 	if !is_ssl {
 		panic('non https requests are not supported right now') 
 	}
-	s := ssl_do(req.typ, url.host, url.path)
-	// s := ssl_do(req.typ, url.host, url.path) 
-	first_header := s.all_before('\n') 
+	
+	return ssl_do(req.typ, url.hostname(), url.path)
+}
+
+fn parse_response(resp string) Response {
+	mut headers := map[string]string{}
+	first_header := resp.all_before('\n') 
 	mut status_code := 0 
 	if first_header.contains('HTTP/') {
 		val := first_header.find_between(' ', ' ')
@@ -122,14 +125,14 @@ pub fn (req &Request) do() Response {
 	mut i := 1 
 	for { 
 		old_pos := nl_pos 
-		nl_pos = s.index_after('\n', nl_pos+1) 
+		nl_pos = resp.index_after('\n', nl_pos+1) 
 		if nl_pos == -1 {
 			break 
 		} 
-		h := s.substr(old_pos + 1, nl_pos) 
+		h := resp.substr(old_pos + 1, nl_pos) 
 		// End of headers 
 		if h.len <= 1 {
-			text = s.right(nl_pos + 1) 
+			text = resp.right(nl_pos + 1) 
 			break 
 		} 
 		i++ 
@@ -144,14 +147,24 @@ pub fn (req &Request) do() Response {
 		val := h.right(pos + 2)
 		headers[key] = val.trim_space()
 	}
+	
 	if headers['Transfer-Encoding'] == 'chunked' {
 		text = chunked.decode( text )
 	}
+
 	return Response {
 		status_code: status_code 
 		headers: headers
 		text: text 
-	} 
+	}
+}
+
+fn build_request_headers(user_agent, method, host_name, path string) string {
+	ua := if user_agent == '' { 'v' } else { user_agent }
+	return '$method $path HTTP/1.1\r\n' + 
+		   'Host: $host_name\r\n' + 
+		   'User-Agent: $ua\r\n' +
+		   'Connection: close\r\n\r\n'
 }
 
 pub fn unescape_url(s string) string {