move http module to net.http

vlib/net/http/backend_nix.v

@@ -0,0 +1,145 @@
+// Copyright (c) 2019 Alexander Medvednikov. All rights reserved.
+// Use of this source code is governed by an MIT license
+// that can be found in the LICENSE file.
+module http
+
+import strings
+// On linux, prefer a localy build openssl, because it is
+// much more likely for it to be newer, than the system
+// openssl from libssl-dev. If there is no local openssl,
+// the next flag is harmless, since it will still use the
+// (older) system openssl.
+#flag linux -I/usr/local/include/openssl -L/usr/local/lib
+#flag -l ssl -l crypto
+// MacPorts
+#flag darwin -I/opt/local/include
+#flag darwin -L/opt/local/lib
+// Brew
+#flag darwin -I/usr/local/opt/openssl/include
+#flag darwin -L/usr/local/opt/openssl/lib
+#include <openssl/ssl.h>
+struct C.SSL {
+}
+
+fn C.SSL_library_init()
+
+
+fn C.TLSv1_2_method() voidptr
+
+
+fn C.SSL_CTX_set_options()
+
+
+fn C.SSL_CTX_new() voidptr
+
+
+fn C.SSL_CTX_set_verify_depth()
+
+
+fn C.SSL_CTX_load_verify_locations() int
+
+
+fn C.BIO_new_ssl_connect() voidptr
+
+
+fn C.BIO_set_conn_hostname() int
+
+
+fn C.BIO_get_ssl()
+
+
+fn C.SSL_set_cipher_list() int
+
+
+fn C.BIO_do_connect() int
+
+
+fn C.BIO_do_handshake() int
+
+
+fn C.SSL_get_peer_certificate() int
+
+
+fn C.SSL_get_verify_result() int
+
+
+fn C.SSL_set_tlsext_host_name() int
+
+
+fn C.BIO_puts()
+
+
+fn C.BIO_read()
+
+
+fn C.BIO_free_all()
+
+
+fn C.SSL_CTX_free()
+
+
+fn init() int {
+	C.SSL_library_init()
+	return 1
+}
+
+fn (req &Request) ssl_do(port int, method, host_name, path string) ?Response {
+	// ssl_method := C.SSLv23_method()
+	ssl_method := C.TLSv1_2_method()
+	if isnil(method) {
+	}
+	ctx := C.SSL_CTX_new(ssl_method)
+	if isnil(ctx) {
+	}
+	C.SSL_CTX_set_verify_depth(ctx, 4)
+	flags := C.SSL_OP_NO_SSLv2 | C.SSL_OP_NO_SSLv3 | C.SSL_OP_NO_COMPRESSION
+	C.SSL_CTX_set_options(ctx, flags)
+	mut res := C.SSL_CTX_load_verify_locations(ctx, 'random-org-chain.pem', 0)
+	if res != 1 {
+	}
+	web := C.BIO_new_ssl_connect(ctx)
+	if isnil(ctx) {
+	}
+	addr := host_name + ':' + port.str()
+	res = C.BIO_set_conn_hostname(web, addr.str)
+	if res != 1 {
+	}
+	ssl := &C.SSL(0)
+	C.BIO_get_ssl(web, &ssl)
+	if isnil(ssl) {
+	}
+	preferred_ciphers := 'HIGH:!aNULL:!kRSA:!PSK:!SRP:!MD5:!RC4'
+	res = C.SSL_set_cipher_list(ssl, preferred_ciphers.str)
+	if res != 1 {
+	}
+	res = C.SSL_set_tlsext_host_name(ssl, host_name.str)
+	res = C.BIO_do_connect(web)
+	if res != 1 {
+		return error('cannot connect the endpoint')
+	}
+	res = C.BIO_do_handshake(web)
+	C.SSL_get_peer_certificate(ssl)
+	res = C.SSL_get_verify_result(ssl)
+	// /////
+	s := req.build_request_headers(method, host_name, path)
+	C.BIO_puts(web, s.str)
+	mut sb := strings.new_builder(100)
+	for {
+		buff := [1536]byte
+		len := int(C.BIO_read(web, buff, 1536))
+		if len > 0 {
+			sb.write(tos(buff, len))
+		}
+		else {
+			break
+		}
+	}
+	if !isnil(web) {
+		C.BIO_free_all(web)
+	}
+	if !isnil(ctx) {
+		C.SSL_CTX_free(ctx)
+	}
+	return parse_response(sb.str())
+}
+

vlib/net/http/backend_windows.v

@@ -0,0 +1,25 @@
+// Copyright (c) 2019 Alexander Medvednikov. All rights reserved.
+// Use of this source code is governed by an MIT license
+// that can be found in the LICENSE file.
+
+module http
+
+#flag windows -I @VROOT/thirdparty/vschannel
+#flag -l ws2_32 -l crypt32 -l secur32
+ 
+#include "vschannel.c"
+
+fn C.new_tls_context() C.TlsContext
+
+fn (req &Request) ssl_do(port int, method, host_name, path string) ?Response {
+	mut ctx := C.new_tls_context()
+	C.vschannel_init(&ctx)
+
+	mut buff := malloc(C.vsc_init_resp_buff_size)
+	addr := host_name
+	sdata := req.build_request_headers(method, host_name, path)
+	length := int(C.request(&ctx, port, addr.to_wide(), sdata.str, &buff))
+
+	C.vschannel_cleanup(&ctx)
+	return parse_response(string(buff, length))
+}

vlib/net/http/chunked/dechunk.v

@@ -0,0 +1,74 @@
+module chunked
+
+import strings
+// See: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
+// /////////////////////////////////////////////////////////////
+// The chunk size is transferred as a hexadecimal number
+// followed by \r\n as a line separator,
+// followed by a chunk of data of the given size.
+// The end is marked with a chunk with size 0.
+struct ChunkScanner {
+mut:
+	pos  int
+	text string
+}
+
+fn (s mut ChunkScanner) read_chunk_size() int {
+	mut n := 0
+	for {
+		if s.pos >= s.text.len {
+			break
+		}
+		c := s.text[s.pos]
+		if !c.is_hex_digit() {
+			break
+		}
+		n = n<<4
+		n += int(unhex(c))
+		s.pos++
+	}
+	return n
+}
+
+fn unhex(c byte) byte {
+	if `0` <= c && c <= `9` {
+		return c - `0`
+	}
+	else if `a` <= c && c <= `f` {
+		return c - `a` + 10
+	}
+	else if `A` <= c && c <= `F` {
+		return c - `A` + 10
+	}
+	return 0
+}
+
+fn (s mut ChunkScanner) skip_crlf() {
+	s.pos += 2
+}
+
+fn (s mut ChunkScanner) read_chunk(chunksize int) string {
+	startpos := s.pos
+	s.pos += chunksize
+	return s.text[startpos..s.pos]
+}
+
+pub fn decode(text string) string {
+	mut sb := strings.new_builder(100)
+	mut cscanner := ChunkScanner{
+		pos: 0
+		text: text
+	}
+	for {
+		csize := cscanner.read_chunk_size()
+		if 0 == csize {
+			break
+		}
+		cscanner.skip_crlf()
+		sb.write(cscanner.read_chunk(csize))
+		cscanner.skip_crlf()
+	}
+	cscanner.skip_crlf()
+	return sb.str()
+}
+

vlib/net/http/download.v

@@ -0,0 +1,16 @@
+// Copyright (c) 2019 Alexander Medvednikov. All rights reserved.
+// Use of this source code is governed by an MIT license
+// that can be found in the LICENSE file.
+module http
+
+import os
+
+pub fn download_file(url, out string) bool {
+	s := get(url) or {
+		return false
+	}
+	os.write_file(out, s.text)
+	return true
+	// download_file_with_progress(url, out, empty, empty)
+}
+

vlib/net/http/download_nix.v

@@ -0,0 +1,52 @@
+// Copyright (c) 2019 Alexander Medvednikov. All rights reserved.
+// Use of this source code is governed by an MIT license
+// that can be found in the LICENSE file.
+module http
+
+type downloadfn fn(written int)type download_finished_fn fn()
+/*
+struct DownloadStruct {
+mut:
+	stream  voidptr
+	written int
+	cb      downloadfn
+}
+*/
+fn download_cb(ptr voidptr, size, nmemb size_t, userp voidptr) {
+	/*
+	mut data := &DownloadStruct(userp)
+	written := C.fwrite(ptr, size, nmemb, data.stream)
+	data.written += written
+	data.cb(data.written)
+	//#data->cb(data->written); // TODO
+	return written
+*/
+}
+
+pub fn download_file_with_progress(url, out string, cb downloadfn, cb_finished fn()) {
+	/*
+	curl := C.curl_easy_init()
+	if isnil(curl) {
+		return
+	}
+	cout := out.str
+	fp := C.fopen(cout, 'wb')
+	C.curl_easy_setopt(curl, CURLOPT_URL, url.str)
+	C.curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, download_cb)
+	data := &DownloadStruct {
+		stream:fp
+		cb: cb
+	}
+	C.curl_easy_setopt(curl, CURLOPT_WRITEDATA, data)
+	mut d := 0.0
+	C.curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &d)
+	C.curl_easy_perform(curl)
+	C.curl_easy_cleanup(curl)
+	C.fclose(fp)
+	cb_finished()
+*/
+}
+
+fn empty() {
+}
+

vlib/net/http/download_windows.v

@@ -0,0 +1,29 @@
+// Copyright (c) 2019 Alexander Medvednikov. All rights reserved.
+// Use of this source code is governed by an MIT license
+// that can be found in the LICENSE file.
+
+module http
+
+#flag -l Urlmon
+
+#include <Urlmon.h>
+
+fn download_file_with_progress(url, out string, cb, cb_finished voidptr) {
+}
+
+/*
+pub fn download_file(url, out string) {
+	C.URLDownloadToFile(0, url.to_wide(), out.to_wide(), 0, 0)
+	/*
+	if (res == S_OK) {
+	println('Download Ok')
+	# } else if(res == E_OUTOFMEMORY) {
+	println('Buffer length invalid, or insufficient memory')
+	# } else if(res == INET_E_DOWNLOAD_FAILURE) {
+	println('URL is invalid')
+	# } else {
+	# printf("Download error: %d\n", res);
+	# }
+	*/
+}
+*/

vlib/net/http/http.v

@@ -0,0 +1,253 @@
+// Copyright (c) 2019 Alexander Medvednikov. All rights reserved.
+// Use of this source code is governed by an MIT license
+// that can be found in the LICENSE file.
+module http
+
+import net.urllib
+import net.http.chunked
+
+const (
+	max_redirects = 4
+)
+
+pub struct Request {
+pub:
+	headers    map[string]string
+	method     string
+	// cookies map[string]string
+	h          string
+	cmd        string
+	typ        string // GET POST
+	data       string
+	url        string
+	verbose    bool
+	user_agent string
+mut:
+	user_ptr   voidptr
+	ws_func    voidptr
+}
+
+pub struct Response {
+pub:
+	text        string
+	headers     map[string]string
+	status_code int
+}
+
+pub fn get(url string) ?Response {
+	req := new_request('GET', url, '') or {
+		return error(err)
+	}
+	res := req.do() or {
+		return error(err)
+	}
+	return res
+}
+
+pub fn post(url, data string) ?Response {
+	req := new_request('POST', url, data) or {
+		return error(err)
+	}
+	res := req.do() or {
+		return error(err)
+	}
+	return res
+}
+
+// new_request creates a new HTTP request
+pub fn new_request(typ, _url, _data string) ?Request {
+	if _url == '' {
+		return error('http.new_request: empty url')
+	}
+	mut url := _url
+	mut data := _data
+	// req.headers['User-Agent'] = 'V $VERSION'
+	if typ == 'GET' && !url.contains('?') && data != '' {
+		url = '$url?$data'
+		data = ''
+	}
+	return Request{
+		typ: typ
+		url: url
+		data: data
+		ws_func: 0
+		user_ptr: 0
+		headers: map[string]string
+		user_agent: 'v'
+	}
+}
+
+pub fn get_text(url string) string {
+	resp := get(url) or {
+		return ''
+	}
+	return resp.text
+}
+
+fn (req mut Request) free() {
+	req.headers.free()
+}
+
+fn (resp mut Response) free() {
+	resp.headers.free()
+}
+
+// add_header adds the key and value of an HTTP request header
+pub fn (req mut Request) add_header(key, val string) {
+	req.headers[key] = val
+}
+
+pub fn parse_headers(lines []string) map[string]string {
+	mut headers := map[string]string
+	for i, line in lines {
+		if i == 0 {
+			continue
+		}
+		words := line.split(': ')
+		if words.len != 2 {
+			continue
+		}
+		headers[words[0]] = words[1]
+	}
+	return headers
+}
+
+// do will send the HTTP request and returns `http.Response` as soon as the response is recevied
+pub fn (req &Request) do() ?Response {
+	if req.typ == 'POST' {
+		// req.headers << 'Content-Type: application/x-www-form-urlencoded'
+	}
+	url := urllib.parse(req.url) or {
+		return error('http.request.do: invalid URL "$req.url"')
+	}
+	mut rurl := url
+	mut resp := Response{}
+	mut no_redirects := 0
+	for {
+		if no_redirects == max_redirects {
+			return error('http.request.do: maximum number of redirects reached ($max_redirects)')
+		}
+		qresp := req.method_and_url_to_response(req.typ, rurl) or {
+			return error(err)
+		}
+		resp = qresp
+		if !(resp.status_code in [301, 302, 303, 307, 308]) {
+			break
+		}
+		// follow any redirects
+		redirect_url := resp.headers['Location']
+		qrurl := urllib.parse(redirect_url) or {
+			return error('http.request.do: invalid URL in redirect "$redirect_url"')
+		}
+		rurl = qrurl
+		no_redirects++
+	}
+	return resp
+}
+
+fn (req &Request) method_and_url_to_response(method string, url net_dot_urllib.URL) ?Response {
+	host_name := url.hostname()
+	scheme := url.scheme
+	p := url.path.trim_left('/')
+	path := if url.query().size > 0 { '/$p?${url.query().encode()}' } else { '/$p' }
+	mut nport := url.port().int()
+	if nport == 0 {
+		if scheme == 'http' {
+			nport = 80
+		}
+		if scheme == 'https' {
+			nport = 443
+		}
+	}
+	// println('fetch $method, $scheme, $host_name, $nport, $path ')
+	if scheme == 'https' {
+		// println('ssl_do( $nport, $method, $host_name, $path )')
+		res := req.ssl_do(nport, method, host_name, path) or {
+			return error(err)
+		}
+		return res
+	}
+	else if scheme == 'http' {
+		// println('http_do( $nport, $method, $host_name, $path )')
+		res := req.http_do(nport, method, host_name, path) or {
+			return error(err)
+		}
+		return res
+	}
+	return error('http.request.method_and_url_to_response: unsupported scheme: "$scheme"')
+}
+
+fn parse_response(resp string) Response {
+	mut headers := map[string]string
+	first_header := resp.all_before('\n')
+	mut status_code := 0
+	if first_header.contains('HTTP/') {
+		val := first_header.find_between(' ', ' ')
+		status_code = val.int()
+	}
+	mut text := ''
+	// Build resp headers map and separate the body
+	mut nl_pos := 3
+	mut i := 1
+	for {
+		old_pos := nl_pos
+		nl_pos = resp.index_after('\n', nl_pos + 1)
+		if nl_pos == -1 {
+			break
+		}
+		h := resp[old_pos + 1..nl_pos]
+		// End of headers
+		if h.len <= 1 {
+			text = resp[nl_pos + 1..]
+			break
+		}
+		i++
+		pos := h.index(':') or {
+			continue
+		}
+		// if h.contains('Content-Type') {
+		// continue
+		// }
+		key := h[..pos]
+		val := h[pos + 2..]
+		headers[key] = val.trim_space()
+	}
+	if headers['Transfer-Encoding'] == 'chunked' {
+		text = chunked.decode(text)
+	}
+	return Response{
+		status_code: status_code
+		headers: headers
+		text: text
+	}
+}
+
+fn (req &Request) build_request_headers(method, host_name, path string) string {
+	ua := req.user_agent
+	mut uheaders := []string
+	for key, val in req.headers {
+		uheaders << '${key}: ${val}\r\n'
+	}
+	if req.data.len > 0 {
+		uheaders << 'Content-Length: ${req.data.len}\r\n'
+	}
+	return '$method $path HTTP/1.1\r\n' + 'Host: $host_name\r\n' + 'User-Agent: $ua\r\n' + uheaders.join('') + 'Connection: close\r\n\r\n' + req.data
+}
+
+pub fn unescape_url(s string) string {
+	panic('http.unescape_url() was replaced with urllib.query_unescape()')
+}
+
+pub fn escape_url(s string) string {
+	panic('http.escape_url() was replaced with urllib.query_escape()')
+}
+
+pub fn unescape(s string) string {
+	panic('http.unescape() was replaced with http.unescape_url()')
+}
+
+pub fn escape(s string) string {
+	panic('http.escape() was replaced with http.escape_url()')
+}
+
+type wsfn fn(s string, ptr voidptr)

vlib/net/http/http_client.v

@@ -0,0 +1,30 @@
+module http
+
+import net
+import strings
+
+fn (req &Request) http_do(port int, method, host_name, path string) ?Response {
+	bufsize := 512
+	rbuffer := [512]byte
+	mut sb := strings.new_builder(100)
+	s := req.build_request_headers(method, host_name, path)
+	client := net.dial(host_name, port) or {
+		return error(err)
+	}
+	client.send(s.str, s.len) or {
+	}
+	for {
+		readbytes := client.crecv(rbuffer, bufsize)
+		if readbytes < 0 {
+			return error('http.request.http_do: error reading response. readbytes=$readbytes')
+		}
+		if readbytes == 0 {
+			break
+		}
+		sb.write(tos(rbuffer, readbytes))
+	}
+	client.close() or {
+	}
+	return parse_response(sb.str())
+}
+

vlib/net/http/http_test.v

@@ -0,0 +1,46 @@
+// import net.urllib
+import net.http
+
+fn test_escape_unescape() {
+/*
+  original := 'те ст: т\\%'
+  escaped := urllib.query_escape(original) or { assert false return}
+  assert escaped == '%D1%82%D0%B5%20%D1%81%D1%82%3A%20%D1%82%5C%25'
+  unescaped := urllib.query_unescape(escaped) or { assert false return }
+  assert unescaped == original
+*/
+}
+
+fn test_http_get() {
+	assert http.get_text('https://vlang.io/version') == '0.1.5'
+	println('http ok')
+}
+
+fn test_http_get_from_vlang_utc_now() {	
+	urls := ['http://vlang.io/utc_now', 'https://vlang.io/utc_now']
+	for url in urls {
+		println('Test getting current time from $url by http.get')
+		res := http.get(url) or { panic(err) }
+		assert 200 == res.status_code
+		assert res.text.len > 0
+		assert res.text.int() > 1566403696
+		println('Current time is: ${res.text.int()}')
+	}
+}
+
+fn test_public_servers() {
+	urls := [
+		'http://github.com/robots.txt',
+		'http://google.com/robots.txt',
+		'http://yahoo.com/robots.txt',
+		'https://github.com/robots.txt',
+		'https://google.com/robots.txt',
+		'https://yahoo.com/robots.txt',
+	]
+	for url in urls {
+		println('Testing http.get on public url: $url ')
+		res :=  http.get( url ) or { panic(err) }
+		assert 200 == res.status_code
+		assert res.text.len > 0
+	}
+}