diff --git a/vlib/net/http/header.v b/vlib/net/http/header.v index 9fb919de89..bd11c2670d 100644 --- a/vlib/net/http/header.v +++ b/vlib/net/http/header.v @@ -3,6 +3,8 @@ // that can be found in the LICENSE file. module http +import strings + // CommonHeader is an enum of the most common HTTP headers pub enum CommonHeader { accept @@ -323,11 +325,15 @@ const common_header_map = map{ pub struct Header { mut: data map[string][]string + // map of lowercase header keys to their original keys + // in order of appearance + keys map[string][]string } pub fn (mut h Header) free() { unsafe { h.data.free() + h.keys.free() } } @@ -349,80 +355,136 @@ pub fn new_header(kvs ...HeaderConfig) Header { // Append a value to the header key. pub fn (mut h Header) add(key CommonHeader, value string) { - h.data[key.str()] << value + k := key.str() + h.data[k] << value + h.add_key(k) } // Append a value to a custom header key. This function will return an error // if the key contains invalid header characters. -pub fn (mut h Header) add_str(key string, value string) ? { - k := canonicalize(key) ? - h.data[k] << value +pub fn (mut h Header) add_custom(key string, value string) ? { + is_valid(key) ? + h.data[key] << value + h.add_key(key) } // Sets the key-value pair. This function will clear any other values // that exist for the CommonHeader. pub fn (mut h Header) set(key CommonHeader, value string) { - h.data[key.str()] = [value] + k := key.str() + h.data[k] = [value] + h.add_key(k) } // Sets the key-value pair for a custom header key. This function will -// clear any other values that exist for the CommonHeader. -pub fn (mut h Header) set_str(key string, value string) { - k := canonicalize(key) or { return } - h.data[k] = [value] +// clear any other values that exist for the header. This function will +// return an error if the key contains invalid header characters. +pub fn (mut h Header) set_custom(key string, value string) ? { + is_valid(key) ? + h.data[key] = [value] + h.add_key(key) } // Delete all values for a key. pub fn (mut h Header) delete(key CommonHeader) { - h.data.delete(key.str()) + h.delete_custom(key.str()) } // Delete all values for a custom header key. -pub fn (mut h Header) delete_str(key string) { - k := canonicalize(key) or { return } - h.data.delete(k) +pub fn (mut h Header) delete_custom(key string) { + h.data.delete(key) + + // remove key from keys metadata + kl := key.to_lower() + if kl in h.keys { + h.keys[kl] = h.keys[kl].filter(it != key) + } +} + +pub struct HeaderCoerceConfig { + canonicalize bool +} + +// Coerce data by joining keys that match case-insensitively into one entry +pub fn (mut h Header) coerce(flags ...HeaderCoerceConfig) { + canon := flags.any(it.canonicalize) + + for kl, data_keys in h.keys { + master_key := if canon { canonicalize(kl) } else { data_keys[0] } + + // save master data + master_data := h.data[master_key] + h.data.delete(master_key) + + for key in data_keys { + if key == master_key { + h.data[master_key] << master_data + continue + } + h.data[master_key] << h.data[key] + h.data.delete(key) + } + h.keys[kl] = [master_key] + } } // Returns whether the header key exists in the map. pub fn (h Header) contains(key CommonHeader) bool { - return key.str() in h.data + return h.contains_custom(key.str()) +} + +pub struct HeaderQueryConfig { + exact bool } // Returns whether the custom header key exists in the map. -pub fn (h Header) contains_str(key string) bool { - k := canonicalize(key) or { return false } - return k in h.data +pub fn (h Header) contains_custom(key string, flags ...HeaderQueryConfig) bool { + if flags.any(it.exact) { + return key in h.data + } + return key.to_lower() in h.keys } // Gets the first value for the CommonHeader, or none if the key does // not exist. pub fn (h Header) get(key CommonHeader) ?string { - k := key.str() - if h.data[k].len == 0 { - return none - } - return h.data[k][0] + return h.get_custom(key.str()) } // Gets the first value for the custom header, or none if the key does // not exist. -pub fn (h Header) get_str(key string) ?string { - k := canonicalize(key) or { return none } - if h.data[k].len == 0 { +pub fn (h Header) get_custom(key string, flags ...HeaderQueryConfig) ?string { + mut data_key := key + if !flags.any(it.exact) { + // get the first key from key metadata + k := key.to_lower() + if h.keys[k].len == 0 { + return none + } + data_key = h.keys[k][0] + } + if h.data[data_key].len == 0 { return none } - return h.data[k][0] + return h.data[data_key][0] } // Gets all values for the CommonHeader. pub fn (h Header) values(key CommonHeader) []string { - return h.data[key.str()] + return h.custom_values(key.str()) } // Gets all values for the custom header. -pub fn (h Header) values_str(key string) []string { - k := canonicalize(key) or { return [] } - return h.data[k] +pub fn (h Header) custom_values(key string, flags ...HeaderQueryConfig) []string { + if flags.any(it.exact) { + return h.data[key] + } + // case insensitive lookup + mut values := []string{cap: 10} + for k in h.keys[key.to_lower()] { + values << h.data[k] + } + return values } // Gets all header keys as strings @@ -430,37 +492,87 @@ pub fn (h Header) keys() []string { return h.data.keys() } -// Validate and canonicalize an HTTP header key -// A canonical header is all lowercase except for the first character -// and any character after a `-`. Example: `Example-Header-Key` -// There are some exceptions like `DNT`, `WWW-Authenticate`, etc. For these we -// check if the lowercase matches any in the common_header_map and return that. -fn canonicalize(s string) ?string { - // check for valid header bytes - for _, c in s { +pub struct HeaderRenderConfig { + version Version + coerce bool + canonicalize bool +} + +// Renders the Header into a string for use in sending +// HTTP requests. All header lines will end in `\n\r` +[manualfree] +pub fn (h Header) render(flags HeaderRenderConfig) string { + // estimate ~48 bytes per header + mut sb := strings.new_builder(h.data.len * 48) + if flags.coerce { + for kl, data_keys in h.keys { + key := if flags.version == .v2_0 { + kl + } else if flags.canonicalize { + canonicalize(kl) + } else { + data_keys[0] + } + sb.write_string(key) + sb.write_string(': ') + for i in 0 .. data_keys.len - 1 { + k := data_keys[i] + for v in h.data[k] { + sb.write_string(v) + sb.write_string(',') + } + } + k := data_keys[data_keys.len - 1] + sb.write_string(h.data[k].join(',')) + sb.write_string('\n\r') + } + } else { + for k, v in h.data { + key := if flags.version == .v2_0 { + k.to_lower() + } else if flags.canonicalize { + canonicalize(k.to_lower()) + } else { + k + } + sb.write_string(key) + sb.write_string(': ') + sb.write_string(v.join(',')) + sb.write_string('\n\r') + } + } + res := sb.str() + unsafe { sb.free() } + return res +} + +// Canonicalize an HTTP header key +// Common headers are determined by the common_header_map +// Custom headers are capitalized on the first letter and any letter after a '-' +// NOTE: Assumes sl is lowercase, since the caller usually already has the lowercase key +fn canonicalize(sl string) string { + // check if we have a common header + if sl in http.common_header_map { + return http.common_header_map[sl].str() + } + return sl.split('-').map(it.capitalize()).join('-') +} + +// Helper function to add a key to the keys map +fn (mut h Header) add_key(key string) { + kl := key.to_lower() + if !h.keys[kl].contains(key) { + h.keys[kl] << key + } +} + +// Checks if the header token is valid +fn is_valid(header string) ? { + for _, c in header { if int(c) >= 128 || !is_token(c) { return error('Invalid header key') } } - - // check if we have a common header - sl := s.to_lower() - if sl in http.common_header_map { - return http.common_header_map[sl].str() - } - - // check for canonicalization; create a new string if not - mut upper := true - for _, c in s { - if upper && `a` <= c && c <= `z` { - return s.to_lower().split('-').map(it.capitalize()).join('-') - } - if !upper && `A` <= c && c <= `Z` { - return s.to_lower().split('-').map(it.capitalize()).join('-') - } - upper = c == `-` - } - return s } // Checks if the byte is valid for a header token @@ -470,3 +582,9 @@ fn is_token(b byte) bool { else { false } } } + +// Returns the headers string as seen in HTTP/1.1 requests +// Key order is not guaranteed +pub fn (h Header) str() string { + return h.render(version: .v1_1) +} diff --git a/vlib/net/http/header_test.v b/vlib/net/http/header_test.v index 4b63193e87..2aca042769 100644 --- a/vlib/net/http/header_test.v +++ b/vlib/net/http/header_test.v @@ -1,11 +1,11 @@ -import net.http +module http fn test_header_new() { h := http.new_header( {key: .accept, value: 'nothing'}, {key: .expires, value: 'yesterday'} ) - assert h.contains_str('accept') + assert h.contains(.accept) assert h.contains(.expires) accept := h.get(.accept) or { '' } expires := h.get(.expires) or { '' } @@ -15,7 +15,7 @@ fn test_header_new() { fn test_header_invalid_key() { mut h := http.new_header() - h.add_str('space is invalid', ':(') or { return } + h.add_custom('space is invalid', ':(') or { return } panic('should have returned') } @@ -27,13 +27,22 @@ fn test_header_adds_multiple() { assert h.values(.accept) == ['one' 'two'] } -fn test_header_set() { +fn test_header_get() ? { + mut h := http.new_header(key: .dnt, value: 'one') + h.add_custom('dnt', 'two') ? + dnt := h.get_custom('dnt') or { '' } + exact := h.get_custom('dnt', exact: true) or { '' } + assert dnt == 'one' + assert exact == 'two' +} + +fn test_header_set() ? { mut h := http.new_header( {key: .dnt, value: 'one'}, {key: .dnt, value: 'two'} ) assert h.values(.dnt) == ['one' 'two'] - h.set_str('dnt', 'three') + h.set_custom('DNT', 'three') ? assert h.values(.dnt) == ['three'] } @@ -43,6 +52,217 @@ fn test_header_delete() { {key: .dnt, value: 'two'} ) assert h.values(.dnt) == ['one' 'two'] - h.delete_str('dnt') + h.delete(.dnt) assert h.values(.dnt) == [] } + +fn test_header_delete_not_existing() { + mut h := http.new_header() + assert h.data.len == 0 + assert h.keys.len == 0 + h.delete(.dnt) + assert h.data.len == 0 + assert h.keys.len == 0 +} + +fn test_custom_header() ? { + mut h := http.new_header() + h.add_custom('AbC', 'dEf') ? + h.add_custom('aBc', 'GhI') ? + assert h.custom_values('AbC', exact: true) == ['dEf'] + assert h.custom_values('aBc', exact: true) == ['GhI'] + assert h.custom_values('ABC') == ['dEf', 'GhI'] + assert h.custom_values('abc') == ['dEf', 'GhI'] + assert h.keys() == ['AbC', 'aBc'] + h.delete_custom('AbC') + h.delete_custom('aBc') + + h.add_custom('abc', 'def') ? + assert h.custom_values('abc') == ['def'] + assert h.custom_values('ABC') == ['def'] + assert h.keys() == ['abc'] + h.delete_custom('abc') + + h.add_custom('accEPT', '*/*') ? + assert h.custom_values('ACCept') == ['*/*'] + assert h.values(.accept) == ['*/*'] + assert h.keys() == ['accEPT'] +} + +fn test_contains_custom() ? { + mut h := http.new_header() + h.add_custom('Hello', 'world') ? + assert h.contains_custom('hello') + assert h.contains_custom('HELLO') + assert h.contains_custom('Hello', exact: true) + assert h.contains_custom('hello', exact: true) == false + assert h.contains_custom('HELLO', exact: true) == false +} + +fn test_get_custom() ? { + mut h := http.new_header() + h.add_custom('Hello', 'world') ? + assert h.get_custom('hello') ? == 'world' + assert h.get_custom('HELLO') ? == 'world' + assert h.get_custom('Hello', exact: true) ? == 'world' + if _ := h.get_custom('hello', exact: true) { + // should be none + assert false + } + if _ := h.get_custom('HELLO', exact: true) { + // should be none + assert false + } +} + +fn test_custom_values() ? { + mut h := http.new_header() + h.add_custom('Hello', 'world') ? + assert h.custom_values('hello') == ['world'] + assert h.custom_values('HELLO') == ['world'] + assert h.custom_values('Hello', exact: true) == ['world'] + assert h.custom_values('hello', exact: true) == [] + assert h.custom_values('HELLO', exact: true) == [] +} + +fn test_coerce() ? { + mut h := http.new_header() + h.add_custom('accept', 'foo') ? + h.add(.accept, 'bar') + assert h.values(.accept) == ['foo', 'bar'] + assert h.keys().len == 2 + + h.coerce() + assert h.values(.accept) == ['foo', 'bar'] + assert h.keys() == ['accept'] // takes the first occurrence +} + +fn test_coerce_canonicalize() ? { + mut h := http.new_header() + h.add_custom('accept', 'foo') ? + h.add(.accept, 'bar') + assert h.values(.accept) == ['foo', 'bar'] + assert h.keys().len == 2 + + h.coerce(canonicalize: true) + assert h.values(.accept) == ['foo', 'bar'] + assert h.keys() == ['Accept'] // canonicalize header +} + +fn test_coerce_custom() ? { + mut h := http.new_header() + h.add_custom('Hello', 'foo') ? + h.add_custom('hello', 'bar') ? + h.add_custom('HELLO', 'baz') ? + assert h.custom_values('hello') == ['foo', 'bar', 'baz'] + assert h.keys().len == 3 + + h.coerce() + assert h.custom_values('hello') == ['foo', 'bar', 'baz'] + assert h.keys() == ['Hello'] // takes the first occurrence +} + +fn test_coerce_canonicalize_custom() ? { + mut h := http.new_header() + h.add_custom('foo-BAR', 'foo') ? + h.add_custom('FOO-bar', 'bar') ? + assert h.custom_values('foo-bar') == ['foo', 'bar'] + assert h.keys().len == 2 + + h.coerce(canonicalize: true) + assert h.custom_values('foo-bar') == ['foo', 'bar'] + assert h.keys() == ['Foo-Bar'] // capitalizes the header +} + +fn test_render_version() ? { + mut h := http.new_header() + h.add_custom('accept', 'foo') ? + h.add_custom('Accept', 'bar') ? + h.add(.accept, 'baz') + + s1_0 := h.render(version: .v1_0) + assert s1_0.contains('accept: foo\n\r') + assert s1_0.contains('Accept: bar,baz\n\r') + + s1_1 := h.render(version: .v1_1) + assert s1_1.contains('accept: foo\n\r') + assert s1_1.contains('Accept: bar,baz\n\r') + + s2_0 := h.render(version: .v2_0) + assert s2_0.contains('accept: foo\n\r') + assert s2_0.contains('accept: bar,baz\n\r') +} + +fn test_render_coerce() ? { + mut h := http.new_header() + h.add_custom('accept', 'foo') ? + h.add_custom('Accept', 'bar') ? + h.add(.accept, 'baz') + h.add(.host, 'host') + + s1_0 := h.render(version: .v1_1, coerce: true) + assert s1_0.contains('accept: foo,bar,baz\n\r') + assert s1_0.contains('Host: host\n\r') + + s1_1 := h.render(version: .v1_1, coerce: true) + assert s1_1.contains('accept: foo,bar,baz\n\r') + assert s1_1.contains('Host: host\n\r') + + s2_0 := h.render(version: .v2_0, coerce: true) + assert s2_0.contains('accept: foo,bar,baz\n\r') + assert s2_0.contains('host: host\n\r') +} + +fn test_render_canonicalize() ? { + mut h := http.new_header() + h.add_custom('accept', 'foo') ? + h.add_custom('Accept', 'bar') ? + h.add(.accept, 'baz') + h.add(.host, 'host') + + s1_0 := h.render(version: .v1_1, canonicalize: true) + assert s1_0.contains('Accept: foo\n\r') + assert s1_0.contains('Accept: bar,baz\n\r') + assert s1_0.contains('Host: host\n\r') + + s1_1 := h.render(version: .v1_1, canonicalize: true) + assert s1_1.contains('Accept: foo\n\r') + assert s1_1.contains('Accept: bar,baz\n\r') + assert s1_1.contains('Host: host\n\r') + + s2_0 := h.render(version: .v2_0, canonicalize: true) + assert s2_0.contains('accept: foo\n\r') + assert s2_0.contains('accept: bar,baz\n\r') + assert s2_0.contains('host: host\n\r') +} + +fn test_render_coerce_canonicalize() ? { + mut h := http.new_header() + h.add_custom('accept', 'foo') ? + h.add_custom('Accept', 'bar') ? + h.add(.accept, 'baz') + h.add(.host, 'host') + + s1_0 := h.render(version: .v1_1, coerce: true, canonicalize: true) + assert s1_0.contains('Accept: foo,bar,baz\n\r') + assert s1_0.contains('Host: host\n\r') + + s1_1 := h.render(version: .v1_1, coerce: true, canonicalize: true) + assert s1_1.contains('Accept: foo,bar,baz\n\r') + assert s1_1.contains('Host: host\n\r') + + s2_0 := h.render(version: .v2_0, coerce: true, canonicalize: true) + assert s2_0.contains('accept: foo,bar,baz\n\r') + assert s2_0.contains('host: host\n\r') +} + +fn test_str() ? { + mut h := http.new_header() + h.add(.accept, 'text/html') + h.add_custom('Accept', 'image/jpeg') ? + h.add_custom('X-custom', 'Hello') ? + + // key order is not guaranteed + assert h.str() == 'Accept: text/html,image/jpeg\n\rX-custom: Hello\n\r' + || h.str() == 'X-custom: Hello\n\rAccept:text/html,image/jpeg\n\r' +} diff --git a/vlib/vweb/request.v b/vlib/vweb/request.v index a1ff88a13a..88dae2ec61 100644 --- a/vlib/vweb/request.v +++ b/vlib/vweb/request.v @@ -15,16 +15,17 @@ fn parse_request(mut reader io.BufferedReader) ?http.Request { line = reader.read_line() ? for line != '' { key, value := parse_header(line) ? - h.add_str(key, value) ? + h.add_custom(key, value) ? line = reader.read_line() ? } + h.coerce(canonicalize: true) // create map[string]string from headers // TODO: replace headers and lheaders with http.Header type mut headers := map[string]string{} mut lheaders := map[string]string{} for key in h.keys() { - values := h.values_str(key).join('; ') + values := h.custom_values(key).join('; ') headers[key] = values lheaders[key.to_lower()] = values } diff --git a/vlib/x/websocket/handshake.v b/vlib/x/websocket/handshake.v index 172ae6a1eb..eac1ffbff1 100644 --- a/vlib/x/websocket/handshake.v +++ b/vlib/x/websocket/handshake.v @@ -24,7 +24,7 @@ fn (mut ws Client) handshake() ? { sb.write_string(seckey) sb.write_string('\r\nSec-WebSocket-Version: 13') for key in ws.header.keys() { - val := ws.header.values_str(key).join(',') + val := ws.header.custom_values(key).join(',') sb.write_string('\r\n$key:$val') } sb.write_string('\r\n\r\n')