From c8b0f51c13f472f5ea617ffaeba580297f16a81f Mon Sep 17 00:00:00 2001 From: playX Date: Mon, 14 Mar 2022 10:20:20 +0300 Subject: [PATCH] builtin.wasm_bare: use walloc as malloc/free implementation (#13731) --- thirdparty/walloc/walloc.c | 541 +++++++++++++++++++++ vlib/builtin/wasm_bare/libc_impl.v | 25 +- vlib/builtin/wasm_bare/memory_management.v | 69 +-- vlib/v/builder/cc.v | 2 +- 4 files changed, 553 insertions(+), 84 deletions(-) create mode 100644 thirdparty/walloc/walloc.c diff --git a/thirdparty/walloc/walloc.c b/thirdparty/walloc/walloc.c new file mode 100644 index 0000000000..7cb3aa670b --- /dev/null +++ b/thirdparty/walloc/walloc.c @@ -0,0 +1,541 @@ +// walloc.c: a small malloc implementation for use in WebAssembly targets +// Copyright (c) 2020 Igalia, S.L. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +typedef __SIZE_TYPE__ size_t; +typedef __UINTPTR_TYPE__ uintptr_t; +typedef __UINT8_TYPE__ uint8_t; + +#define NULL ((void *)0) + +#define STATIC_ASSERT_EQ(a, b) _Static_assert((a) == (b), "eq") + +#ifndef NDEBUG +#define ASSERT(x) \ + do \ + { \ + if (!(x)) \ + __builtin_trap(); \ + } while (0) +#else +#define ASSERT(x) \ + do \ + { \ + } while (0) +#endif +#define ASSERT_EQ(a, b) ASSERT((a) == (b)) + +static inline size_t max(size_t a, size_t b) +{ + return a < b ? b : a; +} +static inline uintptr_t align(uintptr_t val, uintptr_t alignment) +{ + return (val + alignment - 1) & ~(alignment - 1); +} +#define ASSERT_ALIGNED(x, y) ASSERT((x) == align((x), y)) + +#define CHUNK_SIZE 256 +#define CHUNK_SIZE_LOG_2 8 +#define CHUNK_MASK (CHUNK_SIZE - 1) +STATIC_ASSERT_EQ(CHUNK_SIZE, 1 << CHUNK_SIZE_LOG_2); + +#define PAGE_SIZE 65536 +#define PAGE_SIZE_LOG_2 16 +#define PAGE_MASK (PAGE_SIZE - 1) +STATIC_ASSERT_EQ(PAGE_SIZE, 1 << PAGE_SIZE_LOG_2); + +#define CHUNKS_PER_PAGE 256 +STATIC_ASSERT_EQ(PAGE_SIZE, CHUNK_SIZE *CHUNKS_PER_PAGE); + +#define GRANULE_SIZE 8 +#define GRANULE_SIZE_LOG_2 3 +#define LARGE_OBJECT_THRESHOLD 256 +#define LARGE_OBJECT_GRANULE_THRESHOLD 32 + +STATIC_ASSERT_EQ(GRANULE_SIZE, 1 << GRANULE_SIZE_LOG_2); +STATIC_ASSERT_EQ(LARGE_OBJECT_THRESHOLD, + LARGE_OBJECT_GRANULE_THRESHOLD *GRANULE_SIZE); + +struct chunk +{ + char data[CHUNK_SIZE]; +}; + +// There are small object pages for allocations of these sizes. +#define FOR_EACH_SMALL_OBJECT_GRANULES(M) \ + M(1) \ + M(2) M(3) M(4) M(5) M(6) M(8) M(10) M(16) M(32) + +enum chunk_kind +{ +#define DEFINE_SMALL_OBJECT_CHUNK_KIND(i) GRANULES_##i, + FOR_EACH_SMALL_OBJECT_GRANULES(DEFINE_SMALL_OBJECT_CHUNK_KIND) +#undef DEFINE_SMALL_OBJECT_CHUNK_KIND + + SMALL_OBJECT_CHUNK_KINDS, + FREE_LARGE_OBJECT = 254, + LARGE_OBJECT = 255 +}; + +static const uint8_t small_object_granule_sizes[] = + { +#define SMALL_OBJECT_GRANULE_SIZE(i) i, + FOR_EACH_SMALL_OBJECT_GRANULES(SMALL_OBJECT_GRANULE_SIZE) +#undef SMALL_OBJECT_GRANULE_SIZE +}; + +static enum chunk_kind granules_to_chunk_kind(unsigned granules) +{ +#define TEST_GRANULE_SIZE(i) \ + if (granules <= i) \ + return GRANULES_##i; + FOR_EACH_SMALL_OBJECT_GRANULES(TEST_GRANULE_SIZE); +#undef TEST_GRANULE_SIZE + return LARGE_OBJECT; +} + +static unsigned chunk_kind_to_granules(enum chunk_kind kind) +{ + switch (kind) + { +#define CHUNK_KIND_GRANULE_SIZE(i) \ + case GRANULES_##i: \ + return i; + FOR_EACH_SMALL_OBJECT_GRANULES(CHUNK_KIND_GRANULE_SIZE); +#undef CHUNK_KIND_GRANULE_SIZE + default: + return -1; + } +} + +// Given a pointer P returned by malloc(), we get a header pointer via +// P&~PAGE_MASK, and a chunk index via (P&PAGE_MASK)/CHUNKS_PER_PAGE. If +// chunk_kinds[chunk_idx] is [FREE_]LARGE_OBJECT, then the pointer is a large +// object, otherwise the kind indicates the size in granules of the objects in +// the chunk. +struct page_header +{ + uint8_t chunk_kinds[CHUNKS_PER_PAGE]; +}; + +struct page +{ + union + { + struct page_header header; + struct chunk chunks[CHUNKS_PER_PAGE]; + }; +}; + +#define PAGE_HEADER_SIZE (sizeof(struct page_header)) +#define FIRST_ALLOCATABLE_CHUNK 1 +STATIC_ASSERT_EQ(PAGE_HEADER_SIZE, FIRST_ALLOCATABLE_CHUNK *CHUNK_SIZE); + +static struct page *get_page(void *ptr) +{ + return (struct page *)(char *)(((uintptr_t)ptr) & ~PAGE_MASK); +} +static unsigned get_chunk_index(void *ptr) +{ + return (((uintptr_t)ptr) & PAGE_MASK) / CHUNK_SIZE; +} + +struct freelist +{ + struct freelist *next; +}; + +struct large_object +{ + struct large_object *next; + size_t size; +}; + +#define LARGE_OBJECT_HEADER_SIZE (sizeof(struct large_object)) + +static inline void *get_large_object_payload(struct large_object *obj) +{ + return ((char *)obj) + LARGE_OBJECT_HEADER_SIZE; +} +static inline struct large_object *get_large_object(void *ptr) +{ + return (struct large_object *)(((char *)ptr) - LARGE_OBJECT_HEADER_SIZE); +} + +static struct freelist *small_object_freelists[SMALL_OBJECT_CHUNK_KINDS]; +static struct large_object *large_objects; + +extern void __heap_base; +static size_t walloc_heap_size; + +static struct page * +allocate_pages(size_t payload_size, size_t *n_allocated) +{ + size_t needed = payload_size + PAGE_HEADER_SIZE; + size_t heap_size = __builtin_wasm_memory_size(0) * PAGE_SIZE; + uintptr_t base = heap_size; + uintptr_t preallocated = 0, grow = 0; + + if (!walloc_heap_size) + { + // We are allocating the initial pages, if any. We skip the first 64 kB, + // then take any additional space up to the memory size. + uintptr_t heap_base = align((uintptr_t)&__heap_base, PAGE_SIZE); + preallocated = heap_size - heap_base; // Preallocated pages. + walloc_heap_size = preallocated; + base -= preallocated; + } + + if (preallocated < needed) + { + // Always grow the walloc heap at least by 50%. + grow = align(max(walloc_heap_size / 2, needed - preallocated), + PAGE_SIZE); + ASSERT(grow); + if (__builtin_wasm_memory_grow(0, grow >> PAGE_SIZE_LOG_2) == -1) + { + return NULL; + } + walloc_heap_size += grow; + } + + struct page *ret = (struct page *)base; + size_t size = grow + preallocated; + ASSERT(size); + ASSERT_ALIGNED(size, PAGE_SIZE); + *n_allocated = size / PAGE_SIZE; + return ret; +} + +static char * +allocate_chunk(struct page *page, unsigned idx, enum chunk_kind kind) +{ + page->header.chunk_kinds[idx] = kind; + return page->chunks[idx].data; +} + +// It's possible for splitting to produce a large object of size 248 (256 minus +// the header size) -- i.e. spanning a single chunk. In that case, push the +// chunk back on the GRANULES_32 small object freelist. +static void maybe_repurpose_single_chunk_large_objects_head(void) +{ + if (large_objects->size < CHUNK_SIZE) + { + unsigned idx = get_chunk_index(large_objects); + char *ptr = allocate_chunk(get_page(large_objects), idx, GRANULES_32); + large_objects = large_objects->next; + struct freelist *head = (struct freelist *)ptr; + head->next = small_object_freelists[GRANULES_32]; + small_object_freelists[GRANULES_32] = head; + } +} + +// If there have been any large-object frees since the last large object +// allocation, go through the freelist and merge any adjacent objects. +static int pending_large_object_compact = 0; +static struct large_object ** +maybe_merge_free_large_object(struct large_object **prev) +{ + struct large_object *obj = *prev; + while (1) + { + char *end = get_large_object_payload(obj) + obj->size; + ASSERT_ALIGNED((uintptr_t)end, CHUNK_SIZE); + unsigned chunk = get_chunk_index(end); + if (chunk < FIRST_ALLOCATABLE_CHUNK) + { + // Merging can't create a large object that newly spans the header chunk. + // This check also catches the end-of-heap case. + return prev; + } + struct page *page = get_page(end); + if (page->header.chunk_kinds[chunk] != FREE_LARGE_OBJECT) + { + return prev; + } + struct large_object *next = (struct large_object *)end; + + struct large_object **prev_prev = &large_objects, *walk = large_objects; + while (1) + { + ASSERT(walk); + if (walk == next) + { + obj->size += LARGE_OBJECT_HEADER_SIZE + walk->size; + *prev_prev = walk->next; + if (prev == &walk->next) + { + prev = prev_prev; + } + break; + } + prev_prev = &walk->next; + walk = walk->next; + } + } +} +static void +maybe_compact_free_large_objects(void) +{ + if (pending_large_object_compact) + { + pending_large_object_compact = 0; + struct large_object **prev = &large_objects; + while (*prev) + { + prev = &(*maybe_merge_free_large_object(prev))->next; + } + } +} + +// Allocate a large object with enough space for SIZE payload bytes. Returns a +// large object with a header, aligned on a chunk boundary, whose payload size +// may be larger than SIZE, and whose total size (header included) is +// chunk-aligned. Either a suitable allocation is found in the large object +// freelist, or we ask the OS for some more pages and treat those pages as a +// large object. If the allocation fits in that large object and there's more +// than an aligned chunk's worth of data free at the end, the large object is +// split. +// +// The return value's corresponding chunk in the page as starting a large +// object. +static struct large_object * +allocate_large_object(size_t size) +{ + maybe_compact_free_large_objects(); + struct large_object *best = NULL, **best_prev = &large_objects; + size_t best_size = -1; + for (struct large_object **prev = &large_objects, *walk = large_objects; + walk; + prev = &walk->next, walk = walk->next) + { + if (walk->size >= size && walk->size < best_size) + { + best_size = walk->size; + best = walk; + best_prev = prev; + if (best_size + LARGE_OBJECT_HEADER_SIZE == align(size + LARGE_OBJECT_HEADER_SIZE, CHUNK_SIZE)) + // Not going to do any better than this; just return it. + break; + } + } + + if (!best) + { + // The large object freelist doesn't have an object big enough for this + // allocation. Allocate one or more pages from the OS, and treat that new + // sequence of pages as a fresh large object. It will be split if + // necessary. + size_t size_with_header = size + sizeof(struct large_object); + size_t n_allocated = 0; + struct page *page = allocate_pages(size_with_header, &n_allocated); + if (!page) + { + return NULL; + } + char *ptr = allocate_chunk(page, FIRST_ALLOCATABLE_CHUNK, LARGE_OBJECT); + best = (struct large_object *)ptr; + size_t page_header = ptr - ((char *)page); + best->next = large_objects; + best->size = best_size = + n_allocated * PAGE_SIZE - page_header - LARGE_OBJECT_HEADER_SIZE; + ASSERT(best_size >= size_with_header); + } + + allocate_chunk(get_page(best), get_chunk_index(best), LARGE_OBJECT); + + struct large_object *next = best->next; + *best_prev = next; + + size_t tail_size = (best_size - size) & ~CHUNK_MASK; + if (tail_size) + { + // The best-fitting object has 1 or more aligned chunks free after the + // requested allocation; split the tail off into a fresh aligned object. + struct page *start_page = get_page(best); + char *start = get_large_object_payload(best); + char *end = start + best_size; + + if (start_page == get_page(end - tail_size - 1)) + { + // The allocation does not span a page boundary; yay. + ASSERT_ALIGNED((uintptr_t)end, CHUNK_SIZE); + } + else if (size < PAGE_SIZE - LARGE_OBJECT_HEADER_SIZE - CHUNK_SIZE) + { + // If the allocation itself smaller than a page, split off the head, then + // fall through to maybe split the tail. + ASSERT_ALIGNED((uintptr_t)end, PAGE_SIZE); + size_t first_page_size = PAGE_SIZE - (((uintptr_t)start) & PAGE_MASK); + struct large_object *head = best; + allocate_chunk(start_page, get_chunk_index(start), FREE_LARGE_OBJECT); + head->size = first_page_size; + head->next = large_objects; + large_objects = head; + + maybe_repurpose_single_chunk_large_objects_head(); + + struct page *next_page = start_page + 1; + char *ptr = allocate_chunk(next_page, FIRST_ALLOCATABLE_CHUNK, LARGE_OBJECT); + best = (struct large_object *)ptr; + best->size = best_size = best_size - first_page_size - CHUNK_SIZE - LARGE_OBJECT_HEADER_SIZE; + ASSERT(best_size >= size); + start = get_large_object_payload(best); + tail_size = (best_size - size) & ~CHUNK_MASK; + } + else + { + // A large object that spans more than one page will consume all of its + // tail pages. Therefore if the split traverses a page boundary, round up + // to page size. + ASSERT_ALIGNED((uintptr_t)end, PAGE_SIZE); + size_t first_page_size = PAGE_SIZE - (((uintptr_t)start) & PAGE_MASK); + size_t tail_pages_size = align(size - first_page_size, PAGE_SIZE); + size = first_page_size + tail_pages_size; + tail_size = best_size - size; + } + best->size -= tail_size; + + unsigned tail_idx = get_chunk_index(end - tail_size); + while (tail_idx < FIRST_ALLOCATABLE_CHUNK && tail_size) + { + // We would be splitting in a page header; don't do that. + tail_size -= CHUNK_SIZE; + tail_idx++; + } + + if (tail_size) + { + struct page *page = get_page(end - tail_size); + char *tail_ptr = allocate_chunk(page, tail_idx, FREE_LARGE_OBJECT); + struct large_object *tail = (struct large_object *)tail_ptr; + tail->next = large_objects; + tail->size = tail_size - LARGE_OBJECT_HEADER_SIZE; + ASSERT_ALIGNED((uintptr_t)(get_large_object_payload(tail) + tail->size), CHUNK_SIZE); + large_objects = tail; + + maybe_repurpose_single_chunk_large_objects_head(); + } + } + + ASSERT_ALIGNED((uintptr_t)(get_large_object_payload(best) + best->size), CHUNK_SIZE); + return best; +} + +static struct freelist * +obtain_small_objects(enum chunk_kind kind) +{ + struct freelist **whole_chunk_freelist = &small_object_freelists[GRANULES_32]; + void *chunk; + if (*whole_chunk_freelist) + { + chunk = *whole_chunk_freelist; + *whole_chunk_freelist = (*whole_chunk_freelist)->next; + } + else + { + chunk = allocate_large_object(0); + if (!chunk) + { + return NULL; + } + } + char *ptr = allocate_chunk(get_page(chunk), get_chunk_index(chunk), kind); + char *end = ptr + CHUNK_SIZE; + struct freelist *next = NULL; + size_t size = chunk_kind_to_granules(kind) * GRANULE_SIZE; + for (size_t i = size; i <= CHUNK_SIZE; i += size) + { + struct freelist *head = (struct freelist *)(end - i); + head->next = next; + next = head; + } + return next; +} + +static inline size_t size_to_granules(size_t size) +{ + return (size + GRANULE_SIZE - 1) >> GRANULE_SIZE_LOG_2; +} +static struct freelist **get_small_object_freelist(enum chunk_kind kind) +{ + ASSERT(kind < SMALL_OBJECT_CHUNK_KINDS); + return &small_object_freelists[kind]; +} + +static void * +allocate_small(enum chunk_kind kind) +{ + struct freelist **loc = get_small_object_freelist(kind); + if (!*loc) + { + struct freelist *freelist = obtain_small_objects(kind); + if (!freelist) + { + return NULL; + } + *loc = freelist; + } + struct freelist *ret = *loc; + *loc = ret->next; + return (void *)ret; +} + +static void * +allocate_large(size_t size) +{ + struct large_object *obj = allocate_large_object(size); + return obj ? get_large_object_payload(obj) : NULL; +} + +void * +malloc(size_t size) +{ + size_t granules = size_to_granules(size); + enum chunk_kind kind = granules_to_chunk_kind(granules); + return (kind == LARGE_OBJECT) ? allocate_large(size) : allocate_small(kind); +} + +void free(void *ptr) +{ + if (!ptr) + return; + struct page *page = get_page(ptr); + unsigned chunk = get_chunk_index(ptr); + uint8_t kind = page->header.chunk_kinds[chunk]; + if (kind == LARGE_OBJECT) + { + struct large_object *obj = get_large_object(ptr); + obj->next = large_objects; + large_objects = obj; + allocate_chunk(page, chunk, FREE_LARGE_OBJECT); + pending_large_object_compact = 1; + } + else + { + size_t granules = kind; + struct freelist **loc = get_small_object_freelist(granules); + struct freelist *obj = ptr; + obj->next = *loc; + *loc = obj; + } +} \ No newline at end of file diff --git a/vlib/builtin/wasm_bare/libc_impl.v b/vlib/builtin/wasm_bare/libc_impl.v index bbde0a5055..bd94945fbd 100644 --- a/vlib/builtin/wasm_bare/libc_impl.v +++ b/vlib/builtin/wasm_bare/libc_impl.v @@ -1,8 +1,13 @@ module builtin -import dlmalloc +//__global global_allocator dlmalloc.Dlmalloc -__global global_allocator dlmalloc.Dlmalloc +[unsafe] +pub fn __malloc(size usize) voidptr { + unsafe { + return malloc(int(size)) + } +} [unsafe] pub fn memcpy(dest &C.void, src &C.void, n usize) &C.void { @@ -16,12 +21,6 @@ pub fn memcpy(dest &C.void, src &C.void, n usize) &C.void { return unsafe { dest } } -[export: 'malloc'] -[unsafe] -fn __malloc(n usize) &C.void { - return unsafe { global_allocator.malloc(n) } -} - [unsafe] fn strlen(_s &C.void) usize { s := unsafe { &byte(_s) } @@ -106,14 +105,6 @@ fn memcmp(a &C.void, b &C.void, n usize) int { return 0 } -[export: 'free'] -[unsafe] -fn __free(ptr &C.void) { - unsafe { - global_allocator.free_(ptr) - } -} - fn vsprintf(str &char, format &char, ap &byte) int { panic('vsprintf(): string interpolation is not supported in `-freestanding`') } @@ -169,5 +160,5 @@ fn __qsort(base voidptr, nmemb usize, size usize, sort_cb FnSortCB) { } fn init_global_allocator() { - global_allocator = dlmalloc.new(get_wasm_allocator()) + // global_allocator = dlmalloc.new(get_wasm_allocator()) } diff --git a/vlib/builtin/wasm_bare/memory_management.v b/vlib/builtin/wasm_bare/memory_management.v index 65f400d2f3..8dd66b0b94 100644 --- a/vlib/builtin/wasm_bare/memory_management.v +++ b/vlib/builtin/wasm_bare/memory_management.v @@ -1,68 +1,5 @@ +// malloc/free implementation for freestanding webassembly target. We just use walloc at the moment module builtin -import dlmalloc - -// Corresponding intrinsic to wasm’s `memory.grow` instruction -// -// This function, when called, will attempt to grow the default linear memory by the specified delta of pages. -// The current WebAssembly page size is 65536 bytes (64 KB). If memory is successfully grown then the previous size of memory, in pages, is returned. -// If memory cannot be grown then -1 is returned. -// -// The argument mem is the numerical index of which memory to return the size of. Note that currently the WebAssembly specification only supports one memory, -// so it is required that zero is passed in. The argument is present to be forward-compatible with future WebAssembly revisions. -// If a nonzero argument is passed to this function it will currently unconditionally abort -fn C.__builtin_wasm_memory_grow(mem u32, delta usize) usize - -/// Corresponding intrinsic to wasm's `memory.size` instruction -/// -/// This function, when called, will return the current memory size in units of -/// pages. The current WebAssembly page size is 65536 bytes (64 KB). -fn C.__builtin_wasm_memory_size(mem u32) usize - -const page_size = 65536 - -fn system_alloc(_ voidptr, size usize) (voidptr, usize, u32) { - pages := size / page_size - prev := C.__builtin_wasm_memory_grow(0, pages) - if prev == -1 { - return voidptr(0), 0, 0 - } - return voidptr(prev * page_size), pages * page_size, 0 -} - -fn system_remap(_ voidptr, _ voidptr, _ usize, _ usize, _ bool) voidptr { - return voidptr(0) -} - -fn system_free_part(_ voidptr, _ voidptr, _ usize, _ usize) bool { - return false -} - -fn system_free(_ voidptr, _ voidptr, _ usize) bool { - return false -} - -fn system_allocates_zeros(_ voidptr) bool { - return false -} - -fn system_page_size(_ voidptr) usize { - return page_size -} - -fn system_can_release_part(_ voidptr, _ u32) bool { - return false -} - -fn get_wasm_allocator() dlmalloc.Allocator { - return dlmalloc.Allocator{ - alloc: system_alloc - remap: system_remap - free_part: system_free_part - free_: system_free - can_release_part: system_can_release_part - allocates_zeros: system_allocates_zeros - page_size: system_page_size - data: voidptr(0) - } -} +#flag -I @VEXEROOT/thirdparty/walloc/ +#include "walloc.c" diff --git a/vlib/v/builder/cc.v b/vlib/v/builder/cc.v index 25a1b913ff..5c90fdbc11 100644 --- a/vlib/v/builder/cc.v +++ b/vlib/v/builder/cc.v @@ -534,7 +534,7 @@ pub fn (mut v Builder) cc() { } ccompiler = 'xcrun --sdk iphoneos clang -isysroot $isysroot $arch' } else if v.pref.os == .wasm32 { - ccompiler = 'clang-12' + ccompiler = 'clang' } v.setup_ccompiler_options(ccompiler) v.build_thirdparty_obj_files()