You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
359 lines
11 KiB
359 lines
11 KiB
From bcab8c3cd877506de75f50e0f9ed98827ed554b0 Mon Sep 17 00:00:00 2001 |
|
From: Peter Zhu <peter@peterzhu.ca> |
|
Date: Tue, 23 Feb 2021 16:28:56 -0500 |
|
Subject: [PATCH] Use mmap for allocating heap pages |
|
|
|
--- |
|
configure.ac | 16 ++++ |
|
gc.c | 149 ++++++++++++++++++++++++++--------- |
|
test/ruby/test_gc_compact.rb | 41 ++++++---- |
|
3 files changed, 155 insertions(+), 51 deletions(-) |
|
|
|
diff --git a/configure.ac b/configure.ac |
|
index 2dcebdde9f..b1b190004d 100644 |
|
--- a/configure.ac |
|
+++ b/configure.ac |
|
@@ -1944,6 +1944,7 @@ AC_CHECK_FUNCS(memmem) |
|
AC_CHECK_FUNCS(mkfifo) |
|
AC_CHECK_FUNCS(mknod) |
|
AC_CHECK_FUNCS(mktime) |
|
+AC_CHECK_FUNCS(mmap) |
|
AC_CHECK_FUNCS(openat) |
|
AC_CHECK_FUNCS(pipe2) |
|
AC_CHECK_FUNCS(poll) |
|
@@ -2666,6 +2667,21 @@ main(int argc, char *argv[]) |
|
rb_cv_fork_with_pthread=yes)]) |
|
test x$rb_cv_fork_with_pthread = xyes || AC_DEFINE(CANNOT_FORK_WITH_PTHREAD) |
|
]) |
|
+ |
|
+AC_CHECK_HEADERS([sys/user.h]) |
|
+AS_IF([test "x$ac_cv_func_mmap" = xyes], [ |
|
+ AC_CACHE_CHECK([whether PAGE_SIZE is compile-time const], rb_cv_const_page_size, |
|
+ [malloc_headers=`sed -n '/MALLOC_HEADERS_BEGIN/,/MALLOC_HEADERS_END/p' ${srcdir}/gc.c` |
|
+ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$malloc_headers |
|
+ typedef char conftest_page[PAGE_SIZE]; |
|
+ ]], [[]])], |
|
+ [rb_cv_const_page_size=yes], |
|
+ [rb_cv_const_page_size=no])]) |
|
+]) |
|
+AS_IF([test "x$rb_cv_const_page_size" = xyes], |
|
+ [AC_DEFINE(HAVE_CONST_PAGE_SIZE, 1)], |
|
+ [AC_DEFINE(HAVE_CONST_PAGE_SIZE, 0)] |
|
+) |
|
} |
|
|
|
: "runtime section" && { |
|
diff --git a/gc.c b/gc.c |
|
index f6acf3e117..6f8e5f242d 100644 |
|
--- a/gc.c |
|
+++ b/gc.c |
|
@@ -32,6 +32,7 @@ |
|
#include <stdarg.h> |
|
#include <stdio.h> |
|
|
|
+/* MALLOC_HEADERS_BEGIN */ |
|
#ifndef HAVE_MALLOC_USABLE_SIZE |
|
# ifdef _WIN32 |
|
# define HAVE_MALLOC_USABLE_SIZE |
|
@@ -54,6 +55,12 @@ |
|
# endif |
|
#endif |
|
|
|
+#if !defined(PAGE_SIZE) && defined(HAVE_SYS_USER_H) |
|
+/* LIST_HEAD conflicts with sys/queue.h on macOS */ |
|
+# include <sys/user.h> |
|
+#endif |
|
+/* MALLOC_HEADERS_END */ |
|
+ |
|
#ifdef HAVE_SYS_TIME_H |
|
# include <sys/time.h> |
|
#endif |
|
@@ -821,6 +828,25 @@ enum { |
|
HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT), |
|
HEAP_PAGE_BITMAP_PLANES = 4 /* RGENGC: mark, unprotected, uncollectible, marking */ |
|
}; |
|
+#define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG) |
|
+#define HEAP_PAGE_SIZE HEAP_PAGE_ALIGN |
|
+ |
|
+#ifdef HAVE_MMAP |
|
+# if HAVE_CONST_PAGE_SIZE |
|
+/* If we have the HEAP_PAGE and it is a constant, then we can directly use it. */ |
|
+static const bool USE_MMAP_ALIGNED_ALLOC = (PAGE_SIZE <= HEAP_PAGE_SIZE); |
|
+# elif defined(PAGE_MAX_SIZE) && (PAGE_MAX_SIZE <= HEAP_PAGE_SIZE) |
|
+/* PAGE_SIZE <= HEAP_PAGE_SIZE */ |
|
+static const bool USE_MMAP_ALIGNED_ALLOC = true; |
|
+# else |
|
+/* Otherwise, fall back to determining if we can use mmap during runtime. */ |
|
+# define USE_MMAP_ALIGNED_ALLOC (use_mmap_aligned_alloc != false) |
|
+ |
|
+static bool use_mmap_aligned_alloc; |
|
+# endif |
|
+#elif !defined(__MINGW32__) && !defined(_WIN32) |
|
+static const bool USE_MMAP_ALIGNED_ALLOC = false; |
|
+#endif |
|
|
|
struct heap_page { |
|
short total_slots; |
|
@@ -1760,14 +1786,14 @@ heap_unlink_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *pag |
|
heap->total_slots -= page->total_slots; |
|
} |
|
|
|
-static void rb_aligned_free(void *ptr); |
|
+static void rb_aligned_free(void *ptr, size_t size); |
|
|
|
static void |
|
heap_page_free(rb_objspace_t *objspace, struct heap_page *page) |
|
{ |
|
heap_allocated_pages--; |
|
objspace->profile.total_freed_pages++; |
|
- rb_aligned_free(GET_PAGE_BODY(page->start)); |
|
+ rb_aligned_free(GET_PAGE_BODY(page->start), HEAP_PAGE_SIZE); |
|
free(page); |
|
} |
|
|
|
@@ -1819,7 +1845,7 @@ heap_page_allocate(rb_objspace_t *objspace) |
|
/* assign heap_page entry */ |
|
page = calloc1(sizeof(struct heap_page)); |
|
if (page == 0) { |
|
- rb_aligned_free(page_body); |
|
+ rb_aligned_free(page_body, HEAP_PAGE_SIZE); |
|
rb_memerror(); |
|
} |
|
|
|
@@ -3159,15 +3185,18 @@ Init_heap(void) |
|
{ |
|
rb_objspace_t *objspace = &rb_objspace; |
|
|
|
-#if defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE) |
|
- /* If Ruby's heap pages are not a multiple of the system page size, we |
|
- * cannot use mprotect for the read barrier, so we must disable automatic |
|
- * compaction. */ |
|
- int pagesize; |
|
- pagesize = (int)sysconf(_SC_PAGE_SIZE); |
|
- if ((HEAP_PAGE_SIZE % pagesize) != 0) { |
|
- ruby_enable_autocompact = 0; |
|
- } |
|
+#if defined(HAVE_MMAP) && !HAVE_CONST_PAGE_SIZE && !defined(PAGE_MAX_SIZE) |
|
+ /* Need to determine if we can use mmap at runtime. */ |
|
+# ifdef PAGE_SIZE |
|
+ /* If the PAGE_SIZE macro can be used. */ |
|
+ use_mmap_aligned_alloc = PAGE_SIZE <= HEAP_PAGE_SIZE; |
|
+# elif defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE) |
|
+ /* If we can use sysconf to determine the page size. */ |
|
+ use_mmap_aligned_alloc = sysconf(_SC_PAGE_SIZE) <= HEAP_PAGE_SIZE; |
|
+# else |
|
+ /* Otherwise we can't determine the system page size, so don't use mmap. */ |
|
+ use_mmap_aligned_alloc = FALSE; |
|
+# endif |
|
#endif |
|
|
|
objspace->next_object_id = INT2FIX(OBJ_ID_INITIAL); |
|
@@ -8533,6 +8562,14 @@ gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE |
|
|
|
/* For now, compact implies full mark / sweep, so ignore other flags */ |
|
if (RTEST(compact)) { |
|
+ /* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for |
|
+ * the read barrier, so we must disable compaction. */ |
|
+#if !defined(__MINGW32__) && !defined(_WIN32) |
|
+ if (!USE_MMAP_ALIGNED_ALLOC) { |
|
+ rb_raise(rb_eNotImpError, "Compaction isn't available on this platform"); |
|
+ } |
|
+#endif |
|
+ |
|
reason |= GPR_FLAG_COMPACT; |
|
} else { |
|
if (!RTEST(full_mark)) reason &= ~GPR_FLAG_FULL_MARK; |
|
@@ -9944,16 +9981,14 @@ gc_disable(rb_execution_context_t *ec, VALUE _) |
|
static VALUE |
|
gc_set_auto_compact(rb_execution_context_t *ec, VALUE _, VALUE v) |
|
{ |
|
-#if defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE) |
|
- /* If Ruby's heap pages are not a multiple of the system page size, we |
|
- * cannot use mprotect for the read barrier, so we must disable automatic |
|
- * compaction. */ |
|
- int pagesize; |
|
- pagesize = (int)sysconf(_SC_PAGE_SIZE); |
|
- if ((HEAP_PAGE_SIZE % pagesize) != 0) { |
|
+ /* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for |
|
+ * the read barrier, so we must disable automatic compaction. */ |
|
+#if !defined(__MINGW32__) && !defined(_WIN32) |
|
+ if (!USE_MMAP_ALIGNED_ALLOC) { |
|
rb_raise(rb_eNotImpError, "Automatic compaction isn't available on this platform"); |
|
} |
|
#endif |
|
+ |
|
ruby_enable_autocompact = RTEST(v); |
|
return v; |
|
} |
|
@@ -10350,22 +10385,54 @@ rb_aligned_malloc(size_t alignment, size_t size) |
|
#elif defined _WIN32 |
|
void *_aligned_malloc(size_t, size_t); |
|
res = _aligned_malloc(size, alignment); |
|
-#elif defined(HAVE_POSIX_MEMALIGN) |
|
- if (posix_memalign(&res, alignment, size) == 0) { |
|
- return res; |
|
+#else |
|
+ if (USE_MMAP_ALIGNED_ALLOC) { |
|
+ GC_ASSERT(alignment % sysconf(_SC_PAGE_SIZE) == 0); |
|
+ |
|
+ char *ptr = mmap(NULL, alignment + size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); |
|
+ if (ptr == MAP_FAILED) { |
|
+ return NULL; |
|
+ } |
|
+ |
|
+ char *aligned = ptr + alignment; |
|
+ aligned -= ((VALUE)aligned & (alignment - 1)); |
|
+ GC_ASSERT(aligned > ptr); |
|
+ GC_ASSERT(aligned <= ptr + alignment); |
|
+ |
|
+ size_t start_out_of_range_size = aligned - ptr; |
|
+ GC_ASSERT(start_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0); |
|
+ if (start_out_of_range_size > 0) { |
|
+ if (munmap(ptr, start_out_of_range_size)) { |
|
+ rb_bug("rb_aligned_malloc: munmap failed for start"); |
|
+ } |
|
+ } |
|
+ |
|
+ size_t end_out_of_range_size = alignment - start_out_of_range_size; |
|
+ GC_ASSERT(end_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0); |
|
+ if (end_out_of_range_size > 0) { |
|
+ if (munmap(aligned + size, end_out_of_range_size)) { |
|
+ rb_bug("rb_aligned_malloc: munmap failed for end"); |
|
+ } |
|
+ } |
|
+ |
|
+ res = (void *)aligned; |
|
} |
|
else { |
|
- return NULL; |
|
+# if defined(HAVE_POSIX_MEMALIGN) |
|
+ if (posix_memalign(&res, alignment, size) != 0) { |
|
+ return NULL; |
|
+ } |
|
+# elif defined(HAVE_MEMALIGN) |
|
+ res = memalign(alignment, size); |
|
+# else |
|
+ char* aligned; |
|
+ res = malloc(alignment + size + sizeof(void*)); |
|
+ aligned = (char*)res + alignment + sizeof(void*); |
|
+ aligned -= ((VALUE)aligned & (alignment - 1)); |
|
+ ((void**)aligned)[-1] = res; |
|
+ res = (void*)aligned; |
|
+# endif |
|
} |
|
-#elif defined(HAVE_MEMALIGN) |
|
- res = memalign(alignment, size); |
|
-#else |
|
- char* aligned; |
|
- res = malloc(alignment + size + sizeof(void*)); |
|
- aligned = (char*)res + alignment + sizeof(void*); |
|
- aligned -= ((VALUE)aligned & (alignment - 1)); |
|
- ((void**)aligned)[-1] = res; |
|
- res = (void*)aligned; |
|
#endif |
|
|
|
/* alignment must be a power of 2 */ |
|
@@ -10375,16 +10442,26 @@ rb_aligned_malloc(size_t alignment, size_t size) |
|
} |
|
|
|
static void |
|
-rb_aligned_free(void *ptr) |
|
+rb_aligned_free(void *ptr, size_t size) |
|
{ |
|
#if defined __MINGW32__ |
|
__mingw_aligned_free(ptr); |
|
#elif defined _WIN32 |
|
_aligned_free(ptr); |
|
-#elif defined(HAVE_MEMALIGN) || defined(HAVE_POSIX_MEMALIGN) |
|
- free(ptr); |
|
#else |
|
- free(((void**)ptr)[-1]); |
|
+ if (USE_MMAP_ALIGNED_ALLOC) { |
|
+ GC_ASSERT(size % sysconf(_SC_PAGE_SIZE) == 0); |
|
+ if (munmap(ptr, size)) { |
|
+ rb_bug("rb_aligned_free: munmap failed"); |
|
+ } |
|
+ } |
|
+ else { |
|
+# if defined(HAVE_POSIX_MEMALIGN) || defined(HAVE_MEMALIGN) |
|
+ free(ptr); |
|
+# else |
|
+ free(((void**)ptr)[-1]); |
|
+# endif |
|
+ } |
|
#endif |
|
} |
|
|
|
diff --git a/test/ruby/test_gc_compact.rb b/test/ruby/test_gc_compact.rb |
|
index 4a8cff33f4..f5cab55ba7 100644 |
|
--- a/test/ruby/test_gc_compact.rb |
|
+++ b/test/ruby/test_gc_compact.rb |
|
@@ -4,12 +4,32 @@ |
|
require 'etc' |
|
|
|
class TestGCCompact < Test::Unit::TestCase |
|
- class AutoCompact < Test::Unit::TestCase |
|
+ module SupportsCompact |
|
def setup |
|
skip "autocompact not supported on this platform" unless supports_auto_compact? |
|
super |
|
end |
|
|
|
+ private |
|
+ |
|
+ def supports_auto_compact? |
|
+ return true unless defined?(Etc::SC_PAGE_SIZE) |
|
+ |
|
+ begin |
|
+ return GC::INTERNAL_CONSTANTS[:HEAP_PAGE_SIZE] % Etc.sysconf(Etc::SC_PAGE_SIZE) == 0 |
|
+ rescue NotImplementedError |
|
+ rescue ArgumentError |
|
+ end |
|
+ |
|
+ true |
|
+ end |
|
+ end |
|
+ |
|
+ include SupportsCompact |
|
+ |
|
+ class AutoCompact < Test::Unit::TestCase |
|
+ include SupportsCompact |
|
+ |
|
def test_enable_autocompact |
|
before = GC.auto_compact |
|
GC.auto_compact = true |
|
@@ -59,26 +79,17 @@ def test_implicit_compaction_does_something |
|
ensure |
|
GC.auto_compact = before |
|
end |
|
- |
|
- private |
|
- |
|
- def supports_auto_compact? |
|
- return true unless defined?(Etc::SC_PAGE_SIZE) |
|
- |
|
- begin |
|
- return GC::INTERNAL_CONSTANTS[:HEAP_PAGE_SIZE] % Etc.sysconf(Etc::SC_PAGE_SIZE) == 0 |
|
- rescue NotImplementedError |
|
- rescue ArgumentError |
|
- end |
|
- |
|
- true |
|
- end |
|
end |
|
|
|
def os_page_size |
|
return true unless defined?(Etc::SC_PAGE_SIZE) |
|
end |
|
|
|
+ def setup |
|
+ skip "autocompact not supported on this platform" unless supports_auto_compact? |
|
+ super |
|
+ end |
|
+ |
|
def test_gc_compact_stats |
|
list = [] |
|
|
|
-- |
|
2.30.1 (Apple Git-130) |
|
|
|
|