Move more of the GC logic into the runtime.

5 files changed, 131 insertions, 11 deletions

diff --git a/src/rt/rust_internal.h b/src/rt/rust_internal.h
index c9a28946..fafc1924 100644
--- a/src/rt/rust_internal.h
+++ b/src/rt/rust_internal.h
@@ -595,6 +595,19 @@ struct frame_glue_fns {
     uintptr_t reloc_glue_off;
 };
 
+struct gc_alloc {
+    gc_alloc *prev;
+    gc_alloc *next;
+    uintptr_t ctrl_word;
+    uint8_t data[];
+    bool mark() {
+        if (ctrl_word & 1)
+            return false;
+        ctrl_word |= 1;
+        return true;
+    }
+};
+
 struct
 rust_task : public rc_base<rust_task>,
             public dom_owned<rust_task>,
@@ -604,7 +617,7 @@ rust_task : public rc_base<rust_task>,
     stk_seg *stk;
     uintptr_t runtime_sp;      // Runtime sp while task running.
     uintptr_t rust_sp;         // Saved sp when not running.
-    uintptr_t gc_alloc_chain;  // Linked list of GC allocations.
+    gc_alloc *gc_alloc_chain;  // Linked list of GC allocations.
     rust_dom *dom;
     rust_crate_cache *cache;
 
@@ -614,6 +627,8 @@ rust_task : public rc_base<rust_task>,
     uintptr_t* dptr;           // Rendezvous pointer for send/recv.
     rust_task *spawner;        // Parent-link.
     size_t idx;
+    size_t gc_alloc_thresh;
+    size_t gc_alloc_accum;
 
     // Wait queue for tasks waiting for this task.
     rust_wait_queue waiting_tasks;
@@ -633,6 +648,12 @@ rust_task : public rc_base<rust_task>,
     bool blocked_on(rust_cond *cond);
     bool dead();
 
+    void link_gc(gc_alloc *gcm);
+    void unlink_gc(gc_alloc *gcm);
+    void *malloc(size_t sz, type_desc *td=0);
+    void *realloc(void *data, size_t sz, bool gc_mem=false);
+    void free(void *p, bool gc_mem=false);
+
     const char *state_str();
     void transition(ptr_vec<rust_task> *svec, ptr_vec<rust_task> *dvec);
 
diff --git a/src/rt/rust_log.cpp b/src/rt/rust_log.cpp
index 102a2623..5cdf315c 100644
--- a/src/rt/rust_log.cpp
+++ b/src/rt/rust_log.cpp
@@ -21,6 +21,7 @@ static uint32_t read_type_bit_mask() {
         bits |= strstr(env_str, "dwarf") ? rust_log::DWARF : 0;
         bits |= strstr(env_str, "cache") ? rust_log::CACHE : 0;
         bits |= strstr(env_str, "timer") ? rust_log::TIMER : 0;
+        bits |= strstr(env_str, "gc") ? rust_log::GC : 0;
         bits |= strstr(env_str, "all") ? rust_log::ALL : 0;
     }
     return bits;
diff --git a/src/rt/rust_log.h b/src/rt/rust_log.h
index b0c5fbec..bd32c155 100644
--- a/src/rt/rust_log.h
+++ b/src/rt/rust_log.h
@@ -44,6 +44,7 @@ public:
         CACHE = 0x100,
         UPCALL = 0x200,
         TIMER = 0x400,
+        GC = 0x800,
         ALL = 0xffffffff
     };
 
diff --git a/src/rt/rust_task.cpp b/src/rt/rust_task.cpp
index 3f85f51a..bf92ba90 100644
--- a/src/rt/rust_task.cpp
+++ b/src/rt/rust_task.cpp
@@ -408,6 +408,84 @@ rust_task::dead()
 }
 
 void
+rust_task::link_gc(gc_alloc *gcm) {
+    I(dom, gcm->prev == NULL);
+    I(dom, gcm->next == NULL);
+    gcm->prev = NULL;
+    gcm->next = gc_alloc_chain;
+}
+
+void
+rust_task::unlink_gc(gc_alloc *gcm) {
+    if (gcm->prev)
+        gcm->prev->next = gcm->next;
+    if (gcm->next)
+        gcm->next->prev = gcm->prev;
+    gcm->prev = NULL;
+    gcm->next = NULL;
+}
+
+void *
+rust_task::malloc(size_t sz, type_desc *td)
+{
+    if (td) {
+        sz += sizeof(gc_alloc);
+    }
+    void *mem = dom->malloc(sz);
+    if (!mem)
+        return mem;
+    if (td) {
+        gc_alloc *gcm = (gc_alloc*) mem;
+        dom->log(rust_log::TASK|rust_log::MEM|rust_log::GC,
+                 "task 0x%" PRIxPTR " allocated %d GC bytes = 0x%" PRIxPTR,
+                 (uintptr_t)this, sz, gcm);
+        memset((void*) gcm, 0, sizeof(gc_alloc));
+        link_gc(gcm);
+        gcm->ctrl_word = (uintptr_t)td;
+        gc_alloc_accum += sz;
+        mem = (void*) &(gcm->data);
+    }
+    return mem;;
+}
+
+void *
+rust_task::realloc(void *data, size_t sz, bool is_gc)
+{
+    if (is_gc) {
+        gc_alloc *gcm = (gc_alloc*)(((char *)data) - sizeof(gc_alloc));
+        unlink_gc(gcm);
+        sz += sizeof(gc_alloc);
+        gcm = (gc_alloc*) dom->realloc((void*)gcm, sz);
+        dom->log(rust_log::TASK|rust_log::MEM|rust_log::GC,
+                 "task 0x%" PRIxPTR " reallocated %d GC bytes = 0x%" PRIxPTR,
+                 (uintptr_t)this, sz, gcm);
+        if (!gcm)
+            return gcm;
+        link_gc(gcm);
+        data = (void*) &(gcm->data);
+    } else {
+        data = dom->realloc(data, sz);
+    }
+    return data;
+}
+
+void
+rust_task::free(void *p, bool is_gc)
+{
+    if (is_gc) {
+        gc_alloc *gcm = (gc_alloc*)(((char *)p) - sizeof(gc_alloc));
+        unlink_gc(gcm);
+        dom->log(rust_log::TASK|rust_log::MEM|rust_log::GC,
+                 "task 0x%" PRIxPTR " freeing GC memory = 0x%" PRIxPTR,
+                 (uintptr_t)this, gcm);
+        dom->free(gcm);
+    } else {
+        dom->free(p);
+    }
+}
+
+
+void
 rust_task::transition(ptr_vec<rust_task> *src, ptr_vec<rust_task> *dst)
 {
     I(dom, state == src);
diff --git a/src/rt/rust_upcall.cpp b/src/rt/rust_upcall.cpp
index 3a17ea1c..ffe77532 100644
--- a/src/rt/rust_upcall.cpp
+++ b/src/rt/rust_upcall.cpp
@@ -324,19 +324,20 @@ upcall_exit(rust_task *task)
 }
 
 extern "C" CDECL uintptr_t
-upcall_malloc(rust_task *task, size_t nbytes)
+upcall_malloc(rust_task *task, size_t nbytes, type_desc *td)
 {
     LOG_UPCALL_ENTRY(task);
 
-    void *p = task->dom->malloc(nbytes);
+    void *p = task->malloc(nbytes, td);
     task->dom->log(rust_log::UPCALL|rust_log::MEM,
-                   "upcall malloc(%u) = 0x%" PRIxPTR,
-                   nbytes, (uintptr_t)p);
+                   "upcall malloc(%u) = 0x%" PRIxPTR
+                   " with gc-chain head = 0x%" PRIxPTR,
+                   nbytes, (uintptr_t)p, task->gc_alloc_chain);
     return (uintptr_t) p;
 }
 
 extern "C" CDECL void
-upcall_free(rust_task *task, void* ptr)
+upcall_free(rust_task *task, void* ptr, uintptr_t is_gc)
 {
     LOG_UPCALL_ENTRY(task);
 
@@ -344,7 +345,24 @@ upcall_free(rust_task *task, void* ptr)
     dom->log(rust_log::UPCALL|rust_log::MEM,
              "upcall free(0x%" PRIxPTR ")",
              (uintptr_t)ptr);
-    dom->free(ptr);
+    task->free(ptr, (bool) is_gc);
+}
+
+extern "C" CDECL uintptr_t
+upcall_mark(rust_task *task, void* ptr)
+{
+    LOG_UPCALL_ENTRY(task);
+
+    rust_dom *dom = task->dom;
+    if (ptr) {
+        gc_alloc *gcm = (gc_alloc*) (((char*)ptr) - sizeof(gc_alloc));
+        uintptr_t marked = (uintptr_t) gcm->mark();
+        dom->log(rust_log::UPCALL|rust_log::MEM|rust_log::GC,
+                 "upcall mark(0x%" PRIxPTR ") = %" PRIdPTR,
+                 (uintptr_t)gcm, marked);
+        return marked;
+    }
+    return 0;
 }
 
 extern "C" CDECL rust_str *
@@ -368,14 +386,15 @@ upcall_new_str(rust_task *task, char const *s, size_t fill)
 }
 
 extern "C" CDECL rust_vec *
-upcall_new_vec(rust_task *task, size_t fill)
+upcall_new_vec(rust_task *task, size_t fill, type_desc *td)
 {
     LOG_UPCALL_ENTRY(task);
     rust_dom *dom = task->dom;
     dom->log(rust_log::UPCALL|rust_log::MEM,
-             "upcall new_vec(%" PRIdPTR ")", fill);
+             "upcall new_vec(%" PRIdPTR ")",
+             fill);
     size_t alloc = next_power_of_two(sizeof(rust_vec) + fill);
-    void *mem = dom->malloc(alloc);
+    void *mem = task->malloc(alloc, td);
     if (!mem) {
         task->fail(3);
         return NULL;
@@ -389,7 +408,7 @@ upcall_new_vec(rust_task *task, size_t fill)
 
 
 extern "C" CDECL rust_str *
-upcall_vec_grow(rust_task *task, rust_vec *v, size_t n_bytes)
+upcall_vec_grow(rust_task *task, rust_vec *v, size_t n_bytes, uintptr_t is_gc)
 {
     LOG_UPCALL_ENTRY(task);
     rust_dom *dom = task->dom;