Fix forking in multi-ractor scenario

When there are multiple Ractors, there is a chance that one Ractor
triggers GC when another Ractor has just stopped all MMTk GC work
threads in order to fork.  If this happens, the forking thread will be
blocking waiting for the Ractor sched barrier, and the Ractor that
triggered GC will be waiting for GC threads to finish GC, which will
never happen.

We move the invocations of `rb_gc_before_fork()` and
`rb_gc_after_fork()` inside the `RB_VM_LOCKING() { ... }` block in
`rb_fork_ruby`.  By doing this, either the forking Ractor will wait for
GC to finish before stopping GC worker threads, or the Ractor that
triggers GC cannot stop other threads until the forking ractor respawned
GC worker threads.

We also hold the VM lock when running finalizers when the VM is about to
exits.

Author: wks

mmtk_support.c

@@ -937,21 +937,25 @@ rb_mmtk_call_obj_free_for_each_on_exit(VALUE *objects, size_t len)
 void
 rb_mmtk_call_obj_free_on_exit(void)
 {
-    struct MMTk_RawVecOfObjRef registered_candidates = mmtk_get_all_obj_free_candidates();
-    rb_mmtk_call_obj_free_for_each_on_exit((VALUE*)registered_candidates.ptr, registered_candidates.len);
-    mmtk_free_raw_vec_of_obj_ref(registered_candidates);
-
-    rb_vm_t *vm = GET_VM();
-    rb_ractor_t *ractor;
-
-    ccan_list_for_each(&vm->ractor.set, ractor, vmlr_node) {
-        // ractor.set only contains blocking or running ractors
-        GC_ASSERT(rb_ractor_status_p(ractor, ractor_blocking) ||
-                  rb_ractor_status_p(ractor, ractor_running));
-        struct rb_mmtk_mutator_local *local = rb_mmtk_ractor_get_mutator_local(ractor);
-        struct rb_mmtk_values_buffer *buffer = &local->obj_free_candidates;
-        rb_mmtk_call_obj_free_for_each_on_exit(buffer->objects, buffer->len);
+    unsigned int lev = RB_GC_VM_LOCK();
+    {
+        struct MMTk_RawVecOfObjRef registered_candidates = mmtk_get_all_obj_free_candidates();
+        rb_mmtk_call_obj_free_for_each_on_exit((VALUE*)registered_candidates.ptr, registered_candidates.len);
+        mmtk_free_raw_vec_of_obj_ref(registered_candidates);
+
+        rb_vm_t *vm = GET_VM();
+        rb_ractor_t *ractor;
+
+        ccan_list_for_each(&vm->ractor.set, ractor, vmlr_node) {
+            // ractor.set only contains blocking or running ractors
+            GC_ASSERT(rb_ractor_status_p(ractor, ractor_blocking) ||
+                    rb_ractor_status_p(ractor, ractor_running));
+            struct rb_mmtk_mutator_local *local = rb_mmtk_ractor_get_mutator_local(ractor);
+            struct rb_mmtk_values_buffer *buffer = &local->obj_free_candidates;
+            rb_mmtk_call_obj_free_for_each_on_exit(buffer->objects, buffer->len);
+        }
     }
+    RB_GC_VM_UNLOCK(lev);
 }
 
 bool

process.c

@@ -1575,15 +1575,12 @@ after_exec(void)
 static void
 before_fork_ruby(void)
 {
-    rb_gc_before_fork();
     before_exec();
 }
 
 static void
 after_fork_ruby(rb_pid_t pid)
 {
-    rb_gc_after_fork(pid);
-
     if (pid == 0) {
         // child
         clear_pid_cache();
@@ -4138,8 +4135,10 @@ rb_fork_ruby(int *status)
         disable_child_handler_before_fork(&old);
 
         RB_VM_LOCKING() {
+            rb_gc_before_fork();
             child.pid = pid = rb_fork();
             child.error = err = errno;
+            rb_gc_after_fork(pid);
         }
 
         disable_child_handler_fork_parent(&old); /* yes, bad name */