Add support for pinning actors to a dedicated scheduler thread

.release-notes/4547.md

@@ -0,0 +1,50 @@
+##  Add support for pinning actors to a dedicated scheduler thread
+
+Pony programmers can now pin actors to a dedicated  scheduler thread. This can be required/used for interfacing with C libraries that rely on thread local storage. A common example of this is graphics/windowing libraries.
+
+The way it works is that an actor can request that it be pinned (which may or may not happen immediately) and then it must wait and check to confirm that the pinning was successfully applied (prior to running any workload that required the actor to be pinned) after which all subsequent behaviors on that actor will run on the same scheduler thread until the actor is destroyed or the actor requests to be unpinned.
+
+### Caveat
+
+Due to the fact that Pony uses cooperative scheduling of actors and that all pinned actors run on a single shared scheduler thread, any "greedy" actors that monopolize the cpu (with long running behaviors) will negatively inmpact all other pinned actors by starving them of cpu.
+
+### Example program
+
+```pony
+// Here we have the Main actor that upon construction requests a PinUnpinActorAuth
+// token from AmbientAuth and then requests that it be pinned. It then recursively
+// calls the `check_pinned` behavior until the runtime reports that it has
+// successfully been pinned after which it starts `do_stuff` to do whatever
+// work it needs to do that requires it to be pinned. Once it has completed all
+// of its work, it calls `done` to request that the runtime `unpin` it.
+
+use "actor_pinning"
+
+actor Main
+  let _env: Env
+  let _auth: PinUnpinActorAuth
+
+  new create(env: Env) =>
+    _env = env
+    _auth = PinUnpinActorAuth(env.root)
+    ActorPinning.request_pin(_auth)
+    check_pinned()
+
+  be check_pinned() =>
+    if ActorPinning.is_successfully_pinned(_auth) then
+      // do stuff that requires this actor to be pinned
+      do_stuff(10)
+    else
+      check_pinned()
+    end
+
+  be do_stuff(i: I32) =>
+    if i < 0 then
+      done()
+    else
+      do_stuff(i - 1)
+    end
+
+  be done() =>
+    ActorPinning.request_unpin(_auth)
+```

packages/actor_pinning/actor_pinning.pony

@@ -0,0 +1,81 @@
+"""
+# Actor Pinning Package
+
+The Actor Pinning package allows Pony programmers to pin actors to a dedicated 
+scheduler thread. This can be required/used for interfacing with C libraries
+that rely on thread local storage. A common example of this is graphics/windowing
+libraries.
+
+The way it works is that an actor can request that it be pinned (which may or
+may not happen immediately) and then it must wait and check to confirm that the
+pinning was successfully applied (prior to running any workload that required the
+actor to be pinned) after which all subsequent behaviors on that actor will run
+on the same scheduler thread until the actor is destroyed or the actor requests
+to be unpinned.
+
+## Example program
+
+```pony
+// Here we have the Main actor that upon construction requests a PinUnpinActorAuth
+// token from AmbientAuth and then requests that it be pinned. It then recursively
+// calls the `check_pinned` behavior until the runtime reports that it has
+// successfully been pinned after which it starts `do_stuff` to do whatever
+// work it needs to do that requires it to be pinned. Once it has completed all
+// of its work, it calls `done` to request that the runtime `unpin` it.
+
+use "actor_pinning"
+
+actor Main
+  let _env: Env
+  let _auth: PinUnpinActorAuth
+
+  new create(env: Env) =>
+    _env = env
+    _auth = PinUnpinActorAuth(env.root)
+    ActorPinning.request_pin(_auth)
+    check_pinned()
+
+  be check_pinned() =>
+    if ActorPinning.is_successfully_pinned(_auth) then
+      // do stuff that requires this actor to be pinned
+      do_stuff(10)
+    else
+      check_pinned()
+    end
+
+  be do_stuff(i: I32) =>
+    if i < 0 then
+      done()
+    else
+      do_stuff(i - 1)
+    end
+
+  be done() =>
+    ActorPinning.request_unpin(_auth)
+```
+
+## Caveat
+
+Due to the fact that Pony uses cooperative scheduling of actors and that all
+pinned actors run on a single shared scheduler thread, any "greedy" actors that
+monopolize the cpu (with long running behaviors) will negatively inmpact all
+other pinned actors by starving them of cpu.
+"""
+
+use @pony_actor_set_pinned[None]()
+use @pony_actor_unset_pinned[None]()
+use @pony_scheduler_index[I32]()
+
+primitive ActorPinning
+  fun request_pin(auth: PinUnpinActorAuth) =>
+    @pony_actor_set_pinned()
+
+  fun request_unpin(auth: PinUnpinActorAuth) =>
+    @pony_actor_unset_pinned()
+
+  fun is_successfully_pinned(auth: PinUnpinActorAuth): Bool =>
+    let sched: I32 = @pony_scheduler_index()
+
+    // the `-999` constant is the same value as `PONY_PINNED_ACTOR_THREAD_INDEX`
+    // defined in `scheduler.h` in the runtime
+    sched == -999

packages/actor_pinning/auth.pony

@@ -0,0 +1,3 @@
+primitive PinUnpinActorAuth
+  new create(from: AmbientAuth) =>
+    None

packages/builtin/runtime_options.pony

@@ -104,6 +104,12 @@ struct RuntimeOptions
     Requires `--ponypin` to be set to have any effect.
     """
 
+  var ponypinpinnedactorthread: Bool = false
+    """
+    Pin the pinned actor thread to a CPU the way scheduler threads are pinned to CPUs.
+    Requires `--ponypin` to be set to have any effect.
+    """
+
   var ponyprintstatsinterval: U32 = -1
     """
     Print actor stats before an actor is destroyed and print scheduler stats

src/libponyrt/actor/actor.c

@@ -36,6 +36,7 @@ enum
   FLAG_UNSCHEDULED = 1 << 3,
   FLAG_CD_CONTACTED = 1 << 4,
   FLAG_RC_OVER_ZERO_SEEN = 1 << 5,
+  FLAG_PINNED = 1 << 6,
 };
 
 enum
@@ -1046,7 +1047,7 @@ PONY_API void pony_sendv_single(pony_ctx_t* ctx, pony_actor_t* to,
     {
       // if the receiving actor is currently not unscheduled AND it's not
       // muted, schedule it.
-      ponyint_sched_add_inject_or_sched(ctx, to);
+      ponyint_sched_add(ctx, to);
     }
   }
 }
@@ -1219,6 +1220,23 @@ PONY_API void pony_triggergc(pony_ctx_t* ctx)
   ctx->current->heap.next_gc = 0;
 }
 
+bool ponyint_actor_is_pinned(pony_actor_t* actor)
+{
+  return has_internal_flag(actor, FLAG_PINNED);
+}
+
+PONY_API void pony_actor_set_pinned()
+{
+  pony_ctx_t* ctx = pony_ctx();
+  set_internal_flag(ctx->current, FLAG_PINNED);
+}
+
+PONY_API void pony_actor_unset_pinned()
+{
+  pony_ctx_t* ctx = pony_ctx();
+  unset_internal_flag(ctx->current, FLAG_PINNED);
+}
+
 void ponyint_become(pony_ctx_t* ctx, pony_actor_t* actor)
 {
   ctx->current = actor;

src/libponyrt/actor/actor.h

@@ -130,6 +130,12 @@ gc_t* ponyint_actor_gc(pony_actor_t* actor);
 
 heap_t* ponyint_actor_heap(pony_actor_t* actor);
 
+bool ponyint_actor_is_pinned(pony_actor_t* actor);
+
+PONY_API void pony_actor_set_pinned();
+
+PONY_API void pony_actor_unset_pinned();
+
 bool ponyint_actor_pendingdestroy(pony_actor_t* actor);
 
 void ponyint_actor_setpendingdestroy(pony_actor_t* actor);

src/libponyrt/options/options.h

@@ -43,6 +43,10 @@
   "  --ponypinasio    Pin the ASIO thread to a CPU the way scheduler\n" \
   "                   threads are pinned to CPUs. Requires `--ponypin` to\n" \
   "                   be set to have any effect.\n" \
+  "  --ponypinpinnedactorthread\n" \
+  "                   Pin the pinned actor thread to a CPU the way scheduler\n" \
+  "                   threads are pinned to CPUs. Requires `--ponypin` to\n" \
+  "                   be set to have any effect.\n" \
   "  --ponyprintstatsinterval\n" \
   "                   Print actor stats before an actor is destroyed and\n" \
   "                   print scheduler stats every X seconds. Defaults to -1 (never).\n" \

src/libponyrt/sched/cpu.c

@@ -227,9 +227,10 @@ uint32_t ponyint_cpu_count()
 }
 
 uint32_t ponyint_cpu_assign(uint32_t count, scheduler_t* scheduler,
-  bool pin, bool pinasio)
+  bool pin, bool pinasio, bool pinpat)
 {
   uint32_t asio_cpu = -1;
+  uint32_t pat_cpu = -1;
 
   if(!pin)
   {
@@ -255,11 +256,12 @@ uint32_t ponyint_cpu_assign(uint32_t count, scheduler_t* scheduler,
   if(pinasio)
     asio_cpu = avail_cpu_list[count % avail_cpu_count];
 
+  if(pinpat)
+    pat_cpu = avail_cpu_list[(count + 1) % avail_cpu_count];
+
   ponyint_pool_free_size(avail_cpu_size * sizeof(uint32_t), avail_cpu_list);
   avail_cpu_list = NULL;
   avail_cpu_count = avail_cpu_size = 0;
-
-  return asio_cpu;
 #elif defined(PLATFORM_IS_BSD)
   // FreeBSD does not currently do thread pinning, as we can't yet determine
   // which cores are hyperthreads.
@@ -269,6 +271,9 @@ uint32_t ponyint_cpu_assign(uint32_t count, scheduler_t* scheduler,
   if(pinasio)
     asio_cpu = count % hw_cpu_count;
 
+  if(pinpat)
+    pat_cpu = (count + 1) % hw_cpu_count;
+
   for(uint32_t i = 0; i < count; i++)
   {
     scheduler[i].cpu = i % hw_cpu_count;
@@ -286,8 +291,15 @@ uint32_t ponyint_cpu_assign(uint32_t count, scheduler_t* scheduler,
   // asio_cpu of -1
   if(pinasio)
     asio_cpu = count;
+
+  if(pinpat)
+    pat_cpu = (count + 1);
 #endif
 
+  // set the affinity of the current thread (nain thread) which is the pinned
+  // actor thread
+  ponyint_cpu_affinity(pat_cpu);
+
   return asio_cpu;
 }
 

src/libponyrt/sched/cpu.h

@@ -13,7 +13,7 @@ void ponyint_cpu_init();
 uint32_t ponyint_cpu_count();
 
 uint32_t ponyint_cpu_assign(uint32_t count, scheduler_t* scheduler,
-  bool nopin, bool pinasio);
+  bool nopin, bool pinasio, bool pinpat);
 
 void ponyint_cpu_affinity(uint32_t cpu);