add readme for worker system

rslp/common/README.md

@@ -0,0 +1,27 @@
+This contains infrastructure intended to be shared across several rslp projects.
+
+
+Worker
+------
+
+`worker.py` provides a system for launching Beaker jobs that run workers to execute
+tasks from a queue.
+
+Each task specifies an rslp project, pipeline (workflow), and arguments to pass to the
+pipeline. The queue is implemented with Google Cloud Pub/Sub.
+
+First create the topic and subscription via CLI:
+
+    gcloud pubsub topics create --project skylight-proto-1 rslp-job-queue-YOURNAME
+    gcloud pubsub subscriptions create --project skylight-proto-1 rslp-job-queue-YOURNAME-sub --topic rslp-job-queue-YOURNAME
+
+You will then need to write code that writes tasks to the topic.
+See `satlas/write_jobs.py` for an example of this.
+
+Then you can launch the worker. To test on one machine:
+
+    python -m rslp.main common worker skylight-proto-1 rslp-job-queue-YOURNAME-sub
+
+And to launch 100 workers on Beaker:
+
+    python -m rslp.main common launch skylight-proto-1 rslp-job-queue-YOURNAME-sub 100 --gpus 1 --shared_memory 256GiB

rslp/satlas/__init__.py

@@ -7,10 +7,10 @@
 - Tree cover
 """
 
-from .job_launcher_worker import write_jobs, write_jobs_for_year_months
 from .postprocess import merge_points, smooth_points
 from .predict_pipeline import predict_multi, predict_pipeline
 from .publish import publish_points
+from .write_jobs import write_jobs, write_jobs_for_year_months
 
 workflows = {
     "predict": predict_pipeline,