Skip to content

Commit

Permalink
Load patches in static definition
Browse files Browse the repository at this point in the history
  • Loading branch information
Olivier Michaud committed Oct 17, 2024
1 parent f0e1b3a commit 8e5af2e
Show file tree
Hide file tree
Showing 5 changed files with 27 additions and 12 deletions.
2 changes: 2 additions & 0 deletions src/saturn_engine/worker_manager/api/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ def update_job(job_name: str) -> Json[UpdateResponse]:
def post_sync() -> Json[JobsSyncResponse]:
"""Create jobs that are due to be scheduled."""
with session_scope() as session:
# We reset static definition at each jobs sync
current_app.saturn.load_static_definition(session=session)
sync_jobs(
static_definitions=current_app.saturn.static_definitions,
session=session,
Expand Down
10 changes: 6 additions & 4 deletions src/saturn_engine/worker_manager/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,21 +24,23 @@ def static_definitions(self) -> StaticDefinitions:
return self._static_definitions

def load_static_definition(self, session: AnySession) -> None:
self._static_definitions = _load_static_definition(
session=session, config=self.config
)
static_definition = _load_static_definition(session=session, config=self.config)
if static_definition:
self._static_definitions = static_definition


def _load_static_definition(
config: WorkerManagerConfig, session: AnySession
) -> StaticDefinitions:
) -> StaticDefinitions | None:
"""
Static definitions contain objects defined in a declarative configuration:
- Inventories
- Topics
- Jobs
- JobDefinitions
"""
if not config.static_definitions_directories:
return None
patches = topologies_store.get_patches(session=session)
definitions = load_definitions_from_paths(
config.static_definitions_directories, patches=patches
Expand Down
17 changes: 11 additions & 6 deletions src/saturn_engine/worker_manager/server.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
from typing import Optional

from pytest import Config

from saturn_engine.config import default_config_with_env
from saturn_engine.database import create_all
from saturn_engine.database import scoped_session
Expand All @@ -12,19 +14,22 @@


def get_app(
config: Optional[dict] = None,
saturn_config: Config | None = None,
app_config: Optional[dict] = None,
) -> SaturnApp:
worker_manager_context = WorkerManagerContext(
config=default_config_with_env().c.worker_manager,
)
if saturn_config:
worker_manager_config = saturn_config.c.worker_manager
else:
worker_manager_config = default_config_with_env().c.worker_manager
worker_manager_context = WorkerManagerContext(config=worker_manager_config)

app = SaturnApp(
worker_manager_context,
__name__,
)

if config:
app.config.from_mapping(config)
if app_config:
app.config.from_mapping(app_config)

from .api.inventories import bp as bp_inventories
from .api.job_definitions import bp as bp_job_definitions
Expand Down
4 changes: 4 additions & 0 deletions tests/config.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from saturn_engine.config import Env
from saturn_engine.config import SaturnConfig
from saturn_engine.config import ServicesManagerConfig
from saturn_engine.config_definitions import WorkerManagerConfig


class config(SaturnConfig):
Expand All @@ -16,3 +17,6 @@ class services_manager(ServicesManagerConfig):

class job_state:
auto_flush = False

class worker_manager(WorkerManagerConfig):
static_definitions_directories = []
6 changes: 4 additions & 2 deletions tests/worker_manager/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from sqlalchemy.orm import Session

from saturn_engine import database
from saturn_engine.config import Config
from saturn_engine.core import JobId
from saturn_engine.core import api
from saturn_engine.models import Base
Expand Down Expand Up @@ -96,11 +97,12 @@ def fake_job_definition(


@pytest.fixture
def app() -> t.Iterator[SaturnApp]:
def app(config: Config) -> t.Iterator[SaturnApp]:
app = worker_manager_server.get_app(
config={
app_config={
"TESTING": True,
},
saturn_config=config,
)
with app.app_context():
Base.metadata.drop_all(bind=database.engine())
Expand Down

0 comments on commit 8e5af2e

Please sign in to comment.