From e7d01230b90af7569c1f3be3835d9d37c8bc0c90 Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 10 May 2023 10:41:38 +0200 Subject: [PATCH 001/123] big 5 draft --- big_5_draft.py | 223 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 big_5_draft.py diff --git a/big_5_draft.py b/big_5_draft.py new file mode 100644 index 00000000..765b6eab --- /dev/null +++ b/big_5_draft.py @@ -0,0 +1,223 @@ +class PayAllPaymentManager: + def __init__(self, budget, event_bus): + self._budget = budget + self._event_bus = event_bus + + self._allocation = Allocation.create(budget=self._budget) + + event_bus.register('InvoiceReceived', self.on_invoice_received) + event_bus.register('DebitNoteReceived', self.on_debit_note_received) + + def get_allocation(self) -> 'Allocation': + return self._allocation + + def on_invoice_received(self, invoice: 'Invoice') -> None: + invoice.pay() + + def on_debit_note_received(self, debit_note: 'DebitNote') -> None: + debit_note.pay() + + +class FifoOfferManager: + def __init__(self, get_allocation: 'Callable'): + self._proposals = [] + self.get_allocation = get_allocation + + def collect_proposals_for(self, payload) -> None: + allocation = self.get_allocation() + + demand_builder = DemandBuilder() + demand_builder.add(payload) + demand_builder.add(allocation) + demand = demand_builder.create_demand() + self._proposals = demand.initial_proposals() + demand.on_new_proposal(self.on_new_proposal) + + def on_new_proposal(self, proposal: 'Proposal') -> None: + self._proposals.append(proposal) + + def get_proposal(self) -> 'Proposal': + return self._proposals.pop() + + def get_offer(self) -> 'Offer': + while True: + provider_proposal = self.get_proposal() + our_response = provider_proposal.respond() + + try: + return our_response.wait_accept() + except Exception: + pass + + +class FifoAgreementManager: + def __init__(self, get_offer: 'Callable'): + self.get_offer = get_offer + + def get_agreement(self) -> 'Agreement': + while True: + offer = self.get_offer() + + try: + return offer.create_agrement() + except Exception: + pass + + +class PooledActivityManager: + def __init__(self, get_agreement: 'Callable', pool_size: int): + self.get_agreement = get_agreement + self._pool_size = pool_size + + self.activities = [] + + def get_activity(self) -> 'Activity': + while True: + # We need to release agreement if is not used + agreement = self.get_agreement() + try: + return agreement.create_activity() + except Exception: + pass + + def do_work(self, tasks: 'Iterable', before_all=None, after_all=None) -> None: + activities = [self.get_activity() for _ in range(self._pool_size)] + + + if before_all: + activity.do(before_all) + + for task in tasks: + try: + activity.do(task) + except Exception: + pass + + if after_all: + activity.do(after_all) + + +def blacklist_wrapper(get_offer, blacklist): + def wrapper(*args, **kwargs): + while True: + offer = get_offer() + + if offer in blacklist: + continue + + return offer + + return wrapper + +def restart_wrapper(): + pass + + +def redundance_wrapper(): + pass + + +def default_before_all(context): + context.deploy() + context.start() + + +def work(context): + context.run('echo hello world') + + +async def work_service(context): + context.run('app start deamon') + + while context.run('app check-if-running'): + sleep(1) + + +def default_after_all(context): + context.terminate() + + +def main(): + payload = RepositoryVmPayload(image_url='...') + budget = 1.0 + task_list = [ + work, + ] + + payment_manager = PayAllPaymentManager(budget) + offer_manager = FifoOfferManager(payment_manager.get_allocation) + offer_manager.collect_proposals_for(payload) + agreement_manager = FifoAgreementManager( + blacklist_wrapper(offer_manager.get_offer, ['banned_node_id']) + ) + activity_manager = PooledActivityManager(agreement_manager.get_agreement, pool_size=5) + activity_manager.do_work( + task_list, + before_all=default_before_all, + after_all=default_after_all + ) + + # Activity per task + for task in task_list: + with activity_manager.activity() as ctx: + default_before_all() + ctx.run(task) + default_after_all() + + # Single activity + with activity_manager.activity_context( + before_all=default_before_all, + after_all=default_after_all + ) as ctx: + for task in task_list: + ctx.run(task) + + #Activity pool + def pool_work(ctx): + for task in task_list: + yield ctx.run(task) + + activity_manager.activity_pool( + before_all_per_activity=default_before_all, + after_all_per_activity=default_after_all, + work=pool_work, + ) + + def activity_context(): + activity = self.get_activity() + if before_all: + activity.do(before_all) + + yield activity + + if after_all: + activity.do(after_all) + + def retry_failed_task_in_new_activity_plugin(get_task: 'Callable', retry_count_per_task: int): + activity = get_activity() + activity.before_all() + task = get_task() + tries = retry_count_per_task + while True: + try: + activity.do(task) + except Exception as e: + if not retry_count_per_task: + raise Exception('Number of tries exceeded!') from e + + retry_count_per_task -= 1 + activity.after_all() + activity = get_activity() + activity.before_all() + else: + task = get_task() + tries = retry_count_per_task + + activity.after_all() + + def redundance(get_task: 'Callable', redundance_size: int): + task = get_task() + + activities = [get_activity() for _ in range(redundance_size)] + for activity in activities: + activity.before_all() From 4a8d7506b5cad81a118aef388d03de22a350b733 Mon Sep 17 00:00:00 2001 From: approxit Date: Fri, 12 May 2023 08:32:43 +0200 Subject: [PATCH 002/123] activity manager flow --- big_5_draft.py | 238 +++++++++++++++++++++++++++---------------------- 1 file changed, 129 insertions(+), 109 deletions(-) diff --git a/big_5_draft.py b/big_5_draft.py index 765b6eab..22a6a253 100644 --- a/big_5_draft.py +++ b/big_5_draft.py @@ -5,8 +5,8 @@ def __init__(self, budget, event_bus): self._allocation = Allocation.create(budget=self._budget) - event_bus.register('InvoiceReceived', self.on_invoice_received) - event_bus.register('DebitNoteReceived', self.on_debit_note_received) + event_bus.register(InvoiceReceived(allocation=self._allocation), self.on_invoice_received) + event_bus.register(DebitNoteReceived(allocation=self._allocation), self.on_debit_note_received) def get_allocation(self) -> 'Allocation': return self._allocation @@ -19,7 +19,8 @@ def on_debit_note_received(self, debit_note: 'DebitNote') -> None: class FifoOfferManager: - def __init__(self, get_allocation: 'Callable'): + def __init__(self, get_allocation: 'Callable', event_bus): + self._event_bus = event_bus self._proposals = [] self.get_allocation = get_allocation @@ -31,7 +32,8 @@ def collect_proposals_for(self, payload) -> None: demand_builder.add(allocation) demand = demand_builder.create_demand() self._proposals = demand.initial_proposals() - demand.on_new_proposal(self.on_new_proposal) + + self._event_bus.register(ProposalReceived(demand=demand), self.on_new_proposal) def on_new_proposal(self, proposal: 'Proposal') -> None: self._proposals.append(proposal) @@ -64,12 +66,9 @@ def get_agreement(self) -> 'Agreement': pass -class PooledActivityManager: - def __init__(self, get_agreement: 'Callable', pool_size: int): +class SingleUseActivityManager: + def __init__(self, get_agreement: 'Callable'): self.get_agreement = get_agreement - self._pool_size = pool_size - - self.activities = [] def get_activity(self) -> 'Activity': while True: @@ -80,144 +79,165 @@ def get_activity(self) -> 'Activity': except Exception: pass - def do_work(self, tasks: 'Iterable', before_all=None, after_all=None) -> None: - activities = [self.get_activity() for _ in range(self._pool_size)] + def apply_activity_decorators(self, func, work): + if not hasattr(work, '_activity_decorators'): + return func + result = func + for dec in work._activity_decorators: + result = partial(dec, result) - if before_all: - activity.do(before_all) + return result - for task in tasks: - try: - activity.do(task) - except Exception: - pass - if after_all: - activity.do(after_all) + def do_work(self, work: 'Callable', on_activity_begin: 'Callable' = None, on_activity_end: 'Callable' = None) -> 'WorkResult': + activity = self.get_activity() + if on_activity_begin: + activity.do(on_activity_begin) + + decorated_do = self.apply_activity_decorators(activity.do, work) + + try: + return decorated_do(work) + except Exception: + pass + + if on_activity_end: + activity.do(on_activity_end) + + def do_work_list(self, work_list: 'List[Callable]', on_activity_begin: 'Callable' = None, on_activity_end: 'Callable' = None) -> 'List[WorkResult]': + results = [] + + for work in work_list: + results.append( + self.do_work(work, on_activity_begin, on_activity_end) + ) + + return results + + +def blacklist_offers(blacklist: 'List'): + def _blacklist_offers(func): + def wrapper(*args, **kwargs): + while True: + offer = func() + + if offer not in blacklist: + return offer + + return wrapper + + return _blacklist_offers + + +def retry(tries: int = 3): + def _retry(func): + def wrapper(work) -> 'WorkResult': + count = 0 + errors = [] + + while count <= tries: + try: + return func(work) + except Exception as err: + count += 1 + errors.append(err) + + raise errors # List[Exception] to Exception + + return wrapper + + return _retry -def blacklist_wrapper(get_offer, blacklist): - def wrapper(*args, **kwargs): - while True: - offer = get_offer() - if offer in blacklist: - continue +def redundancy_cancel_others_on_first_done(size: int = 3): + def _redundancy(func): + def wrapper(work): + tasks = [func(work) for _ in range(size)] - return offer + task_done, tasks_in_progress = return_on_first_done(tasks) - return wrapper + for task in tasks_in_progress: + task.cancel() -def restart_wrapper(): - pass + return task_done + return wrapper -def redundance_wrapper(): - pass + return _redundancy -def default_before_all(context): +def default_on_begin(context): context.deploy() context.start() +def default_on_end(context): + context.terminate() + + # After this function call we should have check for activity termination + + +def activity_decorator(dec, *args, **kwargs): + def _activity_decorator(func): + if not hasattr(func, '_activity_decorators'): + func._activity_decorators = [] + + func._activity_decorators.append(dec) + + return func + + return _activity_decorator + + +@activity_decorator(redundancy_cancel_others_on_first_done(size=5)) +@activity_decorator(retry(tries=5)) def work(context): context.run('echo hello world') async def work_service(context): - context.run('app start deamon') + context.run('app --daemon') + + +async def work_service_fetch(context): + context.run('app --daemon &') while context.run('app check-if-running'): sleep(1) -def default_after_all(context): - context.terminate() +async def work_batch(context): + script = context.create_batch() + script.add_run('echo 1') + script.add_run('echo 2') + script.add_run('echo 3') + await script.run() def main(): payload = RepositoryVmPayload(image_url='...') budget = 1.0 - task_list = [ + work_list = [ + work, + work, work, ] + event_bus = EventBus() + + payment_manager = PayAllPaymentManager(budget, event_bus) - payment_manager = PayAllPaymentManager(budget) - offer_manager = FifoOfferManager(payment_manager.get_allocation) + offer_manager = FifoOfferManager(payment_manager.get_allocation, event_bus) offer_manager.collect_proposals_for(payload) + agreement_manager = FifoAgreementManager( - blacklist_wrapper(offer_manager.get_offer, ['banned_node_id']) - ) - activity_manager = PooledActivityManager(agreement_manager.get_agreement, pool_size=5) - activity_manager.do_work( - task_list, - before_all=default_before_all, - after_all=default_after_all + blacklist_offers(['banned_node_id'])(offer_manager.get_offer) ) - # Activity per task - for task in task_list: - with activity_manager.activity() as ctx: - default_before_all() - ctx.run(task) - default_after_all() - - # Single activity - with activity_manager.activity_context( - before_all=default_before_all, - after_all=default_after_all - ) as ctx: - for task in task_list: - ctx.run(task) - - #Activity pool - def pool_work(ctx): - for task in task_list: - yield ctx.run(task) - - activity_manager.activity_pool( - before_all_per_activity=default_before_all, - after_all_per_activity=default_after_all, - work=pool_work, + activity_manager = SingleUseActivityManager(agreement_manager.get_agreement) + activity_manager.do_work_list( + work_list, + on_activity_begin=default_on_begin, + on_activity_end=default_on_end ) - - def activity_context(): - activity = self.get_activity() - if before_all: - activity.do(before_all) - - yield activity - - if after_all: - activity.do(after_all) - - def retry_failed_task_in_new_activity_plugin(get_task: 'Callable', retry_count_per_task: int): - activity = get_activity() - activity.before_all() - task = get_task() - tries = retry_count_per_task - while True: - try: - activity.do(task) - except Exception as e: - if not retry_count_per_task: - raise Exception('Number of tries exceeded!') from e - - retry_count_per_task -= 1 - activity.after_all() - activity = get_activity() - activity.before_all() - else: - task = get_task() - tries = retry_count_per_task - - activity.after_all() - - def redundance(get_task: 'Callable', redundance_size: int): - task = get_task() - - activities = [get_activity() for _ in range(redundance_size)] - for activity in activities: - activity.before_all() From 78c05dba17103b438659b613a043c55620486c0c Mon Sep 17 00:00:00 2001 From: approxit Date: Mon, 15 May 2023 15:32:19 +0200 Subject: [PATCH 003/123] activity manager fixed --- big_5_draft.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/big_5_draft.py b/big_5_draft.py index 22a6a253..1304805e 100644 --- a/big_5_draft.py +++ b/big_5_draft.py @@ -89,23 +89,28 @@ def apply_activity_decorators(self, func, work): return result - - def do_work(self, work: 'Callable', on_activity_begin: 'Callable' = None, on_activity_end: 'Callable' = None) -> 'WorkResult': + def _do_work(self, work, on_activity_begin: 'Callable' = None, on_activity_end: 'Callable' = None) -> 'WorkResult': activity = self.get_activity() if on_activity_begin: activity.do(on_activity_begin) - decorated_do = self.apply_activity_decorators(activity.do, work) - try: - return decorated_do(work) + result = activity.do(work) except Exception: pass if on_activity_end: activity.do(on_activity_end) + return result + + def do_work(self, work: 'Callable', on_activity_begin: 'Callable' = None, on_activity_end: 'Callable' = None) -> 'WorkResult': + decorated_do = self.apply_activity_decorators(self._do_work, work) + + return decorated_do(work, on_activity_begin, on_activity_end) + + def do_work_list(self, work_list: 'List[Callable]', on_activity_begin: 'Callable' = None, on_activity_end: 'Callable' = None) -> 'List[WorkResult]': results = [] @@ -172,11 +177,13 @@ def default_on_begin(context): context.deploy() context.start() + # After this function call we should have check if activity is actually started + def default_on_end(context): context.terminate() - # After this function call we should have check for activity termination + # After this function call we should have check if activity is actually terminated def activity_decorator(dec, *args, **kwargs): @@ -193,6 +200,7 @@ def _activity_decorator(func): @activity_decorator(redundancy_cancel_others_on_first_done(size=5)) @activity_decorator(retry(tries=5)) +# activity run here def work(context): context.run('echo hello world') From 31fb06d0ac9ad18289936dcb65f20c835284e187 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 16 May 2023 09:49:53 +0200 Subject: [PATCH 004/123] Add LifoOfferManager and ConfirmALLNegotiationManager --- .gitignore | 2 ++ big_5_draft.py | 58 +++++++++++++++++++++++++++++--------------------- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 35bf6868..64ca5e0a 100644 --- a/.gitignore +++ b/.gitignore @@ -166,3 +166,5 @@ build/ # licheck artifacts .requrements.txt + +temp/ diff --git a/big_5_draft.py b/big_5_draft.py index 1304805e..d176d33e 100644 --- a/big_5_draft.py +++ b/big_5_draft.py @@ -1,3 +1,6 @@ +from typing import List + + class PayAllPaymentManager: def __init__(self, budget, event_bus): self._budget = budget @@ -18,40 +21,43 @@ def on_debit_note_received(self, debit_note: 'DebitNote') -> None: debit_note.pay() -class FifoOfferManager: - def __init__(self, get_allocation: 'Callable', event_bus): +class ConfirmAllNegotiationManager: + def __init__(self, get_allocation: 'Callable', payload, event_bus): self._event_bus = event_bus - self._proposals = [] - self.get_allocation = get_allocation + self._get_allocation = get_allocation + self._allocation = self._get_allocation() + self._payload = payload + demand_builder = DemandBuilder() + demand_builder.add(self._payload) + demand_builder.add(self._allocation) + self._demand = demand_builder.create_demand() - def collect_proposals_for(self, payload) -> None: - allocation = self.get_allocation() + def negotiate(self): + for initial in self._demand.get_proposals(): # infinite loop + pending = initial.respond() + confirmed = pending.confirm() + self._event_bus.register(ProposalConfirmed(demand=self._demand, proposal=confirmed)) - demand_builder = DemandBuilder() - demand_builder.add(payload) - demand_builder.add(allocation) - demand = demand_builder.create_demand() - self._proposals = demand.initial_proposals() - self._event_bus.register(ProposalReceived(demand=demand), self.on_new_proposal) +class LifoOfferManager: + _offers: List['Offer'] - def on_new_proposal(self, proposal: 'Proposal') -> None: - self._proposals.append(proposal) + def __init__(self, event_bus) -> None: + self._event_bus = event_bus + self._event_bus.resource_listen(self.on_new_offer, ProposalConfirmed) - def get_proposal(self) -> 'Proposal': - return self._proposals.pop() + def on_new_offer(self, offer: 'Offer') -> None: + self._offers.append(offer) def get_offer(self) -> 'Offer': while True: - provider_proposal = self.get_proposal() - our_response = provider_proposal.respond() - try: - return our_response.wait_accept() - except Exception: + return self._offers.pop() + except IndexError: + # wait for offers + # await sleep pass - class FifoAgreementManager: def __init__(self, get_offer: 'Callable'): self.get_offer = get_offer @@ -236,8 +242,12 @@ def main(): payment_manager = PayAllPaymentManager(budget, event_bus) - offer_manager = FifoOfferManager(payment_manager.get_allocation, event_bus) - offer_manager.collect_proposals_for(payload) + negotiation_manager = ConfirmAllNegotiationManager( + payment_manager.get_allocation, payload, event_bus + ) + negotiation_manager.negotiate() # run in async, this will generate ProposalConfirmed events + + offer_manager = LifoOfferManager(event_bus) # listen to ProposalConfirmed agreement_manager = FifoAgreementManager( blacklist_offers(['banned_node_id'])(offer_manager.get_offer) From dc29cab4921bcefb478e46f2789bbb6e9152ae57 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 16 May 2023 09:59:47 +0200 Subject: [PATCH 005/123] Add FilterNegotiationManager --- big_5_draft.py | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/big_5_draft.py b/big_5_draft.py index d176d33e..2475cd30 100644 --- a/big_5_draft.py +++ b/big_5_draft.py @@ -38,6 +38,49 @@ def negotiate(self): confirmed = pending.confirm() self._event_bus.register(ProposalConfirmed(demand=self._demand, proposal=confirmed)) +def filter_blacklist(proposal: 'Proposal') -> bool: + providers_blacklist: List[str] = ... + return proposal.provider_id in providers_blacklist + +class FilterNegotiationManager: + INITIAL="INITIAL" + PENDING="PENDING" + + def __init__(self, get_allocation: 'Callable', payload, event_bus): + self._event_bus = event_bus + self._get_allocation = get_allocation + self._allocation = self._get_allocation() + self._payload = payload + demand_builder = DemandBuilder() + demand_builder.add(self._payload) + demand_builder.add(self._allocation) + self._demand = demand_builder.create_demand() + self._filters = { + self.INITIAL: [], + self.PENDING: [], + } + + def add_filter(self, filter: 'Filter', type: str): + self._filters[type].append(filter) + + def _filter(self, initial: 'Proposal', type: str) -> bool: + for f in self._filters[type]: + if f(initial): + return True + return False + + def negotiate(self): + for initial in self._demand.get_proposals(): # infinite loop + if self._filter(initial, self.INITIAL): + continue + + pending = initial.respond() + if self._filter(pending, self.PENDING): + pending.reject() + continue + + confirmed = pending.confirm() + self._event_bus.register(ProposalConfirmed(demand=self._demand, proposal=confirmed)) class LifoOfferManager: _offers: List['Offer'] @@ -242,9 +285,10 @@ def main(): payment_manager = PayAllPaymentManager(budget, event_bus) - negotiation_manager = ConfirmAllNegotiationManager( + negotiation_manager = FilterNegotiationManager( payment_manager.get_allocation, payload, event_bus ) + negotiation_manager.add_filter(filter_blacklist, negotiation_manager.INITIAL) negotiation_manager.negotiate() # run in async, this will generate ProposalConfirmed events offer_manager = LifoOfferManager(event_bus) # listen to ProposalConfirmed From 67db55d5a18b8bfc7a0375474024cebd74b6b95e Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 16 May 2023 10:19:22 +0200 Subject: [PATCH 006/123] AcceptableRangeNegotiationManager draft --- big_5_draft.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/big_5_draft.py b/big_5_draft.py index 2475cd30..423843d9 100644 --- a/big_5_draft.py +++ b/big_5_draft.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional class PayAllPaymentManager: @@ -82,6 +82,69 @@ def negotiate(self): confirmed = pending.confirm() self._event_bus.register(ProposalConfirmed(demand=self._demand, proposal=confirmed)) + +class AcceptableRangeNegotiationManager: + def __init__(self, get_allocation: 'Callable', payload, event_bus): + self._event_bus = event_bus + self._get_allocation = get_allocation + self._allocation = self._get_allocation() + self._payload = payload + demand_builder = DemandBuilder() + demand_builder.add(self._payload) + demand_builder.add(self._allocation) + self._demand = demand_builder.create_demand() + + def negotiate(self): + for initial in self._demand.get_proposals(): # infinite loop + pending = self._negotiate_for_accepted_range(initial) + if pending is None: + continue + confirmed = pending.confirm() + self._event_bus.register(ProposalConfirmed(demand=self._demand, proposal=confirmed)) + + def _validate_values(self, proposal: 'Proposal') -> bool: + """Checks if proposal's values are in accepted range + + e.g. + True + x_accepted range: [2,10] + proposal.x: 9 + + False + x_accepted range: [2,10] + proposal.x: 11 + """ + + def _middle_values(self, our: 'Proposal', their: 'Proposal') -> Optional['Proposal']: + """Create new proposal with new values in accepted range based on given proposals. + + If middle values are outside of accepted range return None + + e.g. + New proposal + x_accepted range: [2,10] + our.x: 5 + their.x : 13 + new: (5+13)//2 -> 9 + + None + x_accepted range: [2,10] + our.x: 9 + their.x : 13 + new: (9+13)//2 -> 11 -> None + """ + + def _negotiate_for_accepted_range(self, our: 'Proposal'): + their = our.respond() + while True: + if self._validate_values(their): + return their + our = self._middle_values(our, their) + if our is None: + return None + their = their.respond_with(our) + + class LifoOfferManager: _offers: List['Offer'] From 5769034b106686d8ee9562e830240a068e73969d Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 17 May 2023 10:06:53 +0200 Subject: [PATCH 007/123] more async --- big_5_draft.py | 260 +++++++++++++++++++++++++++++++------------------ 1 file changed, 166 insertions(+), 94 deletions(-) diff --git a/big_5_draft.py b/big_5_draft.py index 423843d9..e0cdcea0 100644 --- a/big_5_draft.py +++ b/big_5_draft.py @@ -1,5 +1,58 @@ -from typing import List, Optional +import asyncio +from abc import ABC +from functools import partial, wraps +from typing import List, Optional, Callable, Awaitable +from golem_core.core.market_api import RepositoryVmPayload + + +class Batch: + def deploy(self): + pass + + def start(self): + pass + + def terminate(self): + pass + + def run(self, command: str): + pass + + async def __call__(self): + pass + + +class WorkContext: + async def deploy(self): + pass + + async def start(self): + pass + + async def terminate(self): + pass + + async def run(self, command: str): + pass + + async def create_batch(self) -> Batch: + pass + + +class WorkResult: + pass + + +WorkDecorator = Callable[['DoWorkCallable'], 'DoWorkCallable'] + +class Work(ABC): + _work_decorators: Optional[List[WorkDecorator]] + + def __call__(self, context: WorkContext) -> Optional[WorkResult]: + pass + +DoWorkCallable = Callable[[Work], Awaitable[WorkResult]] class PayAllPaymentManager: def __init__(self, budget, event_bus): @@ -59,10 +112,10 @@ def __init__(self, get_allocation: 'Callable', payload, event_bus): self.INITIAL: [], self.PENDING: [], } - + def add_filter(self, filter: 'Filter', type: str): self._filters[type].append(filter) - + def _filter(self, initial: 'Proposal', type: str) -> bool: for f in self._filters[type]: if f(initial): @@ -78,11 +131,11 @@ def negotiate(self): if self._filter(pending, self.PENDING): pending.reject() continue - + confirmed = pending.confirm() self._event_bus.register(ProposalConfirmed(demand=self._demand, proposal=confirmed)) - + class AcceptableRangeNegotiationManager: def __init__(self, get_allocation: 'Callable', payload, event_bus): self._event_bus = event_bus @@ -105,7 +158,7 @@ def negotiate(self): def _validate_values(self, proposal: 'Proposal') -> bool: """Checks if proposal's values are in accepted range - e.g. + e.g. True x_accepted range: [2,10] proposal.x: 9 @@ -114,13 +167,13 @@ def _validate_values(self, proposal: 'Proposal') -> bool: x_accepted range: [2,10] proposal.x: 11 """ - + def _middle_values(self, our: 'Proposal', their: 'Proposal') -> Optional['Proposal']: """Create new proposal with new values in accepted range based on given proposals. - + If middle values are outside of accepted range return None - e.g. + e.g. New proposal x_accepted range: [2,10] our.x: 5 @@ -133,7 +186,7 @@ def _middle_values(self, our: 'Proposal', their: 'Proposal') -> Optional['Propos their.x : 13 new: (9+13)//2 -> 11 -> None """ - + def _negotiate_for_accepted_range(self, our: 'Proposal'): their = our.respond() while True: @@ -145,6 +198,20 @@ def _negotiate_for_accepted_range(self, our: 'Proposal'): their = their.respond_with(our) + +def blacklist_offers(blacklist: 'List'): + def _blacklist_offers(func): + def wrapper(*args, **kwargs): + while True: + offer = func() + + if offer not in blacklist: + return offer + + return wrapper + + return _blacklist_offers + class LifoOfferManager: _offers: List['Offer'] @@ -179,84 +246,78 @@ def get_agreement(self) -> 'Agreement': class SingleUseActivityManager: - def __init__(self, get_agreement: 'Callable'): - self.get_agreement = get_agreement + def __init__(self, get_agreement: 'Callable', on_activity_begin: Optional[Work] = None, on_activity_end: Optional[Work] = None): + self._get_agreement = get_agreement + self._on_activity_begin = on_activity_begin + self._on_activity_end = on_activity_end - def get_activity(self) -> 'Activity': + async def get_activity(self) -> 'Activity': while True: # We need to release agreement if is not used - agreement = self.get_agreement() + agreement = await self._get_agreement() try: - return agreement.create_activity() + return await agreement.create_activity() except Exception: pass - def apply_activity_decorators(self, func, work): - if not hasattr(work, '_activity_decorators'): - return func + async def do_work(self, work) -> WorkResult: + activity = await self.get_activity() - result = func - for dec in work._activity_decorators: - result = partial(dec, result) + if self._on_activity_begin: + await activity.do(self._on_activity_begin) + + try: + result = await activity.do(work) + except Exception as e: + result = WorkResult(exception=e) + + if self._on_activity_end: + await activity.do(self._on_activity_end) return result - def _do_work(self, work, on_activity_begin: 'Callable' = None, on_activity_end: 'Callable' = None) -> 'WorkResult': - activity = self.get_activity() - if on_activity_begin: - activity.do(on_activity_begin) +class SequentialWorkManager: + def __init__(self, do_work: DoWorkCallable): + self._do_work = do_work - try: - result = activity.do(work) - except Exception: - pass + def apply_activity_decorators(self, func: Callable[[Work], Awaitable[WorkResult]], work: Work) -> Callable[[Work], Awaitable[WorkResult]]: + if not hasattr(work, '_work_decorators'): + return func - if on_activity_end: - activity.do(on_activity_end) + result = func + for dec in work._work_decorators: + result = partial(dec, result) return result - def do_work(self, work: 'Callable', on_activity_begin: 'Callable' = None, on_activity_end: 'Callable' = None) -> 'WorkResult': - decorated_do = self.apply_activity_decorators(self._do_work, work) + async def do_work(self, work: Work) -> WorkResult: + decorated_do_work = self.apply_activity_decorators(self._do_work, work) - return decorated_do(work, on_activity_begin, on_activity_end) + return await decorated_do_work(work) - def do_work_list(self, work_list: 'List[Callable]', on_activity_begin: 'Callable' = None, on_activity_end: 'Callable' = None) -> 'List[WorkResult]': + async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: results = [] for work in work_list: results.append( - self.do_work(work, on_activity_begin, on_activity_end) + await self.do_work(work) ) return results -def blacklist_offers(blacklist: 'List'): - def _blacklist_offers(func): - def wrapper(*args, **kwargs): - while True: - offer = func() - - if offer not in blacklist: - return offer - - return wrapper - - return _blacklist_offers - - def retry(tries: int = 3): - def _retry(func): - def wrapper(work) -> 'WorkResult': + def _retry(do_work: DoWorkCallable) -> DoWorkCallable: + @wraps(do_work) + async def wrapper(work: Work) -> WorkResult: count = 0 errors = [] while count <= tries: try: - return func(work) + return await do_work(work) except Exception as err: count += 1 errors.append(err) @@ -269,75 +330,84 @@ def wrapper(work) -> 'WorkResult': def redundancy_cancel_others_on_first_done(size: int = 3): - def _redundancy(func): - def wrapper(work): - tasks = [func(work) for _ in range(size)] + def _redundancy(do_work: DoWorkCallable): + @wraps(do_work) + async def wrapper(work: Work) -> WorkResult: + tasks = [do_work(work) for _ in range(size)] - task_done, tasks_in_progress = return_on_first_done(tasks) + tasks_done, tasks_pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) - for task in tasks_in_progress: + for task in tasks_pending: task.cancel() - return task_done + for task in tasks_pending: + try: + await task + except asyncio.CancelledError: + pass + + return tasks_done.pop().result() return wrapper return _redundancy -def default_on_begin(context): - context.deploy() - context.start() +async def default_on_activity_begin(context: WorkContext): + batch = await context.create_batch() + batch.deploy() + batch.start() + await batch() # After this function call we should have check if activity is actually started -def default_on_end(context): - context.terminate() +async def default_on_activity_end(context: WorkContext): + await context.terminate() # After this function call we should have check if activity is actually terminated -def activity_decorator(dec, *args, **kwargs): - def _activity_decorator(func): - if not hasattr(func, '_activity_decorators'): - func._activity_decorators = [] +def work_decorator(decorator: WorkDecorator): + def _work_decorator(work: Work): + if not hasattr(work, '_work_decorators'): + work._work_decorators = [] - func._activity_decorators.append(dec) + work._work_decorators.append(decorator) - return func + return work - return _activity_decorator + return _work_decorator -@activity_decorator(redundancy_cancel_others_on_first_done(size=5)) -@activity_decorator(retry(tries=5)) -# activity run here -def work(context): - context.run('echo hello world') +@work_decorator(redundancy_cancel_others_on_first_done(size=5)) +@work_decorator(retry(tries=5)) +async def work(context: WorkContext): + await context.run('echo hello world') -async def work_service(context): - context.run('app --daemon') +async def work_service(context: WorkContext): + await context.run('app --daemon') -async def work_service_fetch(context): - context.run('app --daemon &') +async def work_service_fetch(context: WorkContext): + await context.run('app --daemon &') - while context.run('app check-if-running'): - sleep(1) + while await context.run('app check-if-running'): + await asyncio.sleep(1) -async def work_batch(context): - script = context.create_batch() - script.add_run('echo 1') - script.add_run('echo 2') - script.add_run('echo 3') - await script.run() +async def work_batch(context: WorkContext): + batch = await context.create_batch() + batch.run('echo 1') + batch.run('echo 2') + batch.run('echo 3') + + await batch() def main(): - payload = RepositoryVmPayload(image_url='...') + payload = RepositoryVmPayload(image_hash='...') budget = 1.0 work_list = [ work, @@ -360,9 +430,11 @@ def main(): blacklist_offers(['banned_node_id'])(offer_manager.get_offer) ) - activity_manager = SingleUseActivityManager(agreement_manager.get_agreement) - activity_manager.do_work_list( - work_list, - on_activity_begin=default_on_begin, - on_activity_end=default_on_end + activity_manager = SingleUseActivityManager( + agreement_manager.get_agreement, + on_activity_begin=default_on_activity_begin, + on_activity_end=default_on_activity_end, ) + + work_manager = SequentialWorkManager(activity_manager.do_work) + await work_manager.do_work_list(work_list) From a5a28704bd0df93e9c655b1838adea37ed945a57 Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 17 May 2023 11:06:28 +0200 Subject: [PATCH 008/123] typings --- big_5_draft.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/big_5_draft.py b/big_5_draft.py index e0cdcea0..fa09d8e0 100644 --- a/big_5_draft.py +++ b/big_5_draft.py @@ -244,6 +244,8 @@ def get_agreement(self) -> 'Agreement': except Exception: pass + # TODO: Close agreement + class SingleUseActivityManager: def __init__(self, get_agreement: 'Callable', on_activity_begin: Optional[Work] = None, on_activity_end: Optional[Work] = None): @@ -281,18 +283,18 @@ class SequentialWorkManager: def __init__(self, do_work: DoWorkCallable): self._do_work = do_work - def apply_activity_decorators(self, func: Callable[[Work], Awaitable[WorkResult]], work: Work) -> Callable[[Work], Awaitable[WorkResult]]: + def apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: if not hasattr(work, '_work_decorators'): - return func + return do_work - result = func + result = do_work for dec in work._work_decorators: result = partial(dec, result) return result async def do_work(self, work: Work) -> WorkResult: - decorated_do_work = self.apply_activity_decorators(self._do_work, work) + decorated_do_work = self.apply_work_decorators(self._do_work, work) return await decorated_do_work(work) From 0c1264331ea79ececbc078bae36692bab07d8dcd Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Fri, 19 May 2023 11:37:04 +0200 Subject: [PATCH 009/123] Add AlfaNegotiationManager and StackOfferManager --- examples/managers/example01.py | 32 ++++++++++++++ golem_core/managers/negotiation.py | 69 ++++++++++++++++++++++++++++++ golem_core/managers/offer.py | 28 ++++++++++++ 3 files changed, 129 insertions(+) create mode 100644 examples/managers/example01.py create mode 100644 golem_core/managers/negotiation.py create mode 100644 golem_core/managers/offer.py diff --git a/examples/managers/example01.py b/examples/managers/example01.py new file mode 100644 index 00000000..c8e431c0 --- /dev/null +++ b/examples/managers/example01.py @@ -0,0 +1,32 @@ +import asyncio +from datetime import datetime + +from golem_core.core.golem_node.golem_node import GolemNode +from golem_core.core.market_api import RepositoryVmPayload +from golem_core.managers.negotiation import AlfaNegotiationManager +from golem_core.managers.offer import StackOfferManager + + +def get_allocation_factory(golem: GolemNode): + async def _get_allocation(): + return await golem.create_allocation(1) + + return _get_allocation + + +async def main(): + payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") + async with GolemNode() as golem: + offer_manager = StackOfferManager(golem) + negotiation_manager = AlfaNegotiationManager(golem, get_allocation_factory(golem)) + await negotiation_manager.start_negotiation(payload) + + for _ in range(20): + print(f"{datetime.utcnow()} sleeping...") + await asyncio.sleep(1) + print(f"{datetime.utcnow()} stopping negotiations...") + await negotiation_manager.stop_negotiation() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/golem_core/managers/negotiation.py b/golem_core/managers/negotiation.py new file mode 100644 index 00000000..b20aa603 --- /dev/null +++ b/golem_core/managers/negotiation.py @@ -0,0 +1,69 @@ +import asyncio +from datetime import datetime, timezone +from typing import Awaitable, List + +from golem_core.core.events import EventBus +from golem_core.core.golem_node.golem_node import DEFAULT_EXPIRATION_TIMEOUT, SUBNET, GolemNode +from golem_core.core.market_api import Demand, DemandBuilder, Payload +from golem_core.core.market_api.resources.demand.demand_offer_base import defaults as dobm_defaults +from golem_core.core.payment_api import Allocation +from golem_core.core.resources import ResourceEvent + + +class AlfaNegotiationManager: + def __init__(self, golem: GolemNode, get_allocation: Awaitable) -> None: + self._golem = golem + self._event_bus: EventBus = self._golem.event_bus + self._get_allocation = get_allocation + self._negotiations: List[asyncio.Task] = [] + + async def start_negotiation(self, payload: Payload) -> None: + allocation: Allocation = await self._get_allocation() + + demand_builder = DemandBuilder() + + await demand_builder.add( + dobm_defaults.Activity( + expiration=datetime.now(timezone.utc) + DEFAULT_EXPIRATION_TIMEOUT, + multi_activity=True, + ) + ) + await demand_builder.add(dobm_defaults.NodeInfo(subnet_tag=SUBNET)) + + await demand_builder.add(payload) + + ( + allocation_properties, + allocation_constraints, + ) = await allocation.demand_properties_constraints() + demand_builder.add_constraints(*allocation_constraints) + demand_builder.add_properties({p.key: p.value for p in allocation_properties}) + + demand = await demand_builder.create_demand(self._golem) + demand.start_collecting_events() + self._negotiations.append(asyncio.create_task(self._negotiate(demand))) + + async def stop_negotiation(self) -> None: + for task in self._negotiations: + task.cancel() + + async def _negotiate(self, demand: Demand) -> None: + try: + async for initial in demand.initial_proposals(): + try: + pending = await initial.respond() + except Exception as err: + print( + f"Unable to respond to initial proposal {initial.id}. Got {type(err)}\n{err}" + ) + continue + + try: + # TODO IDK how to call `confirm` on a proposal in golem-core + confirmed = await pending.responses().__anext__() + except StopAsyncIteration: + continue + + self._event_bus.emit(ResourceEvent(confirmed)) + finally: + self._golem.add_autoclose_resource(demand) diff --git a/golem_core/managers/offer.py b/golem_core/managers/offer.py new file mode 100644 index 00000000..66b60180 --- /dev/null +++ b/golem_core/managers/offer.py @@ -0,0 +1,28 @@ +import asyncio +from typing import List + +from golem_core.core.golem_node import GolemNode +from golem_core.core.market_api import Proposal +from golem_core.core.resources import ResourceEvent + +Offer = Proposal + + +class StackOfferManager: + def __init__(self, golem: GolemNode) -> None: + self._offers: List[Offer] = [] + self._event_bus = golem.event_bus + self._event_bus.resource_listen(self._on_new_offer, (ResourceEvent,), (Offer,)) + + async def _on_new_offer(self, offer_event: ResourceEvent) -> None: + self._offers.append(offer_event.resource) + + async def get_offer(self) -> Offer: + # TODO add some timeout + while True: + try: + return self._offers.pop() + except IndexError: + # wait for offers + await asyncio.sleep(1) + pass From a4405f62d54f216bde951e034ed6c72ef4ae6e73 Mon Sep 17 00:00:00 2001 From: approxit Date: Fri, 19 May 2023 17:39:39 +0200 Subject: [PATCH 010/123] Partial split to separate modules --- big_5_draft.py | 291 ++++----------------- golem_core/managers/activity/__init__.py | 8 + golem_core/managers/activity/defaults.py | 16 ++ golem_core/managers/activity/single_use.py | 44 ++++ golem_core/managers/base.py | 95 +++++++ golem_core/managers/offer.py | 1 - golem_core/managers/payment/pay_all.py | 23 ++ golem_core/managers/work/__init__.py | 13 + golem_core/managers/work/decorators.py | 63 +++++ golem_core/managers/work/sequential.py | 32 +++ 10 files changed, 344 insertions(+), 242 deletions(-) create mode 100644 golem_core/managers/activity/__init__.py create mode 100644 golem_core/managers/activity/defaults.py create mode 100644 golem_core/managers/activity/single_use.py create mode 100644 golem_core/managers/base.py create mode 100644 golem_core/managers/payment/pay_all.py create mode 100644 golem_core/managers/work/__init__.py create mode 100644 golem_core/managers/work/decorators.py create mode 100644 golem_core/managers/work/sequential.py diff --git a/big_5_draft.py b/big_5_draft.py index fa09d8e0..fdc8f35c 100644 --- a/big_5_draft.py +++ b/big_5_draft.py @@ -1,81 +1,20 @@ import asyncio -from abc import ABC -from functools import partial, wraps -from typing import List, Optional, Callable, Awaitable +from typing import Callable, List, Optional from golem_core.core.market_api import RepositoryVmPayload - - -class Batch: - def deploy(self): - pass - - def start(self): - pass - - def terminate(self): - pass - - def run(self, command: str): - pass - - async def __call__(self): - pass - - -class WorkContext: - async def deploy(self): - pass - - async def start(self): - pass - - async def terminate(self): - pass - - async def run(self, command: str): - pass - - async def create_batch(self) -> Batch: - pass - - -class WorkResult: - pass - - -WorkDecorator = Callable[['DoWorkCallable'], 'DoWorkCallable'] - -class Work(ABC): - _work_decorators: Optional[List[WorkDecorator]] - - def __call__(self, context: WorkContext) -> Optional[WorkResult]: - pass - -DoWorkCallable = Callable[[Work], Awaitable[WorkResult]] - -class PayAllPaymentManager: - def __init__(self, budget, event_bus): - self._budget = budget - self._event_bus = event_bus - - self._allocation = Allocation.create(budget=self._budget) - - event_bus.register(InvoiceReceived(allocation=self._allocation), self.on_invoice_received) - event_bus.register(DebitNoteReceived(allocation=self._allocation), self.on_debit_note_received) - - def get_allocation(self) -> 'Allocation': - return self._allocation - - def on_invoice_received(self, invoice: 'Invoice') -> None: - invoice.pay() - - def on_debit_note_received(self, debit_note: 'DebitNote') -> None: - debit_note.pay() +from golem_core.managers.activity.single_use import SingleUseActivityManager +from golem_core.managers.base import WorkContext +from golem_core.managers.payment.pay_all import PayAllPaymentManager +from golem_core.managers.work.decorators import ( + redundancy_cancel_others_on_first_done, + retry, + work_decorator, +) +from golem_core.managers.work.sequential import SequentialWorkManager class ConfirmAllNegotiationManager: - def __init__(self, get_allocation: 'Callable', payload, event_bus): + def __init__(self, get_allocation: "Callable", payload, event_bus): self._event_bus = event_bus self._get_allocation = get_allocation self._allocation = self._get_allocation() @@ -86,20 +25,22 @@ def __init__(self, get_allocation: 'Callable', payload, event_bus): self._demand = demand_builder.create_demand() def negotiate(self): - for initial in self._demand.get_proposals(): # infinite loop + for initial in self._demand.get_proposals(): # infinite loop pending = initial.respond() confirmed = pending.confirm() self._event_bus.register(ProposalConfirmed(demand=self._demand, proposal=confirmed)) -def filter_blacklist(proposal: 'Proposal') -> bool: + +def filter_blacklist(proposal: "Proposal") -> bool: providers_blacklist: List[str] = ... return proposal.provider_id in providers_blacklist + class FilterNegotiationManager: - INITIAL="INITIAL" - PENDING="PENDING" + INITIAL = "INITIAL" + PENDING = "PENDING" - def __init__(self, get_allocation: 'Callable', payload, event_bus): + def __init__(self, get_allocation: "Callable", payload, event_bus): self._event_bus = event_bus self._get_allocation = get_allocation self._allocation = self._get_allocation() @@ -113,17 +54,17 @@ def __init__(self, get_allocation: 'Callable', payload, event_bus): self.PENDING: [], } - def add_filter(self, filter: 'Filter', type: str): + def add_filter(self, filter: "Filter", type: str): self._filters[type].append(filter) - def _filter(self, initial: 'Proposal', type: str) -> bool: + def _filter(self, initial: "Proposal", type: str) -> bool: for f in self._filters[type]: if f(initial): return True return False def negotiate(self): - for initial in self._demand.get_proposals(): # infinite loop + for initial in self._demand.get_proposals(): # infinite loop if self._filter(initial, self.INITIAL): continue @@ -137,7 +78,7 @@ def negotiate(self): class AcceptableRangeNegotiationManager: - def __init__(self, get_allocation: 'Callable', payload, event_bus): + def __init__(self, get_allocation: "Callable", payload, event_bus): self._event_bus = event_bus self._get_allocation = get_allocation self._allocation = self._get_allocation() @@ -148,14 +89,14 @@ def __init__(self, get_allocation: 'Callable', payload, event_bus): self._demand = demand_builder.create_demand() def negotiate(self): - for initial in self._demand.get_proposals(): # infinite loop + for initial in self._demand.get_proposals(): # infinite loop pending = self._negotiate_for_accepted_range(initial) if pending is None: continue confirmed = pending.confirm() self._event_bus.register(ProposalConfirmed(demand=self._demand, proposal=confirmed)) - def _validate_values(self, proposal: 'Proposal') -> bool: + def _validate_values(self, proposal: "Proposal") -> bool: """Checks if proposal's values are in accepted range e.g. @@ -168,7 +109,7 @@ def _validate_values(self, proposal: 'Proposal') -> bool: proposal.x: 11 """ - def _middle_values(self, our: 'Proposal', their: 'Proposal') -> Optional['Proposal']: + def _middle_values(self, our: "Proposal", their: "Proposal") -> Optional["Proposal"]: """Create new proposal with new values in accepted range based on given proposals. If middle values are outside of accepted range return None @@ -187,7 +128,7 @@ def _middle_values(self, our: 'Proposal', their: 'Proposal') -> Optional['Propos new: (9+13)//2 -> 11 -> None """ - def _negotiate_for_accepted_range(self, our: 'Proposal'): + def _negotiate_for_accepted_range(self, our: "Proposal"): their = our.respond() while True: if self._validate_values(their): @@ -198,8 +139,7 @@ def _negotiate_for_accepted_range(self, our: 'Proposal'): their = their.respond_with(our) - -def blacklist_offers(blacklist: 'List'): +def blacklist_offers(blacklist: "List"): def _blacklist_offers(func): def wrapper(*args, **kwargs): while True: @@ -212,17 +152,18 @@ def wrapper(*args, **kwargs): return _blacklist_offers + class LifoOfferManager: - _offers: List['Offer'] + _offers: List["Offer"] def __init__(self, event_bus) -> None: self._event_bus = event_bus self._event_bus.resource_listen(self.on_new_offer, ProposalConfirmed) - def on_new_offer(self, offer: 'Offer') -> None: + def on_new_offer(self, offer: "Offer") -> None: self._offers.append(offer) - def get_offer(self) -> 'Offer': + def get_offer(self) -> "Offer": while True: try: return self._offers.pop() @@ -231,11 +172,12 @@ def get_offer(self) -> 'Offer': # await sleep pass + class FifoAgreementManager: - def __init__(self, get_offer: 'Callable'): + def __init__(self, get_offer: "Callable"): self.get_offer = get_offer - def get_agreement(self) -> 'Agreement': + def get_agreement(self) -> "Agreement": while True: offer = self.get_offer() @@ -247,169 +189,34 @@ def get_agreement(self) -> 'Agreement': # TODO: Close agreement -class SingleUseActivityManager: - def __init__(self, get_agreement: 'Callable', on_activity_begin: Optional[Work] = None, on_activity_end: Optional[Work] = None): - self._get_agreement = get_agreement - self._on_activity_begin = on_activity_begin - self._on_activity_end = on_activity_end - - async def get_activity(self) -> 'Activity': - while True: - # We need to release agreement if is not used - agreement = await self._get_agreement() - try: - return await agreement.create_activity() - except Exception: - pass - - async def do_work(self, work) -> WorkResult: - activity = await self.get_activity() - - if self._on_activity_begin: - await activity.do(self._on_activity_begin) - - try: - result = await activity.do(work) - except Exception as e: - result = WorkResult(exception=e) - - if self._on_activity_end: - await activity.do(self._on_activity_end) - - return result - - -class SequentialWorkManager: - def __init__(self, do_work: DoWorkCallable): - self._do_work = do_work - - def apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: - if not hasattr(work, '_work_decorators'): - return do_work - - result = do_work - for dec in work._work_decorators: - result = partial(dec, result) - - return result - - async def do_work(self, work: Work) -> WorkResult: - decorated_do_work = self.apply_work_decorators(self._do_work, work) - - return await decorated_do_work(work) - - - async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: - results = [] - - for work in work_list: - results.append( - await self.do_work(work) - ) - - return results - - -def retry(tries: int = 3): - def _retry(do_work: DoWorkCallable) -> DoWorkCallable: - @wraps(do_work) - async def wrapper(work: Work) -> WorkResult: - count = 0 - errors = [] - - while count <= tries: - try: - return await do_work(work) - except Exception as err: - count += 1 - errors.append(err) - - raise errors # List[Exception] to Exception - - return wrapper - - return _retry - - -def redundancy_cancel_others_on_first_done(size: int = 3): - def _redundancy(do_work: DoWorkCallable): - @wraps(do_work) - async def wrapper(work: Work) -> WorkResult: - tasks = [do_work(work) for _ in range(size)] - - tasks_done, tasks_pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) - - for task in tasks_pending: - task.cancel() - - for task in tasks_pending: - try: - await task - except asyncio.CancelledError: - pass - - return tasks_done.pop().result() - - return wrapper - - return _redundancy - - -async def default_on_activity_begin(context: WorkContext): - batch = await context.create_batch() - batch.deploy() - batch.start() - await batch() - - # After this function call we should have check if activity is actually started - - -async def default_on_activity_end(context: WorkContext): - await context.terminate() - - # After this function call we should have check if activity is actually terminated - - -def work_decorator(decorator: WorkDecorator): - def _work_decorator(work: Work): - if not hasattr(work, '_work_decorators'): - work._work_decorators = [] - - work._work_decorators.append(decorator) - - return work - - return _work_decorator - - @work_decorator(redundancy_cancel_others_on_first_done(size=5)) @work_decorator(retry(tries=5)) async def work(context: WorkContext): - await context.run('echo hello world') + await context.run("echo hello world") async def work_service(context: WorkContext): - await context.run('app --daemon') + await context.run("app --daemon") async def work_service_fetch(context: WorkContext): - await context.run('app --daemon &') + await context.run("app --daemon &") - while await context.run('app check-if-running'): + while await context.run("app check-if-running"): await asyncio.sleep(1) async def work_batch(context: WorkContext): batch = await context.create_batch() - batch.run('echo 1') - batch.run('echo 2') - batch.run('echo 3') + batch.run("echo 1") + batch.run("echo 2") + batch.run("echo 3") await batch() -def main(): - payload = RepositoryVmPayload(image_hash='...') +async def main(): + payload = RepositoryVmPayload(image_hash="...") budget = 1.0 work_list = [ work, @@ -424,19 +231,21 @@ def main(): payment_manager.get_allocation, payload, event_bus ) negotiation_manager.add_filter(filter_blacklist, negotiation_manager.INITIAL) - negotiation_manager.negotiate() # run in async, this will generate ProposalConfirmed events + negotiation_manager.negotiate() # run in async, this will generate ProposalConfirmed events - offer_manager = LifoOfferManager(event_bus) # listen to ProposalConfirmed + offer_manager = LifoOfferManager(event_bus) # listen to ProposalConfirmed agreement_manager = FifoAgreementManager( - blacklist_offers(['banned_node_id'])(offer_manager.get_offer) + blacklist_offers(["banned_node_id"])(offer_manager.get_offer) ) activity_manager = SingleUseActivityManager( agreement_manager.get_agreement, - on_activity_begin=default_on_activity_begin, - on_activity_end=default_on_activity_end, ) work_manager = SequentialWorkManager(activity_manager.do_work) await work_manager.do_work_list(work_list) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/golem_core/managers/activity/__init__.py b/golem_core/managers/activity/__init__.py new file mode 100644 index 00000000..570f7117 --- /dev/null +++ b/golem_core/managers/activity/__init__.py @@ -0,0 +1,8 @@ +from golem_core.managers.activity.defaults import default_on_activity_begin, default_on_activity_end +from golem_core.managers.activity.single_use import SingleUseActivityManager + +__all__ = ( + "SingleUseActivityManager", + "default_on_activity_begin", + "default_on_activity_end", +) diff --git a/golem_core/managers/activity/defaults.py b/golem_core/managers/activity/defaults.py new file mode 100644 index 00000000..949543ad --- /dev/null +++ b/golem_core/managers/activity/defaults.py @@ -0,0 +1,16 @@ +from golem_core.managers.base import WorkContext + + +async def default_on_activity_start(context: WorkContext): + batch = await context.create_batch() + batch.deploy() + batch.start() + await batch() + + # TODO: After this function call we should have check if activity is actually started + + +async def default_on_activity_stop(context: WorkContext): + await context.terminate() + + # TODO: After this function call we should have check if activity is actually terminated diff --git a/golem_core/managers/activity/single_use.py b/golem_core/managers/activity/single_use.py new file mode 100644 index 00000000..cb92c83f --- /dev/null +++ b/golem_core/managers/activity/single_use.py @@ -0,0 +1,44 @@ +from typing import Awaitable, Callable, Optional + +from golem_core.managers.activity.defaults import ( + default_on_activity_start, + default_on_activity_stop, +) +from golem_core.managers.base import ActivityManager, Work, WorkResult + + +class SingleUseActivityManager(ActivityManager): + def __init__( + self, + get_agreement: Callable[[], Awaitable["Agreement"]], + on_activity_start: Optional[Work] = default_on_activity_start, + on_activity_stop: Optional[Work] = default_on_activity_stop, + ): + self._get_agreement = get_agreement + self._on_activity_start = on_activity_start + self._on_activity_stop = on_activity_stop + + async def get_activity(self) -> "Activity": + while True: + # We need to release agreement if is not used + agreement = await self._get_agreement() + try: + return await agreement.create_activity() + except Exception: + pass + + async def do_work(self, work) -> WorkResult: + activity = await self.get_activity() + + if self._on_activity_start: + await activity.do(self._on_activity_start) + + try: + result = await activity.do(work) + except Exception as e: + result = WorkResult(exception=e) + + if self._on_activity_stop: + await activity.do(self._on_activity_stop) + + return result diff --git a/golem_core/managers/base.py b/golem_core/managers/base.py new file mode 100644 index 00000000..fb9b70cb --- /dev/null +++ b/golem_core/managers/base.py @@ -0,0 +1,95 @@ +from abc import ABC, abstractmethod +from typing import Awaitable, Callable, List, Optional + + +class Batch: + def deploy(self): + ... + + def start(self): + ... + + def terminate(self): + ... + + def run(self, command: str): + ... + + async def __call__(self): + ... + + +class WorkContext: + async def deploy(self): + ... + + async def start(self): + ... + + async def terminate(self): + ... + + async def run(self, command: str): + ... + + async def create_batch(self) -> Batch: + ... + + +class WorkResult: + ... + + +WorkDecorator = Callable[["DoWorkCallable"], "DoWorkCallable"] + + +class Work(ABC): + _work_decorators: Optional[List[WorkDecorator]] + + def __call__(self, context: WorkContext) -> Optional[WorkResult]: + ... + + +DoWorkCallable = Callable[[Work], Awaitable[WorkResult]] + + +class Manager(ABC): + ... + + +class PaymentManager(Manager, ABC): + @abstractmethod + async def get_allocation(self) -> "Allocation": + ... + + +class NegotiationManager(Manager, ABC): + @abstractmethod + async def get_offer(self) -> "Offer": + ... + + +class OfferManager(Manager, ABC): + @abstractmethod + async def get_offer(self) -> "Offer": + ... + + +class AgreementManager(Manager, ABC): + @abstractmethod + async def get_agreement(self) -> "Agreement": + ... + + +class ActivityManager(Manager, ABC): + @abstractmethod + async def get_activity(self) -> "Activity": + ... + + @abstractmethod + async def do_work(self, work: Work) -> WorkResult: + ... + + +class WorkManager(Manager, ABC): + ... diff --git a/golem_core/managers/offer.py b/golem_core/managers/offer.py index 66b60180..bfd9d9c8 100644 --- a/golem_core/managers/offer.py +++ b/golem_core/managers/offer.py @@ -25,4 +25,3 @@ async def get_offer(self) -> Offer: except IndexError: # wait for offers await asyncio.sleep(1) - pass diff --git a/golem_core/managers/payment/pay_all.py b/golem_core/managers/payment/pay_all.py new file mode 100644 index 00000000..bc91a10b --- /dev/null +++ b/golem_core/managers/payment/pay_all.py @@ -0,0 +1,23 @@ +from golem_core.managers.base import PaymentManager + + +class PayAllPaymentManager(PaymentManager): + def __init__(self, budget, event_bus): + self._budget = budget + self._event_bus = event_bus + + self._allocation = Allocation.create(budget=self._budget) + + event_bus.register(InvoiceReceived(allocation=self._allocation), self.on_invoice_received) + event_bus.register( + DebitNoteReceived(allocation=self._allocation), self.on_debit_note_received + ) + + def get_allocation(self) -> "Allocation": + return self._allocation + + def on_invoice_received(self, invoice: "Invoice") -> None: + invoice.pay() + + def on_debit_note_received(self, debit_note: "DebitNote") -> None: + debit_note.pay() diff --git a/golem_core/managers/work/__init__.py b/golem_core/managers/work/__init__.py new file mode 100644 index 00000000..8796bacb --- /dev/null +++ b/golem_core/managers/work/__init__.py @@ -0,0 +1,13 @@ +from golem_core.managers.work.decorators import ( + redundancy_cancel_others_on_first_done, + retry, + work_decorator, +) +from golem_core.managers.work.sequential import SequentialWorkManager + +__all__ = ( + "SequentialWorkManager", + "work_decorator", + "redundancy_cancel_others_on_first_done", + "retry", +) diff --git a/golem_core/managers/work/decorators.py b/golem_core/managers/work/decorators.py new file mode 100644 index 00000000..c945c8d2 --- /dev/null +++ b/golem_core/managers/work/decorators.py @@ -0,0 +1,63 @@ +import asyncio +from functools import wraps + +from golem_core.managers.base import DoWorkCallable, Work, WorkDecorator, WorkResult + + +def work_decorator(decorator: WorkDecorator): + def _work_decorator(work: Work): + if not hasattr(work, "_work_decorators"): + work._work_decorators = [] + + work._work_decorators.append(decorator) + + return work + + return _work_decorator + + +def retry(tries: int = 3): + def _retry(do_work: DoWorkCallable) -> DoWorkCallable: + @wraps(do_work) + async def wrapper(work: Work) -> WorkResult: + count = 0 + errors = [] + + while count <= tries: + try: + return await do_work(work) + except Exception as err: + count += 1 + errors.append(err) + + raise errors # List[Exception] to Exception + + return wrapper + + return _retry + + +def redundancy_cancel_others_on_first_done(size: int = 3): + def _redundancy(do_work: DoWorkCallable): + @wraps(do_work) + async def wrapper(work: Work) -> WorkResult: + tasks = [do_work(work) for _ in range(size)] + + tasks_done, tasks_pending = await asyncio.wait( + tasks, return_when=asyncio.FIRST_COMPLETED + ) + + for task in tasks_pending: + task.cancel() + + for task in tasks_pending: + try: + await task + except asyncio.CancelledError: + pass + + return tasks_done.pop().result() + + return wrapper + + return _redundancy diff --git a/golem_core/managers/work/sequential.py b/golem_core/managers/work/sequential.py new file mode 100644 index 00000000..b465e176 --- /dev/null +++ b/golem_core/managers/work/sequential.py @@ -0,0 +1,32 @@ +from functools import partial +from typing import List + +from golem_core.managers.base import DoWorkCallable, Work, WorkResult + + +class SequentialWorkManager: + def __init__(self, do_work: DoWorkCallable): + self._do_work = do_work + + def apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: + if not hasattr(work, "_work_decorators"): + return do_work + + result = do_work + for dec in work._work_decorators: + result = partial(dec, result) + + return result + + async def do_work(self, work: Work) -> WorkResult: + decorated_do_work = self.apply_work_decorators(self._do_work, work) + + return await decorated_do_work(work) + + async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: + results = [] + + for work in work_list: + results.append(await self.do_work(work)) + + return results From e7acbc14750091bac3e56732e4f5dedb8c21919b Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 22 May 2023 10:24:58 +0200 Subject: [PATCH 011/123] Move managers to separated modules --- examples/managers/example01.py | 1 + golem_core/managers/negotiation/__init__.py | 3 +++ golem_core/managers/{ => negotiation}/negotiation.py | 0 golem_core/managers/offer/__init__.py | 3 +++ golem_core/managers/{ => offer}/offer.py | 0 5 files changed, 7 insertions(+) create mode 100644 golem_core/managers/negotiation/__init__.py rename golem_core/managers/{ => negotiation}/negotiation.py (100%) create mode 100644 golem_core/managers/offer/__init__.py rename golem_core/managers/{ => offer}/offer.py (100%) diff --git a/examples/managers/example01.py b/examples/managers/example01.py index c8e431c0..cad21bd0 100644 --- a/examples/managers/example01.py +++ b/examples/managers/example01.py @@ -24,6 +24,7 @@ async def main(): for _ in range(20): print(f"{datetime.utcnow()} sleeping...") await asyncio.sleep(1) + print((await offer_manager.get_offer()).id) print(f"{datetime.utcnow()} stopping negotiations...") await negotiation_manager.stop_negotiation() diff --git a/golem_core/managers/negotiation/__init__.py b/golem_core/managers/negotiation/__init__.py new file mode 100644 index 00000000..9136cc9f --- /dev/null +++ b/golem_core/managers/negotiation/__init__.py @@ -0,0 +1,3 @@ +from golem_core.managers.negotiation.negotiation import AlfaNegotiationManager + +__all__ = ("AlfaNegotiationManager",) diff --git a/golem_core/managers/negotiation.py b/golem_core/managers/negotiation/negotiation.py similarity index 100% rename from golem_core/managers/negotiation.py rename to golem_core/managers/negotiation/negotiation.py diff --git a/golem_core/managers/offer/__init__.py b/golem_core/managers/offer/__init__.py new file mode 100644 index 00000000..a29dabdc --- /dev/null +++ b/golem_core/managers/offer/__init__.py @@ -0,0 +1,3 @@ +from golem_core.managers.offer.offer import StackOfferManager + +__all__ = ("StackOfferManager",) diff --git a/golem_core/managers/offer.py b/golem_core/managers/offer/offer.py similarity index 100% rename from golem_core/managers/offer.py rename to golem_core/managers/offer/offer.py From bfff79198e55d49d0c6013f010c7826fc0d96fe8 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 22 May 2023 11:28:01 +0200 Subject: [PATCH 012/123] Hasable Payload fix --- examples/managers/example01.py | 4 +- .../demand/demand_offer_base/payload/base.py | 3 + .../demand/demand_offer_base/payload/vm.py | 8 +++ .../managers/negotiation/negotiation.py | 71 ++++++++++++------- 4 files changed, 58 insertions(+), 28 deletions(-) diff --git a/examples/managers/example01.py b/examples/managers/example01.py index cad21bd0..548522fa 100644 --- a/examples/managers/example01.py +++ b/examples/managers/example01.py @@ -21,10 +21,10 @@ async def main(): negotiation_manager = AlfaNegotiationManager(golem, get_allocation_factory(golem)) await negotiation_manager.start_negotiation(payload) - for _ in range(20): + for i in range(1, 16): + print(f"Got offer {i}: {(await offer_manager.get_offer()).id}...") print(f"{datetime.utcnow()} sleeping...") await asyncio.sleep(1) - print((await offer_manager.get_offer()).id) print(f"{datetime.utcnow()} stopping negotiations...") await negotiation_manager.stop_negotiation() diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/payload/base.py b/golem_core/core/market_api/resources/demand/demand_offer_base/payload/base.py index 8c3fcced..90e0523a 100644 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/payload/base.py +++ b/golem_core/core/market_api/resources/demand/demand_offer_base/payload/base.py @@ -37,3 +37,6 @@ async def main(): {'properties': {'golem.srv.app.myprop': 'othervalue'}, 'constraints': ['(&(golem.runtime.name=my-runtime)\n\t(golem.inf.mem.gib>=32)\n\t(golem.inf.storage.gib>=1024))']} """ # noqa: E501 + + def __hash__(self) -> int: + return hash((str(self._serialize_properties()), self._serialize_constraints())) diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/payload/vm.py b/golem_core/core/market_api/resources/demand/demand_offer_base/payload/vm.py index 26dae55d..0cdf5a68 100644 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/payload/vm.py +++ b/golem_core/core/market_api/resources/demand/demand_offer_base/payload/vm.py @@ -52,6 +52,10 @@ class BaseVmPayload(Payload, ABC): ) + def __hash__(self) -> int: + return super().__hash__() + + @dataclass class _VmPayload(Payload, ABC): package_url: str = prop("golem.srv.comp.task_package") @@ -109,6 +113,10 @@ async def serialize(self) -> Tuple[Dict[str, Any], str]: return await super(RepositoryVmPayload, self).serialize() + def __hash__(self) -> int: + return super().__hash__() + + async def resolve_repository_url( repo_srv: str = DEFAULT_REPO_URL_SRV, fallback_url: str = DEFAULT_REPO_URL_FALLBACK, diff --git a/golem_core/managers/negotiation/negotiation.py b/golem_core/managers/negotiation/negotiation.py index b20aa603..5b5cf6f2 100644 --- a/golem_core/managers/negotiation/negotiation.py +++ b/golem_core/managers/negotiation/negotiation.py @@ -1,25 +1,57 @@ import asyncio from datetime import datetime, timezone -from typing import Awaitable, List +from typing import Any, Awaitable, Dict, List from golem_core.core.events import EventBus from golem_core.core.golem_node.golem_node import DEFAULT_EXPIRATION_TIMEOUT, SUBNET, GolemNode -from golem_core.core.market_api import Demand, DemandBuilder, Payload +from golem_core.core.market_api import Demand, DemandBuilder, Payload, Proposal from golem_core.core.market_api.resources.demand.demand_offer_base import defaults as dobm_defaults from golem_core.core.payment_api import Allocation from golem_core.core.resources import ResourceEvent +from golem_core.managers.base import NegotiationManager -class AlfaNegotiationManager: +class AlfaNegotiationManager(NegotiationManager): def __init__(self, golem: GolemNode, get_allocation: Awaitable) -> None: self._golem = golem self._event_bus: EventBus = self._golem.event_bus self._get_allocation = get_allocation self._negotiations: List[asyncio.Task] = [] + self._offer_generators: Dict[Payload, Any] = {} # TODO typing gneerator - async def start_negotiation(self, payload: Payload) -> None: - allocation: Allocation = await self._get_allocation() + async def get_offer(self, payload) -> Proposal: + offer_generator = await self._get_offer_generator(payload) + return await anext(offer_generator) + + async def _get_offer_generator(self, payload) -> Any: + if self._offer_generators.get(payload) is None: + allocation = await self._get_allocation() + demand = await self._build_demand(allocation, payload) + self._offer_generators[payload] = self._negotiate(demand) + return self._offer_generators[payload] + + async def _negotiate(self, demand: Demand) -> None: + try: + async for initial in demand.initial_proposals(): + try: + pending = await initial.respond() + except Exception as err: + print( + f"Unable to respond to initial proposal {initial.id}. Got {type(err)}\n{err}" + ) + continue + + try: + # TODO IDK how to call `confirm` on a proposal in golem-core + confirmed = await pending.responses().__anext__() + except StopAsyncIteration: + continue + + yield confirmed + finally: + self._golem.add_autoclose_resource(demand) + async def _build_demand(self, allocation: Allocation, payload: Payload) -> Demand: demand_builder = DemandBuilder() await demand_builder.add( @@ -41,29 +73,16 @@ async def start_negotiation(self, payload: Payload) -> None: demand = await demand_builder.create_demand(self._golem) demand.start_collecting_events() - self._negotiations.append(asyncio.create_task(self._negotiate(demand))) + return demand + + async def start_negotiation(self, payload: Payload) -> None: + self._negotiations.append(asyncio.create_task(self._negotiate_task(payload))) async def stop_negotiation(self) -> None: for task in self._negotiations: task.cancel() - async def _negotiate(self, demand: Demand) -> None: - try: - async for initial in demand.initial_proposals(): - try: - pending = await initial.respond() - except Exception as err: - print( - f"Unable to respond to initial proposal {initial.id}. Got {type(err)}\n{err}" - ) - continue - - try: - # TODO IDK how to call `confirm` on a proposal in golem-core - confirmed = await pending.responses().__anext__() - except StopAsyncIteration: - continue - - self._event_bus.emit(ResourceEvent(confirmed)) - finally: - self._golem.add_autoclose_resource(demand) + async def _negotiate_task(self, payload: Payload) -> None: + offer_generator = await self._get_offer_generator(payload) + async for offer in offer_generator: + self._event_bus.emit(ResourceEvent(offer)) From 7c455b630da769497b5ea300e67be767c98f79c1 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 23 May 2023 11:20:09 +0200 Subject: [PATCH 013/123] Offer and Negotiation maangers on callbacks --- examples/managers/example01.py | 7 +- .../demand/demand_offer_base/payload/vm.py | 2 - .../managers/negotiation/negotiation.py | 83 +++++++++---------- golem_core/managers/offer/offer.py | 21 +++-- 4 files changed, 59 insertions(+), 54 deletions(-) diff --git a/examples/managers/example01.py b/examples/managers/example01.py index 548522fa..bd2399f3 100644 --- a/examples/managers/example01.py +++ b/examples/managers/example01.py @@ -17,15 +17,18 @@ async def _get_allocation(): async def main(): payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") async with GolemNode() as golem: - offer_manager = StackOfferManager(golem) negotiation_manager = AlfaNegotiationManager(golem, get_allocation_factory(golem)) + offer_manager = StackOfferManager(negotiation_manager.get_offer) await negotiation_manager.start_negotiation(payload) + await offer_manager.start_consuming_offers() - for i in range(1, 16): + for i in range(10): print(f"Got offer {i}: {(await offer_manager.get_offer()).id}...") print(f"{datetime.utcnow()} sleeping...") await asyncio.sleep(1) + print(f"{datetime.utcnow()} stopping negotiations...") + await offer_manager.stop_consuming_offers() await negotiation_manager.stop_negotiation() diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/payload/vm.py b/golem_core/core/market_api/resources/demand/demand_offer_base/payload/vm.py index 0cdf5a68..fcee1481 100644 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/payload/vm.py +++ b/golem_core/core/market_api/resources/demand/demand_offer_base/payload/vm.py @@ -51,7 +51,6 @@ class BaseVmPayload(Payload, ABC): "golem.srv.comp.vm.package_format", default=VmPackageFormat.GVMKIT_SQUASH ) - def __hash__(self) -> int: return super().__hash__() @@ -112,7 +111,6 @@ async def serialize(self) -> Tuple[Dict[str, Any], str]: return await super(RepositoryVmPayload, self).serialize() - def __hash__(self) -> int: return super().__hash__() diff --git a/golem_core/managers/negotiation/negotiation.py b/golem_core/managers/negotiation/negotiation.py index 5b5cf6f2..f49edc79 100644 --- a/golem_core/managers/negotiation/negotiation.py +++ b/golem_core/managers/negotiation/negotiation.py @@ -1,55 +1,42 @@ import asyncio from datetime import datetime, timezone -from typing import Any, Awaitable, Dict, List +from typing import AsyncIterator, Awaitable, Callable, List -from golem_core.core.events import EventBus from golem_core.core.golem_node.golem_node import DEFAULT_EXPIRATION_TIMEOUT, SUBNET, GolemNode from golem_core.core.market_api import Demand, DemandBuilder, Payload, Proposal from golem_core.core.market_api.resources.demand.demand_offer_base import defaults as dobm_defaults from golem_core.core.payment_api import Allocation -from golem_core.core.resources import ResourceEvent from golem_core.managers.base import NegotiationManager class AlfaNegotiationManager(NegotiationManager): - def __init__(self, golem: GolemNode, get_allocation: Awaitable) -> None: + def __init__( + self, golem: GolemNode, get_allocation: Callable[[], Awaitable[Allocation]] + ) -> None: self._golem = golem - self._event_bus: EventBus = self._golem.event_bus self._get_allocation = get_allocation self._negotiations: List[asyncio.Task] = [] - self._offer_generators: Dict[Payload, Any] = {} # TODO typing gneerator + self._ready_offers: List[Proposal] = [] - async def get_offer(self, payload) -> Proposal: - offer_generator = await self._get_offer_generator(payload) - return await anext(offer_generator) + async def get_offer(self) -> Proposal: + while True: + try: + return self._ready_offers.pop(0) + except IndexError: + await asyncio.sleep(1) - async def _get_offer_generator(self, payload) -> Any: - if self._offer_generators.get(payload) is None: - allocation = await self._get_allocation() - demand = await self._build_demand(allocation, payload) - self._offer_generators[payload] = self._negotiate(demand) - return self._offer_generators[payload] - - async def _negotiate(self, demand: Demand) -> None: - try: - async for initial in demand.initial_proposals(): - try: - pending = await initial.respond() - except Exception as err: - print( - f"Unable to respond to initial proposal {initial.id}. Got {type(err)}\n{err}" - ) - continue + async def start_negotiation(self, payload: Payload) -> None: + self._negotiations.append(asyncio.create_task(self._negotiate_task(payload))) - try: - # TODO IDK how to call `confirm` on a proposal in golem-core - confirmed = await pending.responses().__anext__() - except StopAsyncIteration: - continue + async def stop_negotiation(self) -> None: + for task in self._negotiations: + task.cancel() - yield confirmed - finally: - self._golem.add_autoclose_resource(demand) + async def _negotiate_task(self, payload: Payload) -> None: + allocation = await self._get_allocation() + demand = await self._build_demand(allocation, payload) + async for offer in self._negotiate(demand): + self._ready_offers.append(offer) async def _build_demand(self, allocation: Allocation, payload: Payload) -> Demand: demand_builder = DemandBuilder() @@ -75,14 +62,24 @@ async def _build_demand(self, allocation: Allocation, payload: Payload) -> Deman demand.start_collecting_events() return demand - async def start_negotiation(self, payload: Payload) -> None: - self._negotiations.append(asyncio.create_task(self._negotiate_task(payload))) + async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: + try: + async for initial in demand.initial_proposals(): + print("Got initial...") + try: + pending = await initial.respond() + except Exception as err: + print( + f"Unable to respond to initialproposal {initial.id}. Got {type(err)}\n{err}" + ) + continue - async def stop_negotiation(self) -> None: - for task in self._negotiations: - task.cancel() + try: + # TODO IDK how to call `confirm` on a proposal in golem-core + confirmed = await pending.responses().__anext__() + except StopAsyncIteration: + continue - async def _negotiate_task(self, payload: Payload) -> None: - offer_generator = await self._get_offer_generator(payload) - async for offer in offer_generator: - self._event_bus.emit(ResourceEvent(offer)) + yield confirmed + finally: + self._golem.add_autoclose_resource(demand) diff --git a/golem_core/managers/offer/offer.py b/golem_core/managers/offer/offer.py index bfd9d9c8..06f6c0b1 100644 --- a/golem_core/managers/offer/offer.py +++ b/golem_core/managers/offer/offer.py @@ -1,21 +1,28 @@ import asyncio from typing import List -from golem_core.core.golem_node import GolemNode from golem_core.core.market_api import Proposal -from golem_core.core.resources import ResourceEvent Offer = Proposal class StackOfferManager: - def __init__(self, golem: GolemNode) -> None: + def __init__(self, get_offer) -> None: + self._get_offer = get_offer self._offers: List[Offer] = [] - self._event_bus = golem.event_bus - self._event_bus.resource_listen(self._on_new_offer, (ResourceEvent,), (Offer,)) + self._tasks: List[asyncio.Task] = [] - async def _on_new_offer(self, offer_event: ResourceEvent) -> None: - self._offers.append(offer_event.resource) + async def start_consuming_offers(self) -> None: + self._tasks.append(asyncio.create_task(self._consume_offers())) + + async def stop_consuming_offers(self) -> None: + for task in self._tasks: + task.cancel() + + async def _consume_offers(self) -> None: + while True: + self._offers.append(await self._get_offer()) + await asyncio.sleep(1) async def get_offer(self) -> Offer: # TODO add some timeout From 677455aafad1d4a3199377e2e25813e2348fe1ed Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 24 May 2023 13:30:46 +0200 Subject: [PATCH 014/123] Barely working agreement/work managers --- big_5_draft.py | 35 +++++--- examples/managers/example02.py | 94 ++++++++++++++++++++++ golem_core/managers/activity/__init__.py | 6 +- golem_core/managers/activity/defaults.py | 2 + golem_core/managers/activity/single_use.py | 22 ++--- golem_core/managers/agreement/__init__.py | 0 golem_core/managers/agreement/queue.py | 22 +++++ golem_core/managers/base.py | 60 +++++++++----- golem_core/managers/work/decorators.py | 20 +++-- 9 files changed, 213 insertions(+), 48 deletions(-) create mode 100644 examples/managers/example02.py create mode 100644 golem_core/managers/agreement/__init__.py create mode 100644 golem_core/managers/agreement/queue.py diff --git a/big_5_draft.py b/big_5_draft.py index fdc8f35c..5b36ca54 100644 --- a/big_5_draft.py +++ b/big_5_draft.py @@ -3,6 +3,7 @@ from golem_core.core.market_api import RepositoryVmPayload from golem_core.managers.activity.single_use import SingleUseActivityManager +from golem_core.managers.agreement.queue import QueueAgreementManager from golem_core.managers.base import WorkContext from golem_core.managers.payment.pay_all import PayAllPaymentManager from golem_core.managers.work.decorators import ( @@ -13,6 +14,20 @@ from golem_core.managers.work.sequential import SequentialWorkManager + + + + + + + + + + + + + + class ConfirmAllNegotiationManager: def __init__(self, get_allocation: "Callable", payload, event_bus): self._event_bus = event_bus @@ -173,20 +188,16 @@ def get_offer(self) -> "Offer": pass -class FifoAgreementManager: - def __init__(self, get_offer: "Callable"): - self.get_offer = get_offer - def get_agreement(self) -> "Agreement": - while True: - offer = self.get_offer() - try: - return offer.create_agrement() - except Exception: - pass - # TODO: Close agreement + + + + + + + @work_decorator(redundancy_cancel_others_on_first_done(size=5)) @@ -235,7 +246,7 @@ async def main(): offer_manager = LifoOfferManager(event_bus) # listen to ProposalConfirmed - agreement_manager = FifoAgreementManager( + agreement_manager = QueueAgreementManager( blacklist_offers(["banned_node_id"])(offer_manager.get_offer) ) diff --git a/examples/managers/example02.py b/examples/managers/example02.py new file mode 100644 index 00000000..18bb59bb --- /dev/null +++ b/examples/managers/example02.py @@ -0,0 +1,94 @@ +import asyncio +from contextlib import asynccontextmanager + +from golem_core.core.golem_node import GolemNode +from golem_core.core.market_api import RepositoryVmPayload +from golem_core.managers.activity import SingleUseActivityManager +from golem_core.managers.base import WorkContext +from golem_core.managers.work import SequentialWorkManager + + + +@asynccontextmanager +async def create_agreement(): + payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") + + print('Entering golem context...') + async with GolemNode() as golem: + print('Creating allocation...') + allocation = await golem.create_allocation(1) + print('Creating demand...') + demand = await golem.create_demand(payload, allocations=[allocation]) + + print('Gathering initial proposals...') + async for proposal in demand.initial_proposals(): + print('Responding to initial proposal...') + try: + our_response = await proposal.respond() + except Exception as e: + print(str(e)) + continue + + print('Waiting for initial proposal...') + try: + their_response = await our_response.responses().__anext__() + except StopAsyncIteration: + continue + + print('Creating agreement...') + agreement = await their_response.create_agreement() + await agreement.confirm() + await agreement.wait_for_approval() + + print('Yielding agreement...') + yield agreement + return + + + +async def work1(context: WorkContext): + r = await context.run('echo 1') + await r.wait() + for event in r.events: + print(event.stdout) + +async def work2(context: WorkContext): + r = await context.run('echo 2') + await r.wait() + for event in r.events: + print(event.stdout) + +async def work3(context: WorkContext): + r = await context.run('echo 3') + await r.wait() + for event in r.events: + print(event.stdout) + + +async def main(): + async with create_agreement() as agreement: + async def get_agreement(): + return agreement + + work_list = [ + work1, + work2, + work3, + ] + + activity_manager = SingleUseActivityManager( + get_agreement, + ) + + work_manager = SequentialWorkManager(activity_manager.do_work) + + print('starting to work...') + results = await work_manager.do_work_list(work_list) + print('work done') + print(results) + + await agreement.terminate() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/golem_core/managers/activity/__init__.py b/golem_core/managers/activity/__init__.py index 570f7117..98cdbe18 100644 --- a/golem_core/managers/activity/__init__.py +++ b/golem_core/managers/activity/__init__.py @@ -1,8 +1,8 @@ -from golem_core.managers.activity.defaults import default_on_activity_begin, default_on_activity_end +from golem_core.managers.activity.defaults import default_on_activity_start, default_on_activity_stop from golem_core.managers.activity.single_use import SingleUseActivityManager __all__ = ( "SingleUseActivityManager", - "default_on_activity_begin", - "default_on_activity_end", + "default_on_activity_start", + "default_on_activity_stop", ) diff --git a/golem_core/managers/activity/defaults.py b/golem_core/managers/activity/defaults.py index 949543ad..6cfc33a9 100644 --- a/golem_core/managers/activity/defaults.py +++ b/golem_core/managers/activity/defaults.py @@ -6,11 +6,13 @@ async def default_on_activity_start(context: WorkContext): batch.deploy() batch.start() await batch() + print('activity start') # TODO: After this function call we should have check if activity is actually started async def default_on_activity_stop(context: WorkContext): await context.terminate() + print('activity stop') # TODO: After this function call we should have check if activity is actually terminated diff --git a/golem_core/managers/activity/single_use.py b/golem_core/managers/activity/single_use.py index cb92c83f..e6b95055 100644 --- a/golem_core/managers/activity/single_use.py +++ b/golem_core/managers/activity/single_use.py @@ -4,15 +4,15 @@ default_on_activity_start, default_on_activity_stop, ) -from golem_core.managers.base import ActivityManager, Work, WorkResult +from golem_core.managers.base import ActivityManager, Work, WorkResult, WorkContext class SingleUseActivityManager(ActivityManager): def __init__( self, get_agreement: Callable[[], Awaitable["Agreement"]], - on_activity_start: Optional[Work] = default_on_activity_start, - on_activity_stop: Optional[Work] = default_on_activity_stop, + on_activity_start: Optional[Callable[[WorkContext], Awaitable[None]]] = default_on_activity_start, + on_activity_stop: Optional[Callable[[WorkContext], Awaitable[None]]] = default_on_activity_stop, ): self._get_agreement = get_agreement self._on_activity_start = on_activity_start @@ -27,18 +27,22 @@ async def get_activity(self) -> "Activity": except Exception: pass - async def do_work(self, work) -> WorkResult: + async def do_work(self, work: Work) -> WorkResult: activity = await self.get_activity() + work_context = WorkContext(activity) if self._on_activity_start: - await activity.do(self._on_activity_start) + await self._on_activity_start(work_context) try: - result = await activity.do(work) + work_result = await work(work_context) except Exception as e: - result = WorkResult(exception=e) + work_result = WorkResult(exception=e) + else: + if not isinstance(work_result, WorkResult): + work_result = WorkResult(result=work_result) if self._on_activity_stop: - await activity.do(self._on_activity_stop) + await self._on_activity_stop(work_context) - return result + return work_result \ No newline at end of file diff --git a/golem_core/managers/agreement/__init__.py b/golem_core/managers/agreement/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/golem_core/managers/agreement/queue.py b/golem_core/managers/agreement/queue.py new file mode 100644 index 00000000..8138ed2e --- /dev/null +++ b/golem_core/managers/agreement/queue.py @@ -0,0 +1,22 @@ +from typing import Callable, Awaitable + +from golem_core.core.market_api import Agreement +from golem_core.managers.base import AgreementManager + + +class QueueAgreementManager(AgreementManager): + def __init__(self, get_offer: Callable[[], Awaitable['Offer']]): + self._get_offer = get_offer + + async def get_agreement(self) -> Agreement: + while True: + offer = await self._get_offer() + + try: + agreement = await offer.create_agreement() + await agreement.confirm() + await agreement.wait_for_approval() + except Exception: + pass + else: + return agreement diff --git a/golem_core/managers/base.py b/golem_core/managers/base.py index fb9b70cb..7727ff6e 100644 --- a/golem_core/managers/base.py +++ b/golem_core/managers/base.py @@ -1,43 +1,67 @@ from abc import ABC, abstractmethod -from typing import Awaitable, Callable, List, Optional +from dataclasses import dataclass +from typing import Awaitable, Callable, List, Optional, Any, Dict, Union + +from golem_core.core.activity_api import commands, Activity, Script class Batch: + def __init__(self, activity) -> None: + self._script = Script() + self._activity = activity + def deploy(self): - ... + self._script.add_command(commands.Deploy()) def start(self): - ... + self._script.add_command(commands.Start()) - def terminate(self): - ... - - def run(self, command: str): - ... + def run( + self, + command: Union[str, List[str]], + *, + shell: Optional[bool] = None, + shell_cmd: str = "/bin/sh" + ): + self._script.add_command(commands.Run(command, shell=shell, shell_cmd=shell_cmd)) async def __call__(self): - ... - + pooling_batch = await self._activity.execute_script(self._script) + return await pooling_batch.wait() class WorkContext: + def __init__(self, activity: Activity): + self._activity = activity + async def deploy(self): - ... + pooling_batch = await self._activity.execute_commands(commands.Deploy()) + await pooling_batch.wait() async def start(self): - ... + pooling_batch = await self._activity.execute_commands(commands.Start()) + await pooling_batch.wait() async def terminate(self): - ... + await self._activity.destroy() - async def run(self, command: str): - ... + async def run( + self, + command: Union[str, List[str]], + *, + shell: Optional[bool] = None, + shell_cmd: str = "/bin/sh" + ): + return await self._activity.execute_commands(commands.Run(command, shell=shell, shell_cmd=shell_cmd)) async def create_batch(self) -> Batch: - ... + return Batch(self._activity) +@dataclass class WorkResult: - ... + result: Optional[Any] = None + exception: Optional[Exception] = None + extras: Optional[Dict] = None WorkDecorator = Callable[["DoWorkCallable"], "DoWorkCallable"] @@ -46,7 +70,7 @@ class WorkResult: class Work(ABC): _work_decorators: Optional[List[WorkDecorator]] - def __call__(self, context: WorkContext) -> Optional[WorkResult]: + def __call__(self, context: WorkContext) -> Awaitable[Optional[WorkResult]]: ... diff --git a/golem_core/managers/work/decorators.py b/golem_core/managers/work/decorators.py index c945c8d2..4d60f2f2 100644 --- a/golem_core/managers/work/decorators.py +++ b/golem_core/managers/work/decorators.py @@ -22,15 +22,23 @@ def _retry(do_work: DoWorkCallable) -> DoWorkCallable: async def wrapper(work: Work) -> WorkResult: count = 0 errors = [] + work_result = WorkResult() while count <= tries: - try: - return await do_work(work) - except Exception as err: - count += 1 - errors.append(err) + work_result = await do_work(work) + + if work_result.exception is None: + break + + count += 1 + errors.append(work_result.exception) + + work_result.extras['retry'] = { + 'tries': count, + 'errors': errors, + } - raise errors # List[Exception] to Exception + return work_result return wrapper From 0685962b9f191e68a26e50a255d30b350ee46b96 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 24 May 2023 15:30:21 +0200 Subject: [PATCH 015/123] Replace list with asyncio.Queue --- golem_core/managers/negotiation/negotiation.py | 10 +++------- golem_core/managers/offer/offer.py | 13 +++---------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/golem_core/managers/negotiation/negotiation.py b/golem_core/managers/negotiation/negotiation.py index f49edc79..7f2fd915 100644 --- a/golem_core/managers/negotiation/negotiation.py +++ b/golem_core/managers/negotiation/negotiation.py @@ -16,14 +16,10 @@ def __init__( self._golem = golem self._get_allocation = get_allocation self._negotiations: List[asyncio.Task] = [] - self._ready_offers: List[Proposal] = [] + self._ready_offers: asyncio.Queue[Proposal] = asyncio.Queue() async def get_offer(self) -> Proposal: - while True: - try: - return self._ready_offers.pop(0) - except IndexError: - await asyncio.sleep(1) + return await self._ready_offers.get() async def start_negotiation(self, payload: Payload) -> None: self._negotiations.append(asyncio.create_task(self._negotiate_task(payload))) @@ -36,7 +32,7 @@ async def _negotiate_task(self, payload: Payload) -> None: allocation = await self._get_allocation() demand = await self._build_demand(allocation, payload) async for offer in self._negotiate(demand): - self._ready_offers.append(offer) + await self._ready_offers.put(offer) async def _build_demand(self, allocation: Allocation, payload: Payload) -> Demand: demand_builder = DemandBuilder() diff --git a/golem_core/managers/offer/offer.py b/golem_core/managers/offer/offer.py index 06f6c0b1..2283cce9 100644 --- a/golem_core/managers/offer/offer.py +++ b/golem_core/managers/offer/offer.py @@ -9,7 +9,7 @@ class StackOfferManager: def __init__(self, get_offer) -> None: self._get_offer = get_offer - self._offers: List[Offer] = [] + self._offers: asyncio.Queue[Proposal] = asyncio.Queue() self._tasks: List[asyncio.Task] = [] async def start_consuming_offers(self) -> None: @@ -21,14 +21,7 @@ async def stop_consuming_offers(self) -> None: async def _consume_offers(self) -> None: while True: - self._offers.append(await self._get_offer()) - await asyncio.sleep(1) + await self._offers.put(await self._get_offer()) async def get_offer(self) -> Offer: - # TODO add some timeout - while True: - try: - return self._offers.pop() - except IndexError: - # wait for offers - await asyncio.sleep(1) + return await self._offers.get() From ef0e32271ca0a968007e10388ff24c33646477ab Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 24 May 2023 16:16:26 +0200 Subject: [PATCH 016/123] formatting --- big_5_draft.py | 30 +-------- examples/managers/example02.py | 56 +++++++++-------- golem_core/managers/activity/__init__.py | 5 +- golem_core/managers/activity/defaults.py | 4 +- golem_core/managers/activity/single_use.py | 61 +++++++++++++------ .../agreement/{queue.py => single_use.py} | 14 ++++- golem_core/managers/base.py | 13 ++-- golem_core/managers/work/decorators.py | 6 +- 8 files changed, 102 insertions(+), 87 deletions(-) rename golem_core/managers/agreement/{queue.py => single_use.py} (52%) diff --git a/big_5_draft.py b/big_5_draft.py index 5b36ca54..8c6eeff0 100644 --- a/big_5_draft.py +++ b/big_5_draft.py @@ -3,7 +3,7 @@ from golem_core.core.market_api import RepositoryVmPayload from golem_core.managers.activity.single_use import SingleUseActivityManager -from golem_core.managers.agreement.queue import QueueAgreementManager +from golem_core.managers.agreement.single_use import SingleUseAgreementManager from golem_core.managers.base import WorkContext from golem_core.managers.payment.pay_all import PayAllPaymentManager from golem_core.managers.work.decorators import ( @@ -14,20 +14,6 @@ from golem_core.managers.work.sequential import SequentialWorkManager - - - - - - - - - - - - - - class ConfirmAllNegotiationManager: def __init__(self, get_allocation: "Callable", payload, event_bus): self._event_bus = event_bus @@ -188,18 +174,6 @@ def get_offer(self) -> "Offer": pass - - - - - - - - - - - - @work_decorator(redundancy_cancel_others_on_first_done(size=5)) @work_decorator(retry(tries=5)) async def work(context: WorkContext): @@ -246,7 +220,7 @@ async def main(): offer_manager = LifoOfferManager(event_bus) # listen to ProposalConfirmed - agreement_manager = QueueAgreementManager( + agreement_manager = SingleUseAgreementManager( blacklist_offers(["banned_node_id"])(offer_manager.get_offer) ) diff --git a/examples/managers/example02.py b/examples/managers/example02.py index 18bb59bb..55a3e08e 100644 --- a/examples/managers/example02.py +++ b/examples/managers/example02.py @@ -4,71 +4,75 @@ from golem_core.core.golem_node import GolemNode from golem_core.core.market_api import RepositoryVmPayload from golem_core.managers.activity import SingleUseActivityManager +from golem_core.managers.agreement.single_use import SingleUseAgreementManager from golem_core.managers.base import WorkContext from golem_core.managers.work import SequentialWorkManager - @asynccontextmanager async def create_agreement(): payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") - print('Entering golem context...') + print("Entering golem context...") async with GolemNode() as golem: - print('Creating allocation...') + print("Creating allocation...") allocation = await golem.create_allocation(1) - print('Creating demand...') + print("Creating demand...") demand = await golem.create_demand(payload, allocations=[allocation]) - print('Gathering initial proposals...') + print("Gathering initial proposals...") async for proposal in demand.initial_proposals(): - print('Responding to initial proposal...') + print("Responding to initial proposal...") try: our_response = await proposal.respond() except Exception as e: print(str(e)) continue - print('Waiting for initial proposal...') + print("Waiting for initial proposal...") try: their_response = await our_response.responses().__anext__() except StopAsyncIteration: continue - print('Creating agreement...') - agreement = await their_response.create_agreement() - await agreement.confirm() - await agreement.wait_for_approval() + yield their_response - print('Yielding agreement...') - yield agreement - return + # print('Creating agreement...') + # agreement = await their_response.create_agreement() + # await agreement.confirm() + # await agreement.wait_for_approval() + # print('Yielding agreement...') + # yield agreement + # return async def work1(context: WorkContext): - r = await context.run('echo 1') + r = await context.run("echo 1") await r.wait() for event in r.events: print(event.stdout) + async def work2(context: WorkContext): - r = await context.run('echo 2') + r = await context.run("echo 2") await r.wait() for event in r.events: print(event.stdout) + async def work3(context: WorkContext): - r = await context.run('echo 3') + r = await context.run("echo 3") await r.wait() for event in r.events: print(event.stdout) async def main(): - async with create_agreement() as agreement: - async def get_agreement(): - return agreement + async with create_offer() as offer: + + async def get_offer(): + return offer work_list = [ work1, @@ -76,19 +80,21 @@ async def get_agreement(): work3, ] + agreement_manager = SingleUseAgreementManager(get_offer) + activity_manager = SingleUseActivityManager( - get_agreement, + agreement_manager.get_agreement, ) work_manager = SequentialWorkManager(activity_manager.do_work) - print('starting to work...') + print("starting to work...") results = await work_manager.do_work_list(work_list) - print('work done') + print("work done") print(results) - await agreement.terminate() + await agreement_manager.close() -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) diff --git a/golem_core/managers/activity/__init__.py b/golem_core/managers/activity/__init__.py index 98cdbe18..0cfc216a 100644 --- a/golem_core/managers/activity/__init__.py +++ b/golem_core/managers/activity/__init__.py @@ -1,4 +1,7 @@ -from golem_core.managers.activity.defaults import default_on_activity_start, default_on_activity_stop +from golem_core.managers.activity.defaults import ( + default_on_activity_start, + default_on_activity_stop, +) from golem_core.managers.activity.single_use import SingleUseActivityManager __all__ = ( diff --git a/golem_core/managers/activity/defaults.py b/golem_core/managers/activity/defaults.py index 6cfc33a9..228b6687 100644 --- a/golem_core/managers/activity/defaults.py +++ b/golem_core/managers/activity/defaults.py @@ -6,13 +6,13 @@ async def default_on_activity_start(context: WorkContext): batch.deploy() batch.start() await batch() - print('activity start') + print("activity start") # TODO: After this function call we should have check if activity is actually started async def default_on_activity_stop(context: WorkContext): await context.terminate() - print('activity stop') + print("activity stop") # TODO: After this function call we should have check if activity is actually terminated diff --git a/golem_core/managers/activity/single_use.py b/golem_core/managers/activity/single_use.py index e6b95055..8fb0d86c 100644 --- a/golem_core/managers/activity/single_use.py +++ b/golem_core/managers/activity/single_use.py @@ -1,48 +1,69 @@ +import logging +from contextlib import asynccontextmanager from typing import Awaitable, Callable, Optional +from golem_core.core.activity_api import Activity from golem_core.managers.activity.defaults import ( default_on_activity_start, default_on_activity_stop, ) -from golem_core.managers.base import ActivityManager, Work, WorkResult, WorkContext +from golem_core.managers.base import ActivityManager, Work, WorkContext, WorkResult + +logger = logging.getLogger(__name__) class SingleUseActivityManager(ActivityManager): def __init__( self, get_agreement: Callable[[], Awaitable["Agreement"]], - on_activity_start: Optional[Callable[[WorkContext], Awaitable[None]]] = default_on_activity_start, - on_activity_stop: Optional[Callable[[WorkContext], Awaitable[None]]] = default_on_activity_stop, + event_bus, + on_activity_start: Optional[ + Callable[[WorkContext], Awaitable[None]] + ] = default_on_activity_start, + on_activity_stop: Optional[ + Callable[[WorkContext], Awaitable[None]] + ] = default_on_activity_stop, ): self._get_agreement = get_agreement + self._event_bus = event_bus self._on_activity_start = on_activity_start self._on_activity_stop = on_activity_stop - async def get_activity(self) -> "Activity": + @asynccontextmanager + async def _prepare_activity(self) -> Activity: while True: - # We need to release agreement if is not used agreement = await self._get_agreement() + try: - return await agreement.create_activity() + yield await agreement.create_activity() except Exception: pass + finally: + self._event_bus.emit(AgreementReleased(agreement=agreement)) async def do_work(self, work: Work) -> WorkResult: - activity = await self.get_activity() - work_context = WorkContext(activity) + async with self._prepare_activity() as activity: + work_context = WorkContext(activity) - if self._on_activity_start: - await self._on_activity_start(work_context) + if self._on_activity_start: + await self._on_activity_start(work_context) + + try: + work_result = await work(work_context) + except Exception as e: + work_result = WorkResult(exception=e) + else: + if not isinstance(work_result, WorkResult): + work_result = WorkResult(result=work_result) - try: - work_result = await work(work_context) - except Exception as e: - work_result = WorkResult(exception=e) - else: - if not isinstance(work_result, WorkResult): - work_result = WorkResult(result=work_result) + if self._on_activity_stop: + await self._on_activity_stop(work_context) - if self._on_activity_stop: - await self._on_activity_stop(work_context) + if not activity.terminated: + logger.warning( + "SingleUseActivityManager expects that activity will be terminated" + " after its work is finished. Looks like you forgot calling" + " `context.terminate()` in custom `on_activity_end` callback." + ) - return work_result \ No newline at end of file + return work_result diff --git a/golem_core/managers/agreement/queue.py b/golem_core/managers/agreement/single_use.py similarity index 52% rename from golem_core/managers/agreement/queue.py rename to golem_core/managers/agreement/single_use.py index 8138ed2e..c06f2a15 100644 --- a/golem_core/managers/agreement/queue.py +++ b/golem_core/managers/agreement/single_use.py @@ -1,12 +1,13 @@ -from typing import Callable, Awaitable +from typing import Awaitable, Callable from golem_core.core.market_api import Agreement from golem_core.managers.base import AgreementManager -class QueueAgreementManager(AgreementManager): - def __init__(self, get_offer: Callable[[], Awaitable['Offer']]): +class SingleUseAgreementManager(AgreementManager): + def __init__(self, get_offer: Callable[[], Awaitable["Offer"]], event_bus): self._get_offer = get_offer + self._event_bus = event_bus async def get_agreement(self) -> Agreement: while True: @@ -19,4 +20,11 @@ async def get_agreement(self) -> Agreement: except Exception: pass else: + self._event_bus.register( + AgreementReleased(agreement=agreement), self._on_agreement_released + ) return agreement + + async def _on_agreement_released(self, event) -> None: + agreement = event.agreement + await agreement.terminate() diff --git a/golem_core/managers/base.py b/golem_core/managers/base.py index 7727ff6e..3bc9ba92 100644 --- a/golem_core/managers/base.py +++ b/golem_core/managers/base.py @@ -1,8 +1,8 @@ from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Awaitable, Callable, List, Optional, Any, Dict, Union +from typing import Any, Awaitable, Callable, Dict, List, Optional, Union -from golem_core.core.activity_api import commands, Activity, Script +from golem_core.core.activity_api import Activity, Script, commands class Batch: @@ -21,7 +21,7 @@ def run( command: Union[str, List[str]], *, shell: Optional[bool] = None, - shell_cmd: str = "/bin/sh" + shell_cmd: str = "/bin/sh", ): self._script.add_command(commands.Run(command, shell=shell, shell_cmd=shell_cmd)) @@ -29,6 +29,7 @@ async def __call__(self): pooling_batch = await self._activity.execute_script(self._script) return await pooling_batch.wait() + class WorkContext: def __init__(self, activity: Activity): self._activity = activity @@ -49,9 +50,11 @@ async def run( command: Union[str, List[str]], *, shell: Optional[bool] = None, - shell_cmd: str = "/bin/sh" + shell_cmd: str = "/bin/sh", ): - return await self._activity.execute_commands(commands.Run(command, shell=shell, shell_cmd=shell_cmd)) + return await self._activity.execute_commands( + commands.Run(command, shell=shell, shell_cmd=shell_cmd) + ) async def create_batch(self) -> Batch: return Batch(self._activity) diff --git a/golem_core/managers/work/decorators.py b/golem_core/managers/work/decorators.py index 4d60f2f2..809875cb 100644 --- a/golem_core/managers/work/decorators.py +++ b/golem_core/managers/work/decorators.py @@ -33,9 +33,9 @@ async def wrapper(work: Work) -> WorkResult: count += 1 errors.append(work_result.exception) - work_result.extras['retry'] = { - 'tries': count, - 'errors': errors, + work_result.extras["retry"] = { + "tries": count, + "errors": errors, } return work_result From ba717affc668e0222f612160a95868aabae24cb0 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 24 May 2023 16:20:30 +0200 Subject: [PATCH 017/123] Add Payment Manager --- examples/managers/example01.py | 11 ++---- examples/managers/example02.py | 8 +++- golem_core/managers/payment/pay_all.py | 53 ++++++++++++++++++++------ 3 files changed, 52 insertions(+), 20 deletions(-) diff --git a/examples/managers/example01.py b/examples/managers/example01.py index bd2399f3..00e7aa9a 100644 --- a/examples/managers/example01.py +++ b/examples/managers/example01.py @@ -5,19 +5,14 @@ from golem_core.core.market_api import RepositoryVmPayload from golem_core.managers.negotiation import AlfaNegotiationManager from golem_core.managers.offer import StackOfferManager - - -def get_allocation_factory(golem: GolemNode): - async def _get_allocation(): - return await golem.create_allocation(1) - - return _get_allocation +from golem_core.managers.payment.pay_all import PayAllPaymentManager async def main(): payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") async with GolemNode() as golem: - negotiation_manager = AlfaNegotiationManager(golem, get_allocation_factory(golem)) + payment_manager = PayAllPaymentManager(golem, budget=1.0) + negotiation_manager = AlfaNegotiationManager(golem, payment_manager.get_allocation) offer_manager = StackOfferManager(negotiation_manager.get_offer) await negotiation_manager.start_negotiation(payload) await offer_manager.start_consuming_offers() diff --git a/examples/managers/example02.py b/examples/managers/example02.py index 55a3e08e..04b0ee3d 100644 --- a/examples/managers/example02.py +++ b/examples/managers/example02.py @@ -46,7 +46,6 @@ async def create_agreement(): # yield agreement # return - async def work1(context: WorkContext): r = await context.run("echo 1") await r.wait() @@ -69,10 +68,17 @@ async def work3(context: WorkContext): async def main(): +<<<<<<< HEAD async with create_offer() as offer: async def get_offer(): return offer +======= + async with create_agreement() as agreement: + + async def get_agreement(): + return agreement +>>>>>>> 3389159 (Add Payment Manager) work_list = [ work1, diff --git a/golem_core/managers/payment/pay_all.py b/golem_core/managers/payment/pay_all.py index bc91a10b..b71e1e0b 100644 --- a/golem_core/managers/payment/pay_all.py +++ b/golem_core/managers/payment/pay_all.py @@ -1,23 +1,54 @@ +from decimal import Decimal +from typing import Optional + +from golem_core.core.golem_node.golem_node import PAYMENT_DRIVER, PAYMENT_NETWORK, GolemNode +from golem_core.core.payment_api.resources.allocation import Allocation +from golem_core.core.payment_api.resources.debit_note import DebitNote +from golem_core.core.payment_api.resources.invoice import Invoice +from golem_core.core.resources.events import NewResource from golem_core.managers.base import PaymentManager class PayAllPaymentManager(PaymentManager): - def __init__(self, budget, event_bus): + def __init__( + self, + golem: GolemNode, + budget: float, + network: str = PAYMENT_NETWORK, + driver: str = PAYMENT_DRIVER, + ): + self._golem = golem self._budget = budget - self._event_bus = event_bus + self._network = network + self._driver = driver - self._allocation = Allocation.create(budget=self._budget) + self._allocation: Optional[Allocation] = None - event_bus.register(InvoiceReceived(allocation=self._allocation), self.on_invoice_received) - event_bus.register( - DebitNoteReceived(allocation=self._allocation), self.on_debit_note_received + self._golem.event_bus.resource_listen(self.on_invoice_received, [NewResource], [Invoice]) + self._golem.event_bus.resource_listen( + self.on_debit_note_received, [NewResource], [DebitNote] ) - def get_allocation(self) -> "Allocation": + async def get_allocation(self) -> "Allocation": + if self._allocation is None: + self._allocation = await Allocation.create_any_account( + self._golem, Decimal(self._budget), self._network, self._driver + ) + self._golem.add_autoclose_resource(self._allocation) return self._allocation - def on_invoice_received(self, invoice: "Invoice") -> None: - invoice.pay() + async def on_invoice_received(self, invoice_event: NewResource) -> None: + invoice = invoice_event.resource + assert isinstance(invoice, Invoice) + if (await invoice.get_data(force=True)).status == "RECEIVED": + assert self._allocation is not None # TODO think of a better way + await invoice.accept_full(self._allocation) + await invoice.get_data(force=True) - def on_debit_note_received(self, debit_note: "DebitNote") -> None: - debit_note.pay() + async def on_debit_note_received(self, debit_note_event: NewResource) -> None: + debit_note = debit_note_event.resource + assert isinstance(debit_note, DebitNote) + if (await debit_note.get_data(force=True)).status == "RECEIVED": + assert self._allocation is not None # TODO think of a better way + await debit_note.accept_full(self._allocation) + await debit_note.get_data(force=True) From b19ecad7d0006c3fb97e548ceb1ea4db4ba9b11e Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 24 May 2023 16:26:28 +0200 Subject: [PATCH 018/123] Clean after conflicts --- examples/managers/example02.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/examples/managers/example02.py b/examples/managers/example02.py index 04b0ee3d..fec676d9 100644 --- a/examples/managers/example02.py +++ b/examples/managers/example02.py @@ -68,18 +68,11 @@ async def work3(context: WorkContext): async def main(): -<<<<<<< HEAD async with create_offer() as offer: async def get_offer(): return offer -======= - async with create_agreement() as agreement: - - async def get_agreement(): - return agreement ->>>>>>> 3389159 (Add Payment Manager) - + work_list = [ work1, work2, From eace510e7be1be74bab6160d5af21103be66ef53 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 24 May 2023 16:27:28 +0200 Subject: [PATCH 019/123] Clean after conflicts --- examples/managers/example02.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/managers/example02.py b/examples/managers/example02.py index fec676d9..55a3e08e 100644 --- a/examples/managers/example02.py +++ b/examples/managers/example02.py @@ -46,6 +46,7 @@ async def create_agreement(): # yield agreement # return + async def work1(context: WorkContext): r = await context.run("echo 1") await r.wait() @@ -72,7 +73,7 @@ async def main(): async def get_offer(): return offer - + work_list = [ work1, work2, From b6a4d5d8066833282943f7b86e1f802f0435a3e4 Mon Sep 17 00:00:00 2001 From: approxit Date: Mon, 29 May 2023 12:36:22 +0200 Subject: [PATCH 020/123] agreement release in progress --- golem_core/managers/activity/single_use.py | 60 ++++++++++++++++++--- golem_core/managers/agreement/__init__.py | 7 +++ golem_core/managers/agreement/events.py | 5 ++ golem_core/managers/agreement/single_use.py | 1 + golem_core/managers/base.py | 5 ++ 5 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 golem_core/managers/agreement/events.py diff --git a/golem_core/managers/activity/single_use.py b/golem_core/managers/activity/single_use.py index 8fb0d86c..ea500a37 100644 --- a/golem_core/managers/activity/single_use.py +++ b/golem_core/managers/activity/single_use.py @@ -3,10 +3,13 @@ from typing import Awaitable, Callable, Optional from golem_core.core.activity_api import Activity +from golem_core.core.events import EventBus +from golem_core.core.market_api import Agreement from golem_core.managers.activity.defaults import ( default_on_activity_start, default_on_activity_stop, ) +from golem_core.managers.agreement import AgreementReleased from golem_core.managers.base import ActivityManager, Work, WorkContext, WorkResult logger = logging.getLogger(__name__) @@ -15,8 +18,8 @@ class SingleUseActivityManager(ActivityManager): def __init__( self, - get_agreement: Callable[[], Awaitable["Agreement"]], - event_bus, + get_agreement: Callable[[], Awaitable[Agreement]], + event_bus: EventBus, on_activity_start: Optional[ Callable[[WorkContext], Awaitable[None]] ] = default_on_activity_start, @@ -31,34 +34,75 @@ def __init__( @asynccontextmanager async def _prepare_activity(self) -> Activity: + logging.debug("Calling `_prepare_activity`...") + while True: + logging.debug(f"Getting agreement...") agreement = await self._get_agreement() + logging.debug(f"Getting agreement done with `{agreement}`") try: - yield await agreement.create_activity() + logging.debug(f"Creating activity...") + + activity = await agreement.create_activity() + + logging.debug(f"Creating activity done") + + logging.debug(f"Yielding activity...") + yield activity + + logging.debug(f"Yielding activity done") + + break except Exception: - pass + logging.debug(f"Creating activity failed, but will be retried with new agreement") finally: - self._event_bus.emit(AgreementReleased(agreement=agreement)) + event = AgreementReleased(agreement=agreement) + + logging.debug(f"Releasing agreement by emitting `{event}`...") + + self._event_bus.emit(event) + + logging.debug(f"Releasing agreement by emitting `{event}` done") + + logging.debug("Calling `_prepare_activity` done") async def do_work(self, work: Work) -> WorkResult: + logger.debug("Calling `do_work`...") + + work_result = None + async with self._prepare_activity() as activity: work_context = WorkContext(activity) if self._on_activity_start: + logging.debug("Calling `on_activity_start`...") + await self._on_activity_start(work_context) + logging.debug("Calling `on_activity_start` done") + try: + logging.debug("Calling `work`...") work_result = await work(work_context) except Exception as e: + logging.debug(f"Calling `work` done with exception `{e}`") work_result = WorkResult(exception=e) else: - if not isinstance(work_result, WorkResult): + if isinstance(work_result, WorkResult): + logging.debug(f"Calling `work` done with explicit result `{work_result}`") + else: + logging.debug(f"Calling `work` done with implicit result `{work_result}`") + work_result = WorkResult(result=work_result) if self._on_activity_stop: + logging.debug("Calling `on_activity_stop`...") + await self._on_activity_stop(work_context) + logging.debug("Calling `on_activity_stop` done") + if not activity.terminated: logger.warning( "SingleUseActivityManager expects that activity will be terminated" @@ -66,4 +110,6 @@ async def do_work(self, work: Work) -> WorkResult: " `context.terminate()` in custom `on_activity_end` callback." ) - return work_result + logger.debug("Calling `do_work` done") + + return work_result diff --git a/golem_core/managers/agreement/__init__.py b/golem_core/managers/agreement/__init__.py index e69de29b..887ba0b9 100644 --- a/golem_core/managers/agreement/__init__.py +++ b/golem_core/managers/agreement/__init__.py @@ -0,0 +1,7 @@ +from golem_core.managers.agreement.events import AgreementReleased +from golem_core.managers.agreement.single_use import SingleUseAgreementManager + +__all__ = ( + "AgreementReleased", + "SingleUseAgreementManager", +) diff --git a/golem_core/managers/agreement/events.py b/golem_core/managers/agreement/events.py new file mode 100644 index 00000000..5a14728d --- /dev/null +++ b/golem_core/managers/agreement/events.py @@ -0,0 +1,5 @@ +from golem_core.managers.base import ManagerEvent + + +class AgreementReleased(ManagerEvent): + pass diff --git a/golem_core/managers/agreement/single_use.py b/golem_core/managers/agreement/single_use.py index c06f2a15..db917063 100644 --- a/golem_core/managers/agreement/single_use.py +++ b/golem_core/managers/agreement/single_use.py @@ -1,6 +1,7 @@ from typing import Awaitable, Callable from golem_core.core.market_api import Agreement +from golem_core.managers.agreement.events import AgreementReleased from golem_core.managers.base import AgreementManager diff --git a/golem_core/managers/base.py b/golem_core/managers/base.py index 3bc9ba92..68af31cf 100644 --- a/golem_core/managers/base.py +++ b/golem_core/managers/base.py @@ -3,6 +3,7 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional, Union from golem_core.core.activity_api import Activity, Script, commands +from golem_core.core.events import Event class Batch: @@ -80,6 +81,10 @@ def __call__(self, context: WorkContext) -> Awaitable[Optional[WorkResult]]: DoWorkCallable = Callable[[Work], Awaitable[WorkResult]] +class ManagerEvent(Event, ABC): + pass + + class Manager(ABC): ... From 52f94a2c06af931e02a09a967116d256d674d6b0 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 29 May 2023 13:58:15 +0200 Subject: [PATCH 021/123] Add logging --- golem_core/managers/negotiation/negotiation.py | 11 +++++++++-- golem_core/managers/offer/offer.py | 12 ++++++++++-- golem_core/managers/payment/pay_all.py | 7 +++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/golem_core/managers/negotiation/negotiation.py b/golem_core/managers/negotiation/negotiation.py index 7f2fd915..5436d2e6 100644 --- a/golem_core/managers/negotiation/negotiation.py +++ b/golem_core/managers/negotiation/negotiation.py @@ -1,4 +1,5 @@ import asyncio +import logging from datetime import datetime, timezone from typing import AsyncIterator, Awaitable, Callable, List @@ -8,6 +9,8 @@ from golem_core.core.payment_api import Allocation from golem_core.managers.base import NegotiationManager +logger = logging.getLogger(__name__) + class AlfaNegotiationManager(NegotiationManager): def __init__( @@ -19,13 +22,16 @@ def __init__( self._ready_offers: asyncio.Queue[Proposal] = asyncio.Queue() async def get_offer(self) -> Proposal: + logger.debug("Returning offer") return await self._ready_offers.get() async def start_negotiation(self, payload: Payload) -> None: + logger.debug("Starting negotiations") self._negotiations.append(asyncio.create_task(self._negotiate_task(payload))) async def stop_negotiation(self) -> None: for task in self._negotiations: + logger.debug("Stopping negotiations") task.cancel() async def _negotiate_task(self, payload: Payload) -> None: @@ -35,6 +41,7 @@ async def _negotiate_task(self, payload: Payload) -> None: await self._ready_offers.put(offer) async def _build_demand(self, allocation: Allocation, payload: Payload) -> Demand: + logger.debug("Creating demand") demand_builder = DemandBuilder() await demand_builder.add( @@ -61,11 +68,11 @@ async def _build_demand(self, allocation: Allocation, payload: Payload) -> Deman async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: try: async for initial in demand.initial_proposals(): - print("Got initial...") + logger.debug("Got initial proposal") try: pending = await initial.respond() except Exception as err: - print( + logger.debug( f"Unable to respond to initialproposal {initial.id}. Got {type(err)}\n{err}" ) continue diff --git a/golem_core/managers/offer/offer.py b/golem_core/managers/offer/offer.py index 2283cce9..20c52e1b 100644 --- a/golem_core/managers/offer/offer.py +++ b/golem_core/managers/offer/offer.py @@ -1,8 +1,10 @@ import asyncio +import logging from typing import List from golem_core.core.market_api import Proposal +logger = logging.getLogger(__name__) Offer = Proposal @@ -13,15 +15,21 @@ def __init__(self, get_offer) -> None: self._tasks: List[asyncio.Task] = [] async def start_consuming_offers(self) -> None: + logger.debug("Starting manager") self._tasks.append(asyncio.create_task(self._consume_offers())) async def stop_consuming_offers(self) -> None: for task in self._tasks: + logger.debug("Stopping manager") task.cancel() async def _consume_offers(self) -> None: while True: - await self._offers.put(await self._get_offer()) + offer = await self._get_offer() + logger.debug("Adding offer to the stack") + await self._offers.put(offer) async def get_offer(self) -> Offer: - return await self._offers.get() + offer = await self._offers.get() + logger.debug("Returning offer from the stack") + return offer diff --git a/golem_core/managers/payment/pay_all.py b/golem_core/managers/payment/pay_all.py index b71e1e0b..78ff8959 100644 --- a/golem_core/managers/payment/pay_all.py +++ b/golem_core/managers/payment/pay_all.py @@ -1,3 +1,4 @@ +import logging from decimal import Decimal from typing import Optional @@ -8,6 +9,8 @@ from golem_core.core.resources.events import NewResource from golem_core.managers.base import PaymentManager +logger = logging.getLogger(__name__) + class PayAllPaymentManager(PaymentManager): def __init__( @@ -31,13 +34,16 @@ def __init__( async def get_allocation(self) -> "Allocation": if self._allocation is None: + logger.debug("Creating allocation") self._allocation = await Allocation.create_any_account( self._golem, Decimal(self._budget), self._network, self._driver ) self._golem.add_autoclose_resource(self._allocation) + logger.debug("Returning allocation") return self._allocation async def on_invoice_received(self, invoice_event: NewResource) -> None: + logger.debug("Got invoice") invoice = invoice_event.resource assert isinstance(invoice, Invoice) if (await invoice.get_data(force=True)).status == "RECEIVED": @@ -46,6 +52,7 @@ async def on_invoice_received(self, invoice_event: NewResource) -> None: await invoice.get_data(force=True) async def on_debit_note_received(self, debit_note_event: NewResource) -> None: + logger.debug("Got debit note") debit_note = debit_note_event.resource assert isinstance(debit_note, DebitNote) if (await debit_note.get_data(force=True)).status == "RECEIVED": From d3322437a601b43a682e1d604c0681d6840da208 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 29 May 2023 14:08:07 +0200 Subject: [PATCH 022/123] Add example3 --- examples/managers/example03.py | 71 ++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 examples/managers/example03.py diff --git a/examples/managers/example03.py b/examples/managers/example03.py new file mode 100644 index 00000000..bfd6422b --- /dev/null +++ b/examples/managers/example03.py @@ -0,0 +1,71 @@ +import asyncio +from datetime import datetime + +from golem_core.core.golem_node.golem_node import GolemNode +from golem_core.core.market_api import RepositoryVmPayload +from golem_core.managers.activity.single_use import SingleUseActivityManager +from golem_core.managers.agreement.single_use import SingleUseAgreementManager +from golem_core.managers.base import WorkContext +from golem_core.managers.negotiation import AlfaNegotiationManager +from golem_core.managers.offer import StackOfferManager +from golem_core.managers.payment.pay_all import PayAllPaymentManager +from golem_core.managers.work.sequential import SequentialWorkManager + + +async def work1(context: WorkContext): + r = await context.run("echo 1") + await r.wait() + for event in r.events: + print(event.stdout) + + +async def work2(context: WorkContext): + r = await context.run("echo 2") + await r.wait() + for event in r.events: + print(event.stdout) + + +async def work3(context: WorkContext): + r = await context.run("echo 3") + await r.wait() + for event in r.events: + print(event.stdout) + + +async def main(): + payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") + + work_list = [ + work1, + # work2, + # work3, + ] + + async with GolemNode() as golem: + payment_manager = PayAllPaymentManager(golem, budget=1.0) + negotiation_manager = AlfaNegotiationManager(golem, payment_manager.get_allocation) + offer_manager = StackOfferManager(negotiation_manager.get_offer) + await negotiation_manager.start_negotiation(payload) + await offer_manager.start_consuming_offers() + + agreement_manager = SingleUseAgreementManager(offer_manager.get_offer) + + activity_manager = SingleUseActivityManager( + agreement_manager.get_agreement, + ) + + work_manager = SequentialWorkManager(activity_manager.do_work) + + print("starting to work...") + results = await work_manager.do_work_list(work_list) + print("work done") + print(results) + + print(f"{datetime.utcnow()} stopping example...") + await offer_manager.stop_consuming_offers() + await negotiation_manager.stop_negotiation() + + +if __name__ == "__main__": + asyncio.run(main()) From f6617211658bfbfb232674678c960312ca3f1c25 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 29 May 2023 14:11:19 +0200 Subject: [PATCH 023/123] Add example3 --- examples/managers/example03.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/examples/managers/example03.py b/examples/managers/example03.py index bfd6422b..4d47d78d 100644 --- a/examples/managers/example03.py +++ b/examples/managers/example03.py @@ -46,17 +46,15 @@ async def main(): payment_manager = PayAllPaymentManager(golem, budget=1.0) negotiation_manager = AlfaNegotiationManager(golem, payment_manager.get_allocation) offer_manager = StackOfferManager(negotiation_manager.get_offer) - await negotiation_manager.start_negotiation(payload) - await offer_manager.start_consuming_offers() - - agreement_manager = SingleUseAgreementManager(offer_manager.get_offer) - + agreement_manager = SingleUseAgreementManager(offer_manager.get_offer, golem.event_bus) activity_manager = SingleUseActivityManager( - agreement_manager.get_agreement, + agreement_manager.get_agreement, golem.event_bus ) - work_manager = SequentialWorkManager(activity_manager.do_work) + await negotiation_manager.start_negotiation(payload) + await offer_manager.start_consuming_offers() + print("starting to work...") results = await work_manager.do_work_list(work_list) print("work done") From 6dd8048071d782a1da8813113c7316f6fa852e44 Mon Sep 17 00:00:00 2001 From: approxit Date: Mon, 29 May 2023 14:16:10 +0200 Subject: [PATCH 024/123] working activity manager --- examples/managers/example02.py | 31 +++++++------ golem_core/managers/activity/single_use.py | 50 +++++++++++---------- golem_core/managers/agreement/single_use.py | 45 ++++++++++++++----- golem_core/managers/base.py | 17 ++++--- golem_core/utils/logging.py | 28 ++++++++++++ 5 files changed, 115 insertions(+), 56 deletions(-) diff --git a/examples/managers/example02.py b/examples/managers/example02.py index 55a3e08e..8f17b8eb 100644 --- a/examples/managers/example02.py +++ b/examples/managers/example02.py @@ -1,4 +1,5 @@ import asyncio +import logging.config from contextlib import asynccontextmanager from golem_core.core.golem_node import GolemNode @@ -7,10 +8,11 @@ from golem_core.managers.agreement.single_use import SingleUseAgreementManager from golem_core.managers.base import WorkContext from golem_core.managers.work import SequentialWorkManager +from golem_core.utils.logging import DEFAULT_LOGGING @asynccontextmanager -async def create_agreement(): +async def create_proposal(): payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") print("Entering golem context...") @@ -35,7 +37,7 @@ async def create_agreement(): except StopAsyncIteration: continue - yield their_response + yield golem, their_response # print('Creating agreement...') # agreement = await their_response.create_agreement() @@ -51,39 +53,41 @@ async def work1(context: WorkContext): r = await context.run("echo 1") await r.wait() for event in r.events: - print(event.stdout) + print(event.stdout, flush=True) async def work2(context: WorkContext): r = await context.run("echo 2") await r.wait() for event in r.events: - print(event.stdout) + print(event.stdout, flush=True) async def work3(context: WorkContext): r = await context.run("echo 3") await r.wait() for event in r.events: - print(event.stdout) + print(event.stdout, flush=True) async def main(): - async with create_offer() as offer: + logging.config.dictConfig(DEFAULT_LOGGING) - async def get_offer(): - return offer + async with create_proposal() as (golem, proposal): + async def get_proposal(): + return proposal work_list = [ work1, - work2, - work3, + # work2, + # work3, ] - agreement_manager = SingleUseAgreementManager(get_offer) + agreement_manager = SingleUseAgreementManager(get_proposal, golem.event_bus) activity_manager = SingleUseActivityManager( agreement_manager.get_agreement, + golem.event_bus, ) work_manager = SequentialWorkManager(activity_manager.do_work) @@ -92,8 +96,9 @@ async def get_offer(): results = await work_manager.do_work_list(work_list) print("work done") print(results) - - await agreement_manager.close() + print("sleeping...") + await asyncio.sleep(10) + print("sleeping done") if __name__ == "__main__": diff --git a/golem_core/managers/activity/single_use.py b/golem_core/managers/activity/single_use.py index ea500a37..8839b66d 100644 --- a/golem_core/managers/activity/single_use.py +++ b/golem_core/managers/activity/single_use.py @@ -13,6 +13,7 @@ from golem_core.managers.base import ActivityManager, Work, WorkContext, WorkResult logger = logging.getLogger(__name__) +print(logger) class SingleUseActivityManager(ActivityManager): @@ -34,76 +35,77 @@ def __init__( @asynccontextmanager async def _prepare_activity(self) -> Activity: - logging.debug("Calling `_prepare_activity`...") + logger.debug("Calling `_prepare_activity`...") while True: - logging.debug(f"Getting agreement...") + logger.debug(f"Getting agreement...") + agreement = await self._get_agreement() - logging.debug(f"Getting agreement done with `{agreement}`") + + logger.debug(f"Getting agreement done with `{agreement}`") try: - logging.debug(f"Creating activity...") + logger.debug(f"Creating activity...") activity = await agreement.create_activity() - logging.debug(f"Creating activity done") + logger.debug(f"Creating activity done") + + logger.debug(f"Yielding activity...") - logging.debug(f"Yielding activity...") yield activity - logging.debug(f"Yielding activity done") + logger.debug(f"Yielding activity done") break - except Exception: - logging.debug(f"Creating activity failed, but will be retried with new agreement") + except Exception as e: + logger.debug(f"Creating activity failed with {e}, but will be retried with new agreement") finally: - event = AgreementReleased(agreement=agreement) + event = AgreementReleased(agreement) - logging.debug(f"Releasing agreement by emitting `{event}`...") + logger.debug(f"Releasing agreement by emitting `{event}`...") self._event_bus.emit(event) - logging.debug(f"Releasing agreement by emitting `{event}` done") + logger.debug(f"Releasing agreement by emitting `{event}` done") - logging.debug("Calling `_prepare_activity` done") + logger.debug("Calling `_prepare_activity` done") async def do_work(self, work: Work) -> WorkResult: logger.debug("Calling `do_work`...") - work_result = None - async with self._prepare_activity() as activity: work_context = WorkContext(activity) if self._on_activity_start: - logging.debug("Calling `on_activity_start`...") + logger.debug("Calling `on_activity_start`...") await self._on_activity_start(work_context) - logging.debug("Calling `on_activity_start` done") + logger.debug("Calling `on_activity_start` done") try: - logging.debug("Calling `work`...") + logger.debug("Calling `work`...") work_result = await work(work_context) except Exception as e: - logging.debug(f"Calling `work` done with exception `{e}`") + logger.debug(f"Calling `work` done with exception `{e}`") work_result = WorkResult(exception=e) else: if isinstance(work_result, WorkResult): - logging.debug(f"Calling `work` done with explicit result `{work_result}`") + logger.debug(f"Calling `work` done with explicit result `{work_result}`") else: - logging.debug(f"Calling `work` done with implicit result `{work_result}`") + logger.debug(f"Calling `work` done with implicit result `{work_result}`") work_result = WorkResult(result=work_result) if self._on_activity_stop: - logging.debug("Calling `on_activity_stop`...") + logger.debug("Calling `on_activity_stop`...") await self._on_activity_stop(work_context) - logging.debug("Calling `on_activity_stop` done") + logger.debug("Calling `on_activity_stop` done") - if not activity.terminated: + if not activity.destroyed: logger.warning( "SingleUseActivityManager expects that activity will be terminated" " after its work is finished. Looks like you forgot calling" diff --git a/golem_core/managers/agreement/single_use.py b/golem_core/managers/agreement/single_use.py index db917063..b5eb014a 100644 --- a/golem_core/managers/agreement/single_use.py +++ b/golem_core/managers/agreement/single_use.py @@ -1,31 +1,56 @@ +import logging from typing import Awaitable, Callable -from golem_core.core.market_api import Agreement +from golem_core.core.events import EventBus +from golem_core.core.market_api import Agreement, Proposal from golem_core.managers.agreement.events import AgreementReleased from golem_core.managers.base import AgreementManager +logger = logging.getLogger(__name__) + class SingleUseAgreementManager(AgreementManager): - def __init__(self, get_offer: Callable[[], Awaitable["Offer"]], event_bus): - self._get_offer = get_offer + def __init__(self, get_proposal: Callable[[], Awaitable[Proposal]], event_bus: EventBus): + self._get_proposal = get_proposal self._event_bus = event_bus async def get_agreement(self) -> Agreement: + logger.debug('Getting agreement...') + while True: - offer = await self._get_offer() + logger.debug('Getting proposal...') + + proposal = await self._get_proposal() + + logger.debug(f'Getting proposal done with {proposal}') try: - agreement = await offer.create_agreement() + logger.debug(f'Creating agreement...') + + agreement = await proposal.create_agreement() + + logger.debug(f'Sending agreement to provider...') + await agreement.confirm() + + logger.debug(f'Waiting for provider approval...') + await agreement.wait_for_approval() - except Exception: - pass + except Exception as e: + logger.debug(f'Creating agreement failed with {e}. Retrying...') else: - self._event_bus.register( - AgreementReleased(agreement=agreement), self._on_agreement_released - ) + logger.debug(f'Creating agreement done') + + # TODO: Support removing callback on resource close + self._event_bus.resource_listen(self._on_agreement_released, [AgreementReleased], [Agreement], [agreement.id]) + + logger.debug(f'Getting agreement done with {agreement}') return agreement async def _on_agreement_released(self, event) -> None: + logger.debug('Calling `_on_agreement_released`...') + agreement = event.agreement await agreement.terminate() + + logger.debug('Calling `_on_agreement_released` done') diff --git a/golem_core/managers/base.py b/golem_core/managers/base.py index 68af31cf..874b9b6d 100644 --- a/golem_core/managers/base.py +++ b/golem_core/managers/base.py @@ -4,6 +4,9 @@ from golem_core.core.activity_api import Activity, Script, commands from golem_core.core.events import Event +from golem_core.core.market_api import Proposal, Agreement +from golem_core.core.payment_api import Allocation +from golem_core.core.resources import ResourceEvent class Batch: @@ -81,7 +84,7 @@ def __call__(self, context: WorkContext) -> Awaitable[Optional[WorkResult]]: DoWorkCallable = Callable[[Work], Awaitable[WorkResult]] -class ManagerEvent(Event, ABC): +class ManagerEvent(ResourceEvent, ABC): pass @@ -91,33 +94,29 @@ class Manager(ABC): class PaymentManager(Manager, ABC): @abstractmethod - async def get_allocation(self) -> "Allocation": + async def get_allocation(self) -> Allocation: ... class NegotiationManager(Manager, ABC): @abstractmethod - async def get_offer(self) -> "Offer": + async def get_offer(self) -> Proposal: ... class OfferManager(Manager, ABC): @abstractmethod - async def get_offer(self) -> "Offer": + async def get_offer(self) -> Proposal: ... class AgreementManager(Manager, ABC): @abstractmethod - async def get_agreement(self) -> "Agreement": + async def get_agreement(self) -> Agreement: ... class ActivityManager(Manager, ABC): - @abstractmethod - async def get_activity(self) -> "Activity": - ... - @abstractmethod async def do_work(self, work: Work) -> WorkResult: ... diff --git a/golem_core/utils/logging.py b/golem_core/utils/logging.py index 56c3b49d..c215e9af 100644 --- a/golem_core/utils/logging.py +++ b/golem_core/utils/logging.py @@ -6,6 +6,34 @@ from golem_core.core.events import Event +DEFAULT_LOGGING = { + "version": 1, + 'disable_existing_loggers': False, + 'formatters': { + 'default': { + 'format': '[%(asctime)s %(levelname)s %(name)s] %(message)s', + }, + }, + 'handlers': { + 'console': { + 'formatter': 'default', + 'class': 'logging.StreamHandler' + } + }, + 'loggers': { + '': { + 'level': 'INFO', + 'handlers': [ + 'console', + ] + }, + 'golem_core': { + 'level': 'DEBUG', + } + } +} + + class _YagnaDatetimeFormatter(logging.Formatter): """Custom log Formatter that formats datetime using the same convention yagna uses.""" From afa8e47ae9851281b31c8fc52665ca76c29d102f Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 29 May 2023 14:46:02 +0200 Subject: [PATCH 025/123] Fix _on_agreement_released --- examples/managers/example03.py | 10 ++++++++++ golem_core/managers/agreement/single_use.py | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/examples/managers/example03.py b/examples/managers/example03.py index 4d47d78d..500077e8 100644 --- a/examples/managers/example03.py +++ b/examples/managers/example03.py @@ -1,5 +1,6 @@ import asyncio from datetime import datetime +import logging.config from golem_core.core.golem_node.golem_node import GolemNode from golem_core.core.market_api import RepositoryVmPayload @@ -10,13 +11,17 @@ from golem_core.managers.offer import StackOfferManager from golem_core.managers.payment.pay_all import PayAllPaymentManager from golem_core.managers.work.sequential import SequentialWorkManager +from golem_core.utils.logging import DEFAULT_LOGGING async def work1(context: WorkContext): r = await context.run("echo 1") await r.wait() + result = "" for event in r.events: print(event.stdout) + result += event.stdout + return result async def work2(context: WorkContext): @@ -34,6 +39,7 @@ async def work3(context: WorkContext): async def main(): + logging.config.dictConfig(DEFAULT_LOGGING) payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") work_list = [ @@ -64,6 +70,10 @@ async def main(): await offer_manager.stop_consuming_offers() await negotiation_manager.stop_negotiation() + # TODO wait for invoices and debit notes + for _ in range(20): + await asyncio.sleep(1) + if __name__ == "__main__": asyncio.run(main()) diff --git a/golem_core/managers/agreement/single_use.py b/golem_core/managers/agreement/single_use.py index b5eb014a..a1df0825 100644 --- a/golem_core/managers/agreement/single_use.py +++ b/golem_core/managers/agreement/single_use.py @@ -47,10 +47,10 @@ async def get_agreement(self) -> Agreement: logger.debug(f'Getting agreement done with {agreement}') return agreement - async def _on_agreement_released(self, event) -> None: + async def _on_agreement_released(self, event: AgreementReleased) -> None: logger.debug('Calling `_on_agreement_released`...') - agreement = event.agreement + agreement: Agreement = event.resource await agreement.terminate() logger.debug('Calling `_on_agreement_released` done') From 074b0be6a5424128fddc1a44de436812d5432981 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 29 May 2023 14:47:19 +0200 Subject: [PATCH 026/123] Join examples into basic composition --- .../{example03.py => basic_composition.py} | 0 examples/managers/example01.py | 31 ------ examples/managers/example02.py | 105 ------------------ 3 files changed, 136 deletions(-) rename examples/managers/{example03.py => basic_composition.py} (100%) delete mode 100644 examples/managers/example01.py delete mode 100644 examples/managers/example02.py diff --git a/examples/managers/example03.py b/examples/managers/basic_composition.py similarity index 100% rename from examples/managers/example03.py rename to examples/managers/basic_composition.py diff --git a/examples/managers/example01.py b/examples/managers/example01.py deleted file mode 100644 index 00e7aa9a..00000000 --- a/examples/managers/example01.py +++ /dev/null @@ -1,31 +0,0 @@ -import asyncio -from datetime import datetime - -from golem_core.core.golem_node.golem_node import GolemNode -from golem_core.core.market_api import RepositoryVmPayload -from golem_core.managers.negotiation import AlfaNegotiationManager -from golem_core.managers.offer import StackOfferManager -from golem_core.managers.payment.pay_all import PayAllPaymentManager - - -async def main(): - payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") - async with GolemNode() as golem: - payment_manager = PayAllPaymentManager(golem, budget=1.0) - negotiation_manager = AlfaNegotiationManager(golem, payment_manager.get_allocation) - offer_manager = StackOfferManager(negotiation_manager.get_offer) - await negotiation_manager.start_negotiation(payload) - await offer_manager.start_consuming_offers() - - for i in range(10): - print(f"Got offer {i}: {(await offer_manager.get_offer()).id}...") - print(f"{datetime.utcnow()} sleeping...") - await asyncio.sleep(1) - - print(f"{datetime.utcnow()} stopping negotiations...") - await offer_manager.stop_consuming_offers() - await negotiation_manager.stop_negotiation() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/managers/example02.py b/examples/managers/example02.py deleted file mode 100644 index 8f17b8eb..00000000 --- a/examples/managers/example02.py +++ /dev/null @@ -1,105 +0,0 @@ -import asyncio -import logging.config -from contextlib import asynccontextmanager - -from golem_core.core.golem_node import GolemNode -from golem_core.core.market_api import RepositoryVmPayload -from golem_core.managers.activity import SingleUseActivityManager -from golem_core.managers.agreement.single_use import SingleUseAgreementManager -from golem_core.managers.base import WorkContext -from golem_core.managers.work import SequentialWorkManager -from golem_core.utils.logging import DEFAULT_LOGGING - - -@asynccontextmanager -async def create_proposal(): - payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") - - print("Entering golem context...") - async with GolemNode() as golem: - print("Creating allocation...") - allocation = await golem.create_allocation(1) - print("Creating demand...") - demand = await golem.create_demand(payload, allocations=[allocation]) - - print("Gathering initial proposals...") - async for proposal in demand.initial_proposals(): - print("Responding to initial proposal...") - try: - our_response = await proposal.respond() - except Exception as e: - print(str(e)) - continue - - print("Waiting for initial proposal...") - try: - their_response = await our_response.responses().__anext__() - except StopAsyncIteration: - continue - - yield golem, their_response - - # print('Creating agreement...') - # agreement = await their_response.create_agreement() - # await agreement.confirm() - # await agreement.wait_for_approval() - - # print('Yielding agreement...') - # yield agreement - # return - - -async def work1(context: WorkContext): - r = await context.run("echo 1") - await r.wait() - for event in r.events: - print(event.stdout, flush=True) - - -async def work2(context: WorkContext): - r = await context.run("echo 2") - await r.wait() - for event in r.events: - print(event.stdout, flush=True) - - -async def work3(context: WorkContext): - r = await context.run("echo 3") - await r.wait() - for event in r.events: - print(event.stdout, flush=True) - - -async def main(): - logging.config.dictConfig(DEFAULT_LOGGING) - - async with create_proposal() as (golem, proposal): - async def get_proposal(): - return proposal - - work_list = [ - work1, - # work2, - # work3, - ] - - agreement_manager = SingleUseAgreementManager(get_proposal, golem.event_bus) - - activity_manager = SingleUseActivityManager( - agreement_manager.get_agreement, - golem.event_bus, - ) - - work_manager = SequentialWorkManager(activity_manager.do_work) - - print("starting to work...") - results = await work_manager.do_work_list(work_list) - print("work done") - print(results) - print("sleeping...") - await asyncio.sleep(10) - print("sleeping done") - - -if __name__ == "__main__": - asyncio.run(main()) From 881c4c6b1ba0cde902c34827ffcca59d53bbe1c0 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 29 May 2023 14:48:21 +0200 Subject: [PATCH 027/123] Formatting --- examples/managers/basic_composition.py | 2 +- golem_core/managers/activity/single_use.py | 4 ++- golem_core/managers/agreement/single_use.py | 28 +++++++++-------- golem_core/managers/base.py | 3 +- golem_core/utils/logging.py | 35 +++++++++------------ 5 files changed, 35 insertions(+), 37 deletions(-) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index 500077e8..09da8810 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -1,6 +1,6 @@ import asyncio -from datetime import datetime import logging.config +from datetime import datetime from golem_core.core.golem_node.golem_node import GolemNode from golem_core.core.market_api import RepositoryVmPayload diff --git a/golem_core/managers/activity/single_use.py b/golem_core/managers/activity/single_use.py index 8839b66d..e7b563e6 100644 --- a/golem_core/managers/activity/single_use.py +++ b/golem_core/managers/activity/single_use.py @@ -59,7 +59,9 @@ async def _prepare_activity(self) -> Activity: break except Exception as e: - logger.debug(f"Creating activity failed with {e}, but will be retried with new agreement") + logger.debug( + f"Creating activity failed with {e}, but will be retried with new agreement" + ) finally: event = AgreementReleased(agreement) diff --git a/golem_core/managers/agreement/single_use.py b/golem_core/managers/agreement/single_use.py index a1df0825..8877fad1 100644 --- a/golem_core/managers/agreement/single_use.py +++ b/golem_core/managers/agreement/single_use.py @@ -6,51 +6,53 @@ from golem_core.managers.agreement.events import AgreementReleased from golem_core.managers.base import AgreementManager - logger = logging.getLogger(__name__) + class SingleUseAgreementManager(AgreementManager): def __init__(self, get_proposal: Callable[[], Awaitable[Proposal]], event_bus: EventBus): self._get_proposal = get_proposal self._event_bus = event_bus async def get_agreement(self) -> Agreement: - logger.debug('Getting agreement...') + logger.debug("Getting agreement...") while True: - logger.debug('Getting proposal...') + logger.debug("Getting proposal...") proposal = await self._get_proposal() - logger.debug(f'Getting proposal done with {proposal}') + logger.debug(f"Getting proposal done with {proposal}") try: - logger.debug(f'Creating agreement...') + logger.debug(f"Creating agreement...") agreement = await proposal.create_agreement() - logger.debug(f'Sending agreement to provider...') + logger.debug(f"Sending agreement to provider...") await agreement.confirm() - logger.debug(f'Waiting for provider approval...') + logger.debug(f"Waiting for provider approval...") await agreement.wait_for_approval() except Exception as e: - logger.debug(f'Creating agreement failed with {e}. Retrying...') + logger.debug(f"Creating agreement failed with {e}. Retrying...") else: - logger.debug(f'Creating agreement done') + logger.debug(f"Creating agreement done") # TODO: Support removing callback on resource close - self._event_bus.resource_listen(self._on_agreement_released, [AgreementReleased], [Agreement], [agreement.id]) + self._event_bus.resource_listen( + self._on_agreement_released, [AgreementReleased], [Agreement], [agreement.id] + ) - logger.debug(f'Getting agreement done with {agreement}') + logger.debug(f"Getting agreement done with {agreement}") return agreement async def _on_agreement_released(self, event: AgreementReleased) -> None: - logger.debug('Calling `_on_agreement_released`...') + logger.debug("Calling `_on_agreement_released`...") agreement: Agreement = event.resource await agreement.terminate() - logger.debug('Calling `_on_agreement_released` done') + logger.debug("Calling `_on_agreement_released` done") diff --git a/golem_core/managers/base.py b/golem_core/managers/base.py index 874b9b6d..18c92cb3 100644 --- a/golem_core/managers/base.py +++ b/golem_core/managers/base.py @@ -3,8 +3,7 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional, Union from golem_core.core.activity_api import Activity, Script, commands -from golem_core.core.events import Event -from golem_core.core.market_api import Proposal, Agreement +from golem_core.core.market_api import Agreement, Proposal from golem_core.core.payment_api import Allocation from golem_core.core.resources import ResourceEvent diff --git a/golem_core/utils/logging.py b/golem_core/utils/logging.py index c215e9af..d9690ef4 100644 --- a/golem_core/utils/logging.py +++ b/golem_core/utils/logging.py @@ -8,29 +8,24 @@ DEFAULT_LOGGING = { "version": 1, - 'disable_existing_loggers': False, - 'formatters': { - 'default': { - 'format': '[%(asctime)s %(levelname)s %(name)s] %(message)s', + "disable_existing_loggers": False, + "formatters": { + "default": { + "format": "[%(asctime)s %(levelname)s %(name)s] %(message)s", }, }, - 'handlers': { - 'console': { - 'formatter': 'default', - 'class': 'logging.StreamHandler' - } - }, - 'loggers': { - '': { - 'level': 'INFO', - 'handlers': [ - 'console', - ] + "handlers": {"console": {"formatter": "default", "class": "logging.StreamHandler"}}, + "loggers": { + "": { + "level": "INFO", + "handlers": [ + "console", + ], + }, + "golem_core": { + "level": "DEBUG", }, - 'golem_core': { - 'level': 'DEBUG', - } - } + }, } From d59780e36c5d4d44cba3544c362a23be8e3ceafa Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 29 May 2023 15:01:41 +0200 Subject: [PATCH 028/123] Improve example --- examples/managers/basic_composition.py | 28 +++++++++++++------------- golem_core/managers/payment/pay_all.py | 4 ++++ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index 09da8810..fcb79d17 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -19,7 +19,7 @@ async def work1(context: WorkContext): await r.wait() result = "" for event in r.events: - print(event.stdout) + print(f"Work1 got: {event.stdout}") result += event.stdout return result @@ -27,15 +27,21 @@ async def work1(context: WorkContext): async def work2(context: WorkContext): r = await context.run("echo 2") await r.wait() + result = "" for event in r.events: - print(event.stdout) + print(f"Work2 got: {event.stdout}") + result += event.stdout + return result async def work3(context: WorkContext): r = await context.run("echo 3") await r.wait() + result = "" for event in r.events: - print(event.stdout) + print(f"Work3 got: {event.stdout}") + result += event.stdout + return result async def main(): @@ -44,8 +50,8 @@ async def main(): work_list = [ work1, - # work2, - # work3, + work2, + work3, ] async with GolemNode() as golem: @@ -57,22 +63,16 @@ async def main(): agreement_manager.get_agreement, golem.event_bus ) work_manager = SequentialWorkManager(activity_manager.do_work) - + await negotiation_manager.start_negotiation(payload) await offer_manager.start_consuming_offers() - print("starting to work...") results = await work_manager.do_work_list(work_list) - print("work done") - print(results) + print(f"work done: {results}") - print(f"{datetime.utcnow()} stopping example...") await offer_manager.stop_consuming_offers() await negotiation_manager.stop_negotiation() - - # TODO wait for invoices and debit notes - for _ in range(20): - await asyncio.sleep(1) + await payment_manager.wait_for_invoices() if __name__ == "__main__": diff --git a/golem_core/managers/payment/pay_all.py b/golem_core/managers/payment/pay_all.py index 78ff8959..82dd63d1 100644 --- a/golem_core/managers/payment/pay_all.py +++ b/golem_core/managers/payment/pay_all.py @@ -1,3 +1,4 @@ +import asyncio import logging from decimal import Decimal from typing import Optional @@ -59,3 +60,6 @@ async def on_debit_note_received(self, debit_note_event: NewResource) -> None: assert self._allocation is not None # TODO think of a better way await debit_note.accept_full(self._allocation) await debit_note.get_data(force=True) + + async def wait_for_invoices(self): + await asyncio.sleep(30) From 86f34ed5ff299f88713dd0863ccc853bff5837a7 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 29 May 2023 15:30:23 +0200 Subject: [PATCH 029/123] Rename offer and negotiation managers --- examples/managers/basic_composition.py | 19 +++++----- golem_core/managers/base.py | 8 ++--- golem_core/managers/negotiation/__init__.py | 4 +-- .../{negotiation.py => accept_all.py} | 22 ++++++------ golem_core/managers/offer/__init__.py | 3 -- golem_core/managers/offer/offer.py | 35 ------------------- golem_core/managers/proposal/__init__.py | 3 ++ golem_core/managers/proposal/stack.py | 35 +++++++++++++++++++ 8 files changed, 65 insertions(+), 64 deletions(-) rename golem_core/managers/negotiation/{negotiation.py => accept_all.py} (81%) delete mode 100644 golem_core/managers/offer/__init__.py delete mode 100644 golem_core/managers/offer/offer.py create mode 100644 golem_core/managers/proposal/__init__.py create mode 100644 golem_core/managers/proposal/stack.py diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index fcb79d17..b7e18c48 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -1,15 +1,14 @@ import asyncio import logging.config -from datetime import datetime from golem_core.core.golem_node.golem_node import GolemNode from golem_core.core.market_api import RepositoryVmPayload from golem_core.managers.activity.single_use import SingleUseActivityManager from golem_core.managers.agreement.single_use import SingleUseAgreementManager from golem_core.managers.base import WorkContext -from golem_core.managers.negotiation import AlfaNegotiationManager -from golem_core.managers.offer import StackOfferManager +from golem_core.managers.negotiation import AcceptAllNegotiationManager from golem_core.managers.payment.pay_all import PayAllPaymentManager +from golem_core.managers.proposal import StackProposalManager from golem_core.managers.work.sequential import SequentialWorkManager from golem_core.utils.logging import DEFAULT_LOGGING @@ -56,21 +55,23 @@ async def main(): async with GolemNode() as golem: payment_manager = PayAllPaymentManager(golem, budget=1.0) - negotiation_manager = AlfaNegotiationManager(golem, payment_manager.get_allocation) - offer_manager = StackOfferManager(negotiation_manager.get_offer) - agreement_manager = SingleUseAgreementManager(offer_manager.get_offer, golem.event_bus) + negotiation_manager = AcceptAllNegotiationManager(golem, payment_manager.get_allocation) + proposal_manager = StackProposalManager(negotiation_manager.get_proposal) + agreement_manager = SingleUseAgreementManager( + proposal_manager.get_proposal, golem.event_bus + ) activity_manager = SingleUseActivityManager( agreement_manager.get_agreement, golem.event_bus ) work_manager = SequentialWorkManager(activity_manager.do_work) - + await negotiation_manager.start_negotiation(payload) - await offer_manager.start_consuming_offers() + await proposal_manager.start_consuming_proposals() results = await work_manager.do_work_list(work_list) print(f"work done: {results}") - await offer_manager.stop_consuming_offers() + await proposal_manager.stop_consuming_proposals() await negotiation_manager.stop_negotiation() await payment_manager.wait_for_invoices() diff --git a/golem_core/managers/base.py b/golem_core/managers/base.py index 18c92cb3..55b96883 100644 --- a/golem_core/managers/base.py +++ b/golem_core/managers/base.py @@ -97,15 +97,15 @@ async def get_allocation(self) -> Allocation: ... -class NegotiationManager(Manager, ABC): +class ProposalNegotiationManager(Manager, ABC): @abstractmethod - async def get_offer(self) -> Proposal: + async def get_proposal(self) -> Proposal: ... -class OfferManager(Manager, ABC): +class ProposalAggregationManager(Manager, ABC): @abstractmethod - async def get_offer(self) -> Proposal: + async def get_proposal(self) -> Proposal: ... diff --git a/golem_core/managers/negotiation/__init__.py b/golem_core/managers/negotiation/__init__.py index 9136cc9f..a47383ee 100644 --- a/golem_core/managers/negotiation/__init__.py +++ b/golem_core/managers/negotiation/__init__.py @@ -1,3 +1,3 @@ -from golem_core.managers.negotiation.negotiation import AlfaNegotiationManager +from golem_core.managers.negotiation.accept_all import AcceptAllNegotiationManager -__all__ = ("AlfaNegotiationManager",) +__all__ = ("AcceptAllNegotiationManager",) diff --git a/golem_core/managers/negotiation/negotiation.py b/golem_core/managers/negotiation/accept_all.py similarity index 81% rename from golem_core/managers/negotiation/negotiation.py rename to golem_core/managers/negotiation/accept_all.py index 5436d2e6..98b89eb2 100644 --- a/golem_core/managers/negotiation/negotiation.py +++ b/golem_core/managers/negotiation/accept_all.py @@ -7,23 +7,23 @@ from golem_core.core.market_api import Demand, DemandBuilder, Payload, Proposal from golem_core.core.market_api.resources.demand.demand_offer_base import defaults as dobm_defaults from golem_core.core.payment_api import Allocation -from golem_core.managers.base import NegotiationManager +from golem_core.managers.base import ProposalNegotiationManager logger = logging.getLogger(__name__) -class AlfaNegotiationManager(NegotiationManager): +class AcceptAllNegotiationManager(ProposalNegotiationManager): def __init__( self, golem: GolemNode, get_allocation: Callable[[], Awaitable[Allocation]] ) -> None: self._golem = golem self._get_allocation = get_allocation self._negotiations: List[asyncio.Task] = [] - self._ready_offers: asyncio.Queue[Proposal] = asyncio.Queue() + self._eligible_proposals: asyncio.Queue[Proposal] = asyncio.Queue() - async def get_offer(self) -> Proposal: - logger.debug("Returning offer") - return await self._ready_offers.get() + async def get_proposal(self) -> Proposal: + logger.debug("Returning proposal") + return await self._eligible_proposals.get() async def start_negotiation(self, payload: Payload) -> None: logger.debug("Starting negotiations") @@ -37,8 +37,8 @@ async def stop_negotiation(self) -> None: async def _negotiate_task(self, payload: Payload) -> None: allocation = await self._get_allocation() demand = await self._build_demand(allocation, payload) - async for offer in self._negotiate(demand): - await self._ready_offers.put(offer) + async for proposal in self._negotiate(demand): + await self._eligible_proposals.put(proposal) async def _build_demand(self, allocation: Allocation, payload: Payload) -> Demand: logger.debug("Creating demand") @@ -70,7 +70,7 @@ async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: async for initial in demand.initial_proposals(): logger.debug("Got initial proposal") try: - pending = await initial.respond() + demand_proposal = await initial.respond() except Exception as err: logger.debug( f"Unable to respond to initialproposal {initial.id}. Got {type(err)}\n{err}" @@ -79,10 +79,10 @@ async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: try: # TODO IDK how to call `confirm` on a proposal in golem-core - confirmed = await pending.responses().__anext__() + offer_proposal = await demand_proposal.responses().__anext__() except StopAsyncIteration: continue - yield confirmed + yield offer_proposal finally: self._golem.add_autoclose_resource(demand) diff --git a/golem_core/managers/offer/__init__.py b/golem_core/managers/offer/__init__.py deleted file mode 100644 index a29dabdc..00000000 --- a/golem_core/managers/offer/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from golem_core.managers.offer.offer import StackOfferManager - -__all__ = ("StackOfferManager",) diff --git a/golem_core/managers/offer/offer.py b/golem_core/managers/offer/offer.py deleted file mode 100644 index 20c52e1b..00000000 --- a/golem_core/managers/offer/offer.py +++ /dev/null @@ -1,35 +0,0 @@ -import asyncio -import logging -from typing import List - -from golem_core.core.market_api import Proposal - -logger = logging.getLogger(__name__) -Offer = Proposal - - -class StackOfferManager: - def __init__(self, get_offer) -> None: - self._get_offer = get_offer - self._offers: asyncio.Queue[Proposal] = asyncio.Queue() - self._tasks: List[asyncio.Task] = [] - - async def start_consuming_offers(self) -> None: - logger.debug("Starting manager") - self._tasks.append(asyncio.create_task(self._consume_offers())) - - async def stop_consuming_offers(self) -> None: - for task in self._tasks: - logger.debug("Stopping manager") - task.cancel() - - async def _consume_offers(self) -> None: - while True: - offer = await self._get_offer() - logger.debug("Adding offer to the stack") - await self._offers.put(offer) - - async def get_offer(self) -> Offer: - offer = await self._offers.get() - logger.debug("Returning offer from the stack") - return offer diff --git a/golem_core/managers/proposal/__init__.py b/golem_core/managers/proposal/__init__.py new file mode 100644 index 00000000..9a163edc --- /dev/null +++ b/golem_core/managers/proposal/__init__.py @@ -0,0 +1,3 @@ +from golem_core.managers.proposal.stack import StackProposalManager + +__all__ = ("StackProposalManager",) diff --git a/golem_core/managers/proposal/stack.py b/golem_core/managers/proposal/stack.py new file mode 100644 index 00000000..f9292dfa --- /dev/null +++ b/golem_core/managers/proposal/stack.py @@ -0,0 +1,35 @@ +import asyncio +import logging +from typing import List + +from golem_core.core.market_api import Proposal +from golem_core.managers.base import ProposalAggregationManager + +logger = logging.getLogger(__name__) + + +class StackProposalManager(ProposalAggregationManager): + def __init__(self, get_proposal) -> None: + self._get_proposal = get_proposal + self._proposals: asyncio.Queue[Proposal] = asyncio.Queue() + self._tasks: List[asyncio.Task] = [] + + async def start_consuming_proposals(self) -> None: + logger.debug("Starting manager") + self._tasks.append(asyncio.create_task(self._consume_proposals())) + + async def stop_consuming_proposals(self) -> None: + for task in self._tasks: + logger.debug("Stopping manager") + task.cancel() + + async def _consume_proposals(self) -> None: + while True: + proposal = await self._get_proposal() + logger.debug("Adding proposal to the stack") + await self._proposals.put(proposal) + + async def get_proposal(self) -> Proposal: + proposal = await self._proposals.get() + logger.debug("Returning proposal from the stack") + return proposal From b8217ab5c501794908b8ea1ce10544f3e533de1a Mon Sep 17 00:00:00 2001 From: approxit Date: Mon, 29 May 2023 16:18:29 +0200 Subject: [PATCH 030/123] a little bit more of logs --- golem_core/managers/activity/single_use.py | 13 +++++----- golem_core/managers/work/sequential.py | 28 ++++++++++++++++++---- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/golem_core/managers/activity/single_use.py b/golem_core/managers/activity/single_use.py index e7b563e6..cc1e10ff 100644 --- a/golem_core/managers/activity/single_use.py +++ b/golem_core/managers/activity/single_use.py @@ -13,7 +13,6 @@ from golem_core.managers.base import ActivityManager, Work, WorkContext, WorkResult logger = logging.getLogger(__name__) -print(logger) class SingleUseActivityManager(ActivityManager): @@ -35,7 +34,7 @@ def __init__( @asynccontextmanager async def _prepare_activity(self) -> Activity: - logger.debug("Calling `_prepare_activity`...") + logger.debug("Preparing activity...") while True: logger.debug(f"Getting agreement...") @@ -49,7 +48,7 @@ async def _prepare_activity(self) -> Activity: activity = await agreement.create_activity() - logger.debug(f"Creating activity done") + logger.debug(f"Creating activity done with `{activity}`") logger.debug(f"Yielding activity...") @@ -60,7 +59,7 @@ async def _prepare_activity(self) -> Activity: break except Exception as e: logger.debug( - f"Creating activity failed with {e}, but will be retried with new agreement" + f"Creating activity failed with `{e}`, but will be retried with new agreement" ) finally: event = AgreementReleased(agreement) @@ -71,10 +70,10 @@ async def _prepare_activity(self) -> Activity: logger.debug(f"Releasing agreement by emitting `{event}` done") - logger.debug("Calling `_prepare_activity` done") + logger.debug("Preparing done") async def do_work(self, work: Work) -> WorkResult: - logger.debug("Calling `do_work`...") + logger.debug("Doing work...") async with self._prepare_activity() as activity: work_context = WorkContext(activity) @@ -114,6 +113,6 @@ async def do_work(self, work: Work) -> WorkResult: " `context.terminate()` in custom `on_activity_end` callback." ) - logger.debug("Calling `do_work` done") + logger.debug("Doing work done") return work_result diff --git a/golem_core/managers/work/sequential.py b/golem_core/managers/work/sequential.py index b465e176..da81a309 100644 --- a/golem_core/managers/work/sequential.py +++ b/golem_core/managers/work/sequential.py @@ -4,11 +4,15 @@ from golem_core.managers.base import DoWorkCallable, Work, WorkResult +logger = logging.getLogger(__name__) + class SequentialWorkManager: def __init__(self, do_work: DoWorkCallable): self._do_work = do_work - def apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: + def _apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: + logger.debug(f'Applying decorators on `{work}`...') + if not hasattr(work, "_work_decorators"): return do_work @@ -16,17 +20,33 @@ def apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkCa for dec in work._work_decorators: result = partial(dec, result) + logger.debug(f'Applying decorators on `{work}` done') + return result async def do_work(self, work: Work) -> WorkResult: - decorated_do_work = self.apply_work_decorators(self._do_work, work) + logger.debug("Calling `do_work`...") - return await decorated_do_work(work) + decorated_do_work = self._apply_work_decorators(self._do_work, work) + + result = await decorated_do_work(work) + + logger.debug(f"Calling `do_work` done with `{result}`") + + return result async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: + logger.debug("Doing work sequence...") + results = [] - for work in work_list: + for i, work in enumerate(work_list): + logger.debug(f'Doing work sequence #{i}...') + results.append(await self.do_work(work)) + logger.debug(f'Doing work sequence #{i} done') + + logger.debug(f"Doing work sequence done with `{results}`") + return results From e6864492644ba24eaabd633bf8170eb9271c1a59 Mon Sep 17 00:00:00 2001 From: approxit Date: Mon, 29 May 2023 18:04:34 +0200 Subject: [PATCH 031/123] initial new event bus --- golem_core/core/events/__init__.py | 2 +- golem_core/core/events/base.py | 106 +++++++++++++++++++++++++ golem_core/core/events/event.py | 8 -- golem_core/core/events/event_bus.py | 2 +- golem_core/core/resources/events.py | 2 +- golem_core/managers/work/sequential.py | 10 +-- 6 files changed, 114 insertions(+), 16 deletions(-) create mode 100644 golem_core/core/events/base.py delete mode 100644 golem_core/core/events/event.py diff --git a/golem_core/core/events/__init__.py b/golem_core/core/events/__init__.py index e4a3be41..42a81be0 100644 --- a/golem_core/core/events/__init__.py +++ b/golem_core/core/events/__init__.py @@ -1,4 +1,4 @@ -from golem_core.core.events.event import Event, TEvent +from golem_core.core.events.base import Event, TEvent from golem_core.core.events.event_bus import EventBus from golem_core.core.events.event_filters import AnyEventFilter, EventFilter diff --git a/golem_core/core/events/base.py b/golem_core/core/events/base.py new file mode 100644 index 00000000..a1678edb --- /dev/null +++ b/golem_core/core/events/base.py @@ -0,0 +1,106 @@ +from abc import ABC, abstractmethod +from collections import defaultdict +from dataclasses import dataclass +from typing import Awaitable, Callable, DefaultDict, List, Optional, Type, TypeVar + +TEvent = TypeVar("TEvent", bound="Event") + + +class Event(ABC): + """Base class for all events.""" + + +TCallbackHandler = TypeVar("TCallbackHandler") + + +class EventBus(ABC): + @abstractmethod + async def on( + self, + event_type: Type[TEvent], + callback: Callable[[TEvent], Awaitable[None]], + filter_func: Optional[Callable[[TEvent], bool]] = None, + ) -> TCallbackHandler: + ... + + @abstractmethod + async def on_once( + self, event_type: Type[TEvent], callback: Callable[[TEvent], Awaitable[None]] + ) -> TCallbackHandler: + ... + + @abstractmethod + async def off(self, callback_handler: TCallbackHandler) -> None: + ... + + @abstractmethod + async def emit(self, event: TEvent) -> None: + ... + + +@dataclass +class CallbackInfo: + callback: Callable[[TEvent], Awaitable[None]] + filter_func: Optional[Callable[[TEvent], bool]] + once: bool + + +class InMemoryEventBus(EventBus): + def __init__(self): + self._callbacks: DefaultDict[Type[TEvent], List[CallbackInfo]] = defaultdict(list) + + async def on( + self, + event_type: Type[TEvent], + callback: Callable[[TEvent], Awaitable[None]], + filter_func: Optional[Callable[[TEvent], bool]] = None, + ) -> TCallbackHandler: + callback_info = CallbackInfo( + callback=callback, + filter_func=filter_func, + once=False, + ) + + self._callbacks[event_type].append(callback_info) + + return (event_type, callback_info) + + async def on_once( + self, + event_type: Type[TEvent], + callback: Callable[[TEvent], Awaitable[None]], + filter_func: Optional[Callable[[TEvent], bool]] = None, + ) -> TCallbackHandler: + callback_info = CallbackInfo( + callback=callback, + filter_func=filter_func, + once=True, + ) + + self._callbacks[event_type].append(callback_info) + + return (event_type, callback_info) + + async def off(self, callback_handler: TCallbackHandler) -> None: + event_type, callback_info = callback_handler + + try: + self._callbacks[event_type].remove(callback_info) + except (KeyError, ValueError): + raise ValueError(f"Given callback handler is not found in event bus!") + + async def emit(self, event: TEvent) -> None: + for event_type, callback_infos in self._callbacks.items(): + if not isinstance(event, event_type): + continue + + callback_infos_copy = callback_infos[:] + + for callback_info in callback_infos_copy: + if callback_info.filter_func is not None and not callback_info.filter_func(event): + continue + + callback_info.callback(event) + + if callback_info.once: + callback_infos.remove(callback_info) diff --git a/golem_core/core/events/event.py b/golem_core/core/events/event.py deleted file mode 100644 index cf70c907..00000000 --- a/golem_core/core/events/event.py +++ /dev/null @@ -1,8 +0,0 @@ -from abc import ABC -from typing import TypeVar - -TEvent = TypeVar("TEvent", bound="Event") - - -class Event(ABC): - """Base class for all events.""" diff --git a/golem_core/core/events/event_bus.py b/golem_core/core/events/event_bus.py index b5070e6d..af21356a 100644 --- a/golem_core/core/events/event_bus.py +++ b/golem_core/core/events/event_bus.py @@ -12,7 +12,7 @@ Type, ) -from golem_core.core.events.event import Event, TEvent +from golem_core.core.events.base import Event, TEvent from golem_core.core.events.event_filters import AnyEventFilter, EventFilter if TYPE_CHECKING: diff --git a/golem_core/core/resources/events.py b/golem_core/core/resources/events.py index bb0368d2..12d6f81f 100644 --- a/golem_core/core/resources/events.py +++ b/golem_core/core/resources/events.py @@ -1,7 +1,7 @@ from abc import ABC from typing import TYPE_CHECKING, Any, Dict, Tuple, TypeVar -from golem_core.core.events.event import Event +from golem_core.core.events.base import Event if TYPE_CHECKING: from golem_core.core.resources.base import Resource diff --git a/golem_core/managers/work/sequential.py b/golem_core/managers/work/sequential.py index da81a309..3dc63dce 100644 --- a/golem_core/managers/work/sequential.py +++ b/golem_core/managers/work/sequential.py @@ -3,15 +3,15 @@ from golem_core.managers.base import DoWorkCallable, Work, WorkResult - logger = logging.getLogger(__name__) + class SequentialWorkManager: def __init__(self, do_work: DoWorkCallable): self._do_work = do_work def _apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: - logger.debug(f'Applying decorators on `{work}`...') + logger.debug(f"Applying decorators on `{work}`...") if not hasattr(work, "_work_decorators"): return do_work @@ -20,7 +20,7 @@ def _apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkC for dec in work._work_decorators: result = partial(dec, result) - logger.debug(f'Applying decorators on `{work}` done') + logger.debug(f"Applying decorators on `{work}` done") return result @@ -41,11 +41,11 @@ async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: results = [] for i, work in enumerate(work_list): - logger.debug(f'Doing work sequence #{i}...') + logger.debug(f"Doing work sequence #{i}...") results.append(await self.do_work(work)) - logger.debug(f'Doing work sequence #{i} done') + logger.debug(f"Doing work sequence #{i} done") logger.debug(f"Doing work sequence done with `{results}`") From 90b1da3a6306708ef8e47eca2f6ae5b78f165b49 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 30 May 2023 10:10:20 +0200 Subject: [PATCH 032/123] Pass golem node to all amnagers --- examples/managers/basic_composition.py | 45 +++++---------------- golem_core/managers/activity/single_use.py | 6 +-- golem_core/managers/agreement/single_use.py | 6 +-- golem_core/managers/proposal/stack.py | 3 +- golem_core/managers/work/sequential.py | 4 +- 5 files changed, 22 insertions(+), 42 deletions(-) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index b7e18c48..7332d505 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -1,5 +1,6 @@ import asyncio import logging.config +from functools import partial from golem_core.core.golem_node.golem_node import GolemNode from golem_core.core.market_api import RepositoryVmPayload @@ -13,32 +14,12 @@ from golem_core.utils.logging import DEFAULT_LOGGING -async def work1(context: WorkContext): - r = await context.run("echo 1") +async def work(context: WorkContext, label: str): + r = await context.run(f"echo {label}") await r.wait() result = "" for event in r.events: - print(f"Work1 got: {event.stdout}") - result += event.stdout - return result - - -async def work2(context: WorkContext): - r = await context.run("echo 2") - await r.wait() - result = "" - for event in r.events: - print(f"Work2 got: {event.stdout}") - result += event.stdout - return result - - -async def work3(context: WorkContext): - r = await context.run("echo 3") - await r.wait() - result = "" - for event in r.events: - print(f"Work3 got: {event.stdout}") + print(f"Work {label} got: {event.stdout}") result += event.stdout return result @@ -48,22 +29,18 @@ async def main(): payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") work_list = [ - work1, - work2, - work3, + partial(work, label="label-1"), + partial(work, label="label-2"), + partial(work, label="label-3"), ] async with GolemNode() as golem: payment_manager = PayAllPaymentManager(golem, budget=1.0) negotiation_manager = AcceptAllNegotiationManager(golem, payment_manager.get_allocation) - proposal_manager = StackProposalManager(negotiation_manager.get_proposal) - agreement_manager = SingleUseAgreementManager( - proposal_manager.get_proposal, golem.event_bus - ) - activity_manager = SingleUseActivityManager( - agreement_manager.get_agreement, golem.event_bus - ) - work_manager = SequentialWorkManager(activity_manager.do_work) + proposal_manager = StackProposalManager(golem, negotiation_manager.get_proposal) + agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) + activity_manager = SingleUseActivityManager(golem, agreement_manager.get_agreement) + work_manager = SequentialWorkManager(golem, activity_manager.do_work) await negotiation_manager.start_negotiation(payload) await proposal_manager.start_consuming_proposals() diff --git a/golem_core/managers/activity/single_use.py b/golem_core/managers/activity/single_use.py index cc1e10ff..e6512668 100644 --- a/golem_core/managers/activity/single_use.py +++ b/golem_core/managers/activity/single_use.py @@ -3,7 +3,7 @@ from typing import Awaitable, Callable, Optional from golem_core.core.activity_api import Activity -from golem_core.core.events import EventBus +from golem_core.core.golem_node.golem_node import GolemNode from golem_core.core.market_api import Agreement from golem_core.managers.activity.defaults import ( default_on_activity_start, @@ -18,8 +18,8 @@ class SingleUseActivityManager(ActivityManager): def __init__( self, + golem: GolemNode, get_agreement: Callable[[], Awaitable[Agreement]], - event_bus: EventBus, on_activity_start: Optional[ Callable[[WorkContext], Awaitable[None]] ] = default_on_activity_start, @@ -28,7 +28,7 @@ def __init__( ] = default_on_activity_stop, ): self._get_agreement = get_agreement - self._event_bus = event_bus + self._event_bus = golem.event_bus self._on_activity_start = on_activity_start self._on_activity_stop = on_activity_stop diff --git a/golem_core/managers/agreement/single_use.py b/golem_core/managers/agreement/single_use.py index 8877fad1..c67f9675 100644 --- a/golem_core/managers/agreement/single_use.py +++ b/golem_core/managers/agreement/single_use.py @@ -1,7 +1,7 @@ import logging from typing import Awaitable, Callable -from golem_core.core.events import EventBus +from golem_core.core.golem_node.golem_node import GolemNode from golem_core.core.market_api import Agreement, Proposal from golem_core.managers.agreement.events import AgreementReleased from golem_core.managers.base import AgreementManager @@ -10,9 +10,9 @@ class SingleUseAgreementManager(AgreementManager): - def __init__(self, get_proposal: Callable[[], Awaitable[Proposal]], event_bus: EventBus): + def __init__(self, golem: GolemNode, get_proposal: Callable[[], Awaitable[Proposal]]): self._get_proposal = get_proposal - self._event_bus = event_bus + self._event_bus = golem.event_bus async def get_agreement(self) -> Agreement: logger.debug("Getting agreement...") diff --git a/golem_core/managers/proposal/stack.py b/golem_core/managers/proposal/stack.py index f9292dfa..03390568 100644 --- a/golem_core/managers/proposal/stack.py +++ b/golem_core/managers/proposal/stack.py @@ -2,6 +2,7 @@ import logging from typing import List +from golem_core.core.golem_node.golem_node import GolemNode from golem_core.core.market_api import Proposal from golem_core.managers.base import ProposalAggregationManager @@ -9,7 +10,7 @@ class StackProposalManager(ProposalAggregationManager): - def __init__(self, get_proposal) -> None: + def __init__(self, golem: GolemNode, get_proposal) -> None: self._get_proposal = get_proposal self._proposals: asyncio.Queue[Proposal] = asyncio.Queue() self._tasks: List[asyncio.Task] = [] diff --git a/golem_core/managers/work/sequential.py b/golem_core/managers/work/sequential.py index 3dc63dce..3ad91f8d 100644 --- a/golem_core/managers/work/sequential.py +++ b/golem_core/managers/work/sequential.py @@ -1,13 +1,15 @@ +import logging from functools import partial from typing import List +from golem_core.core.golem_node.golem_node import GolemNode from golem_core.managers.base import DoWorkCallable, Work, WorkResult logger = logging.getLogger(__name__) class SequentialWorkManager: - def __init__(self, do_work: DoWorkCallable): + def __init__(self, golem: GolemNode, do_work: DoWorkCallable): self._do_work = do_work def _apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: From af2948c9c3e03e0da558af58d1f32d139d04e086 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 30 May 2023 10:53:29 +0200 Subject: [PATCH 033/123] Affed info logging --- examples/managers/basic_composition.py | 13 +++++++----- golem_core/managers/activity/defaults.py | 2 -- golem_core/managers/activity/single_use.py | 11 ++++------ golem_core/managers/agreement/single_use.py | 4 ++-- golem_core/managers/negotiation/accept_all.py | 20 ++++++++++++------- golem_core/managers/payment/pay_all.py | 16 +++++++++++---- golem_core/managers/proposal/stack.py | 9 ++++++--- golem_core/managers/work/sequential.py | 8 ++++---- golem_core/utils/logging.py | 2 +- 9 files changed, 50 insertions(+), 35 deletions(-) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index 7332d505..933afcb8 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -1,12 +1,13 @@ import asyncio import logging.config from functools import partial +from typing import List from golem_core.core.golem_node.golem_node import GolemNode from golem_core.core.market_api import RepositoryVmPayload from golem_core.managers.activity.single_use import SingleUseActivityManager from golem_core.managers.agreement.single_use import SingleUseAgreementManager -from golem_core.managers.base import WorkContext +from golem_core.managers.base import WorkContext, WorkResult from golem_core.managers.negotiation import AcceptAllNegotiationManager from golem_core.managers.payment.pay_all import PayAllPaymentManager from golem_core.managers.proposal import StackProposalManager @@ -14,12 +15,11 @@ from golem_core.utils.logging import DEFAULT_LOGGING -async def work(context: WorkContext, label: str): +async def work(context: WorkContext, label: str) -> str: r = await context.run(f"echo {label}") await r.wait() result = "" for event in r.events: - print(f"Work {label} got: {event.stdout}") result += event.stdout return result @@ -45,8 +45,11 @@ async def main(): await negotiation_manager.start_negotiation(payload) await proposal_manager.start_consuming_proposals() - results = await work_manager.do_work_list(work_list) - print(f"work done: {results}") + results: List[WorkResult] = await work_manager.do_work_list(work_list) + for result in results: + print( + f"\nWORK MANAGER RETURNED:\nResult:\t{result.result}\nException:\t{result.exception}\nExtras:\t{result.extras}" + ) await proposal_manager.stop_consuming_proposals() await negotiation_manager.stop_negotiation() diff --git a/golem_core/managers/activity/defaults.py b/golem_core/managers/activity/defaults.py index 228b6687..949543ad 100644 --- a/golem_core/managers/activity/defaults.py +++ b/golem_core/managers/activity/defaults.py @@ -6,13 +6,11 @@ async def default_on_activity_start(context: WorkContext): batch.deploy() batch.start() await batch() - print("activity start") # TODO: After this function call we should have check if activity is actually started async def default_on_activity_stop(context: WorkContext): await context.terminate() - print("activity stop") # TODO: After this function call we should have check if activity is actually terminated diff --git a/golem_core/managers/activity/single_use.py b/golem_core/managers/activity/single_use.py index e6512668..0b849ec6 100644 --- a/golem_core/managers/activity/single_use.py +++ b/golem_core/managers/activity/single_use.py @@ -34,9 +34,8 @@ def __init__( @asynccontextmanager async def _prepare_activity(self) -> Activity: - logger.debug("Preparing activity...") - while True: + logger.info("Preparing activity") logger.debug(f"Getting agreement...") agreement = await self._get_agreement() @@ -50,8 +49,8 @@ async def _prepare_activity(self) -> Activity: logger.debug(f"Creating activity done with `{activity}`") + logger.info("Done preparing activity") logger.debug(f"Yielding activity...") - yield activity logger.debug(f"Yielding activity done") @@ -70,10 +69,8 @@ async def _prepare_activity(self) -> Activity: logger.debug(f"Releasing agreement by emitting `{event}` done") - logger.debug("Preparing done") - async def do_work(self, work: Work) -> WorkResult: - logger.debug("Doing work...") + logger.info("Doing work on activity") async with self._prepare_activity() as activity: work_context = WorkContext(activity) @@ -113,6 +110,6 @@ async def do_work(self, work: Work) -> WorkResult: " `context.terminate()` in custom `on_activity_end` callback." ) - logger.debug("Doing work done") + logger.info("Done doing work on activity") return work_result diff --git a/golem_core/managers/agreement/single_use.py b/golem_core/managers/agreement/single_use.py index c67f9675..589235d5 100644 --- a/golem_core/managers/agreement/single_use.py +++ b/golem_core/managers/agreement/single_use.py @@ -15,7 +15,7 @@ def __init__(self, golem: GolemNode, get_proposal: Callable[[], Awaitable[Propos self._event_bus = golem.event_bus async def get_agreement(self) -> Agreement: - logger.debug("Getting agreement...") + logger.info("Getting agreement") while True: logger.debug("Getting proposal...") @@ -46,7 +46,7 @@ async def get_agreement(self) -> Agreement: self._on_agreement_released, [AgreementReleased], [Agreement], [agreement.id] ) - logger.debug(f"Getting agreement done with {agreement}") + logger.info(f"Done getting agreement") return agreement async def _on_agreement_released(self, event: AgreementReleased) -> None: diff --git a/golem_core/managers/negotiation/accept_all.py b/golem_core/managers/negotiation/accept_all.py index 98b89eb2..fbbf2764 100644 --- a/golem_core/managers/negotiation/accept_all.py +++ b/golem_core/managers/negotiation/accept_all.py @@ -22,17 +22,21 @@ def __init__( self._eligible_proposals: asyncio.Queue[Proposal] = asyncio.Queue() async def get_proposal(self) -> Proposal: - logger.debug("Returning proposal") - return await self._eligible_proposals.get() + logger.info("Getting proposal") + proposal = await self._eligible_proposals.get() + logger.info("Done getting proposal") + return proposal async def start_negotiation(self, payload: Payload) -> None: - logger.debug("Starting negotiations") + logger.info("Starting negotiations") self._negotiations.append(asyncio.create_task(self._negotiate_task(payload))) + logger.info("Done starting negotiations") async def stop_negotiation(self) -> None: + logger.info("Stopping negotiations") for task in self._negotiations: - logger.debug("Stopping negotiations") task.cancel() + logger.info("Done stopping negotiations") async def _negotiate_task(self, payload: Payload) -> None: allocation = await self._get_allocation() @@ -41,7 +45,7 @@ async def _negotiate_task(self, payload: Payload) -> None: await self._eligible_proposals.put(proposal) async def _build_demand(self, allocation: Allocation, payload: Payload) -> Demand: - logger.debug("Creating demand") + logger.info("Creating demand") demand_builder = DemandBuilder() await demand_builder.add( @@ -63,17 +67,18 @@ async def _build_demand(self, allocation: Allocation, payload: Payload) -> Deman demand = await demand_builder.create_demand(self._golem) demand.start_collecting_events() + logger.info("Done creating demand") return demand async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: try: async for initial in demand.initial_proposals(): - logger.debug("Got initial proposal") + logger.info("Negotiating initial proposal") try: demand_proposal = await initial.respond() except Exception as err: logger.debug( - f"Unable to respond to initialproposal {initial.id}. Got {type(err)}\n{err}" + f"Unable to respond to initial proposal {initial.id}.Got {type(err)}\n{err}" ) continue @@ -83,6 +88,7 @@ async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: except StopAsyncIteration: continue + logger.info("Done negotiating initial proposal") yield offer_proposal finally: self._golem.add_autoclose_resource(demand) diff --git a/golem_core/managers/payment/pay_all.py b/golem_core/managers/payment/pay_all.py index 82dd63d1..4c703c0d 100644 --- a/golem_core/managers/payment/pay_all.py +++ b/golem_core/managers/payment/pay_all.py @@ -34,32 +34,40 @@ def __init__( ) async def get_allocation(self) -> "Allocation": + logger.info("Getting allocation") if self._allocation is None: - logger.debug("Creating allocation") + logger.info("Creating allocation") self._allocation = await Allocation.create_any_account( self._golem, Decimal(self._budget), self._network, self._driver ) self._golem.add_autoclose_resource(self._allocation) - logger.debug("Returning allocation") + logger.info("Done creating allocation") + logger.info("Done getting allocation") return self._allocation async def on_invoice_received(self, invoice_event: NewResource) -> None: - logger.debug("Got invoice") + logger.info("Received invoice") invoice = invoice_event.resource assert isinstance(invoice, Invoice) if (await invoice.get_data(force=True)).status == "RECEIVED": + logger.info("Accepting invoice") assert self._allocation is not None # TODO think of a better way await invoice.accept_full(self._allocation) await invoice.get_data(force=True) + logger.info("Done accepting invoice") async def on_debit_note_received(self, debit_note_event: NewResource) -> None: - logger.debug("Got debit note") + logger.info("Received debit note") debit_note = debit_note_event.resource assert isinstance(debit_note, DebitNote) if (await debit_note.get_data(force=True)).status == "RECEIVED": + logger.info("Accepting debit note") assert self._allocation is not None # TODO think of a better way await debit_note.accept_full(self._allocation) await debit_note.get_data(force=True) + logger.info("Done accepting debit note") async def wait_for_invoices(self): + logger.info("Waiting for invoices") await asyncio.sleep(30) + logger.info("Done waiting for invoices") diff --git a/golem_core/managers/proposal/stack.py b/golem_core/managers/proposal/stack.py index 03390568..8bbc1728 100644 --- a/golem_core/managers/proposal/stack.py +++ b/golem_core/managers/proposal/stack.py @@ -16,13 +16,15 @@ def __init__(self, golem: GolemNode, get_proposal) -> None: self._tasks: List[asyncio.Task] = [] async def start_consuming_proposals(self) -> None: - logger.debug("Starting manager") + logger.info("Starting consuming proposals") self._tasks.append(asyncio.create_task(self._consume_proposals())) + logger.info("Done starting consuming proposals") async def stop_consuming_proposals(self) -> None: + logger.info("Stopping consuming proposals") for task in self._tasks: - logger.debug("Stopping manager") task.cancel() + logger.info("Done stopping consuming proposals") async def _consume_proposals(self) -> None: while True: @@ -31,6 +33,7 @@ async def _consume_proposals(self) -> None: await self._proposals.put(proposal) async def get_proposal(self) -> Proposal: + logger.info("Getting proposals") proposal = await self._proposals.get() - logger.debug("Returning proposal from the stack") + logger.info("Done getting proposals") return proposal diff --git a/golem_core/managers/work/sequential.py b/golem_core/managers/work/sequential.py index 3ad91f8d..eac08209 100644 --- a/golem_core/managers/work/sequential.py +++ b/golem_core/managers/work/sequential.py @@ -27,18 +27,18 @@ def _apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkC return result async def do_work(self, work: Work) -> WorkResult: - logger.debug("Calling `do_work`...") + logger.info("Running work") decorated_do_work = self._apply_work_decorators(self._do_work, work) result = await decorated_do_work(work) - logger.debug(f"Calling `do_work` done with `{result}`") + logger.info("Done running work") return result async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: - logger.debug("Doing work sequence...") + logger.info("Running work sequence") results = [] @@ -49,6 +49,6 @@ async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: logger.debug(f"Doing work sequence #{i} done") - logger.debug(f"Doing work sequence done with `{results}`") + logger.info(f"Done running work sequence") return results diff --git a/golem_core/utils/logging.py b/golem_core/utils/logging.py index d9690ef4..87dd7cac 100644 --- a/golem_core/utils/logging.py +++ b/golem_core/utils/logging.py @@ -23,7 +23,7 @@ ], }, "golem_core": { - "level": "DEBUG", + "level": "INFO", }, }, } From eb4c3a6f6f718becc2eb51fcb845c3bd7e09d258 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 30 May 2023 11:54:48 +0200 Subject: [PATCH 034/123] Move Done to end of log lines --- examples/managers/basic_composition.py | 8 +++---- golem_core/managers/activity/single_use.py | 8 +++---- golem_core/managers/agreement/single_use.py | 4 ++-- golem_core/managers/negotiation/accept_all.py | 20 ++++++++-------- golem_core/managers/payment/pay_all.py | 24 +++++++++---------- golem_core/managers/proposal/stack.py | 12 +++++----- golem_core/managers/work/sequential.py | 8 +++---- 7 files changed, 42 insertions(+), 42 deletions(-) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index 933afcb8..db8aa8ab 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -24,6 +24,9 @@ async def work(context: WorkContext, label: str) -> str: return result +# TODO add Batch example + + async def main(): logging.config.dictConfig(DEFAULT_LOGGING) payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") @@ -46,10 +49,7 @@ async def main(): await proposal_manager.start_consuming_proposals() results: List[WorkResult] = await work_manager.do_work_list(work_list) - for result in results: - print( - f"\nWORK MANAGER RETURNED:\nResult:\t{result.result}\nException:\t{result.exception}\nExtras:\t{result.extras}" - ) + print(f"\nWORK MANAGER RESULTS:{[result.result for result in results]}\n") await proposal_manager.stop_consuming_proposals() await negotiation_manager.stop_negotiation() diff --git a/golem_core/managers/activity/single_use.py b/golem_core/managers/activity/single_use.py index 0b849ec6..f7453d71 100644 --- a/golem_core/managers/activity/single_use.py +++ b/golem_core/managers/activity/single_use.py @@ -35,7 +35,7 @@ def __init__( @asynccontextmanager async def _prepare_activity(self) -> Activity: while True: - logger.info("Preparing activity") + logger.info("Preparing activity...") logger.debug(f"Getting agreement...") agreement = await self._get_agreement() @@ -49,7 +49,7 @@ async def _prepare_activity(self) -> Activity: logger.debug(f"Creating activity done with `{activity}`") - logger.info("Done preparing activity") + logger.info("Preparing activity done") logger.debug(f"Yielding activity...") yield activity @@ -70,7 +70,7 @@ async def _prepare_activity(self) -> Activity: logger.debug(f"Releasing agreement by emitting `{event}` done") async def do_work(self, work: Work) -> WorkResult: - logger.info("Doing work on activity") + logger.info("Doing work on activity...") async with self._prepare_activity() as activity: work_context = WorkContext(activity) @@ -110,6 +110,6 @@ async def do_work(self, work: Work) -> WorkResult: " `context.terminate()` in custom `on_activity_end` callback." ) - logger.info("Done doing work on activity") + logger.info("Doing work on activity done") return work_result diff --git a/golem_core/managers/agreement/single_use.py b/golem_core/managers/agreement/single_use.py index 589235d5..fd9c678d 100644 --- a/golem_core/managers/agreement/single_use.py +++ b/golem_core/managers/agreement/single_use.py @@ -15,7 +15,7 @@ def __init__(self, golem: GolemNode, get_proposal: Callable[[], Awaitable[Propos self._event_bus = golem.event_bus async def get_agreement(self) -> Agreement: - logger.info("Getting agreement") + logger.info("Getting agreement...") while True: logger.debug("Getting proposal...") @@ -46,7 +46,7 @@ async def get_agreement(self) -> Agreement: self._on_agreement_released, [AgreementReleased], [Agreement], [agreement.id] ) - logger.info(f"Done getting agreement") + logger.info(f"Getting agreement done") return agreement async def _on_agreement_released(self, event: AgreementReleased) -> None: diff --git a/golem_core/managers/negotiation/accept_all.py b/golem_core/managers/negotiation/accept_all.py index fbbf2764..33c1163d 100644 --- a/golem_core/managers/negotiation/accept_all.py +++ b/golem_core/managers/negotiation/accept_all.py @@ -22,21 +22,21 @@ def __init__( self._eligible_proposals: asyncio.Queue[Proposal] = asyncio.Queue() async def get_proposal(self) -> Proposal: - logger.info("Getting proposal") + logger.info("Getting proposal...") proposal = await self._eligible_proposals.get() - logger.info("Done getting proposal") + logger.info("Getting proposal done") return proposal async def start_negotiation(self, payload: Payload) -> None: - logger.info("Starting negotiations") + logger.debug("Starting negotiations...") self._negotiations.append(asyncio.create_task(self._negotiate_task(payload))) - logger.info("Done starting negotiations") + logger.debug("Starting negotiations done") async def stop_negotiation(self) -> None: - logger.info("Stopping negotiations") + logger.debug("Stopping negotiations...") for task in self._negotiations: task.cancel() - logger.info("Done stopping negotiations") + logger.debug("Stopping negotiations done") async def _negotiate_task(self, payload: Payload) -> None: allocation = await self._get_allocation() @@ -45,7 +45,7 @@ async def _negotiate_task(self, payload: Payload) -> None: await self._eligible_proposals.put(proposal) async def _build_demand(self, allocation: Allocation, payload: Payload) -> Demand: - logger.info("Creating demand") + logger.info("Creating demand...") demand_builder = DemandBuilder() await demand_builder.add( @@ -67,13 +67,13 @@ async def _build_demand(self, allocation: Allocation, payload: Payload) -> Deman demand = await demand_builder.create_demand(self._golem) demand.start_collecting_events() - logger.info("Done creating demand") + logger.info("Creating demand done") return demand async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: try: async for initial in demand.initial_proposals(): - logger.info("Negotiating initial proposal") + logger.info("Negotiating initial proposal...") try: demand_proposal = await initial.respond() except Exception as err: @@ -88,7 +88,7 @@ async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: except StopAsyncIteration: continue - logger.info("Done negotiating initial proposal") + logger.info("Negotiating initial proposal done") yield offer_proposal finally: self._golem.add_autoclose_resource(demand) diff --git a/golem_core/managers/payment/pay_all.py b/golem_core/managers/payment/pay_all.py index 4c703c0d..e3fa36d0 100644 --- a/golem_core/managers/payment/pay_all.py +++ b/golem_core/managers/payment/pay_all.py @@ -34,40 +34,40 @@ def __init__( ) async def get_allocation(self) -> "Allocation": - logger.info("Getting allocation") + logger.info("Getting allocation...") if self._allocation is None: - logger.info("Creating allocation") + logger.info("Creating allocation...") self._allocation = await Allocation.create_any_account( self._golem, Decimal(self._budget), self._network, self._driver ) self._golem.add_autoclose_resource(self._allocation) - logger.info("Done creating allocation") - logger.info("Done getting allocation") + logger.info("Creating allocation done") + logger.info("Getting allocation done") return self._allocation async def on_invoice_received(self, invoice_event: NewResource) -> None: - logger.info("Received invoice") + logger.info("Received invoice...") invoice = invoice_event.resource assert isinstance(invoice, Invoice) if (await invoice.get_data(force=True)).status == "RECEIVED": - logger.info("Accepting invoice") + logger.info("Accepting invoice...") assert self._allocation is not None # TODO think of a better way await invoice.accept_full(self._allocation) await invoice.get_data(force=True) - logger.info("Done accepting invoice") + logger.info("Accepting invoice done") async def on_debit_note_received(self, debit_note_event: NewResource) -> None: - logger.info("Received debit note") + logger.info("Received debit note...") debit_note = debit_note_event.resource assert isinstance(debit_note, DebitNote) if (await debit_note.get_data(force=True)).status == "RECEIVED": - logger.info("Accepting debit note") + logger.info("Accepting debit note...") assert self._allocation is not None # TODO think of a better way await debit_note.accept_full(self._allocation) await debit_note.get_data(force=True) - logger.info("Done accepting debit note") + logger.info("Accepting debit note done") async def wait_for_invoices(self): - logger.info("Waiting for invoices") + logger.info("Waiting for invoices...") await asyncio.sleep(30) - logger.info("Done waiting for invoices") + logger.info("Waiting for invoices done") diff --git a/golem_core/managers/proposal/stack.py b/golem_core/managers/proposal/stack.py index 8bbc1728..f89b8651 100644 --- a/golem_core/managers/proposal/stack.py +++ b/golem_core/managers/proposal/stack.py @@ -16,15 +16,15 @@ def __init__(self, golem: GolemNode, get_proposal) -> None: self._tasks: List[asyncio.Task] = [] async def start_consuming_proposals(self) -> None: - logger.info("Starting consuming proposals") + logger.debug("Starting consuming proposals...") self._tasks.append(asyncio.create_task(self._consume_proposals())) - logger.info("Done starting consuming proposals") + logger.debug("Starting consuming proposals done") async def stop_consuming_proposals(self) -> None: - logger.info("Stopping consuming proposals") + logger.debug("Stopping consuming proposals...") for task in self._tasks: task.cancel() - logger.info("Done stopping consuming proposals") + logger.debug("Stopping consuming proposals done") async def _consume_proposals(self) -> None: while True: @@ -33,7 +33,7 @@ async def _consume_proposals(self) -> None: await self._proposals.put(proposal) async def get_proposal(self) -> Proposal: - logger.info("Getting proposals") + logger.info("Getting proposals...") proposal = await self._proposals.get() - logger.info("Done getting proposals") + logger.info("Getting proposals done") return proposal diff --git a/golem_core/managers/work/sequential.py b/golem_core/managers/work/sequential.py index eac08209..f9189b16 100644 --- a/golem_core/managers/work/sequential.py +++ b/golem_core/managers/work/sequential.py @@ -27,18 +27,18 @@ def _apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkC return result async def do_work(self, work: Work) -> WorkResult: - logger.info("Running work") + logger.info("Running work...") decorated_do_work = self._apply_work_decorators(self._do_work, work) result = await decorated_do_work(work) - logger.info("Done running work") + logger.info("Running work done") return result async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: - logger.info("Running work sequence") + logger.info("Running work sequence...") results = [] @@ -49,6 +49,6 @@ async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: logger.debug(f"Doing work sequence #{i} done") - logger.info(f"Done running work sequence") + logger.info(f"Running work sequence done") return results From 51331b40ca4c02dc28e01f4e781f78c468aa6dda Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 31 May 2023 10:04:21 +0200 Subject: [PATCH 035/123] Added ids to logs --- golem_core/managers/activity/single_use.py | 6 +++--- golem_core/managers/agreement/single_use.py | 6 +++--- golem_core/managers/negotiation/accept_all.py | 8 ++++---- golem_core/managers/payment/pay_all.py | 12 ++++++------ golem_core/managers/proposal/stack.py | 2 +- golem_core/managers/work/sequential.py | 8 ++++---- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/golem_core/managers/activity/single_use.py b/golem_core/managers/activity/single_use.py index f7453d71..6dabe29b 100644 --- a/golem_core/managers/activity/single_use.py +++ b/golem_core/managers/activity/single_use.py @@ -49,7 +49,7 @@ async def _prepare_activity(self) -> Activity: logger.debug(f"Creating activity done with `{activity}`") - logger.info("Preparing activity done") + logger.info(f"Preparing activity done {activity.id}") logger.debug(f"Yielding activity...") yield activity @@ -70,7 +70,7 @@ async def _prepare_activity(self) -> Activity: logger.debug(f"Releasing agreement by emitting `{event}` done") async def do_work(self, work: Work) -> WorkResult: - logger.info("Doing work on activity...") + logger.info(f"Doing work {work} on activity") async with self._prepare_activity() as activity: work_context = WorkContext(activity) @@ -110,6 +110,6 @@ async def do_work(self, work: Work) -> WorkResult: " `context.terminate()` in custom `on_activity_end` callback." ) - logger.info("Doing work on activity done") + logger.info(f"Doing work done {work} on activity {activity.id}") return work_result diff --git a/golem_core/managers/agreement/single_use.py b/golem_core/managers/agreement/single_use.py index fd9c678d..08676ec6 100644 --- a/golem_core/managers/agreement/single_use.py +++ b/golem_core/managers/agreement/single_use.py @@ -25,7 +25,7 @@ async def get_agreement(self) -> Agreement: logger.debug(f"Getting proposal done with {proposal}") try: - logger.debug(f"Creating agreement...") + logger.info(f"Creating agreement...") agreement = await proposal.create_agreement() @@ -39,14 +39,14 @@ async def get_agreement(self) -> Agreement: except Exception as e: logger.debug(f"Creating agreement failed with {e}. Retrying...") else: - logger.debug(f"Creating agreement done") + logger.info(f"Creating agreement done {agreement.id}") # TODO: Support removing callback on resource close self._event_bus.resource_listen( self._on_agreement_released, [AgreementReleased], [Agreement], [agreement.id] ) - logger.info(f"Getting agreement done") + logger.info(f"Getting agreement done {agreement.id}") return agreement async def _on_agreement_released(self, event: AgreementReleased) -> None: diff --git a/golem_core/managers/negotiation/accept_all.py b/golem_core/managers/negotiation/accept_all.py index 33c1163d..a29822c2 100644 --- a/golem_core/managers/negotiation/accept_all.py +++ b/golem_core/managers/negotiation/accept_all.py @@ -24,7 +24,7 @@ def __init__( async def get_proposal(self) -> Proposal: logger.info("Getting proposal...") proposal = await self._eligible_proposals.get() - logger.info("Getting proposal done") + logger.info(f"Getting proposal done {proposal.id}") return proposal async def start_negotiation(self, payload: Payload) -> None: @@ -67,13 +67,13 @@ async def _build_demand(self, allocation: Allocation, payload: Payload) -> Deman demand = await demand_builder.create_demand(self._golem) demand.start_collecting_events() - logger.info("Creating demand done") + logger.info(f"Creating demand done {demand.id}") return demand async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: try: async for initial in demand.initial_proposals(): - logger.info("Negotiating initial proposal...") + logger.info(f"Negotiating initial proposal {initial.id}") try: demand_proposal = await initial.respond() except Exception as err: @@ -88,7 +88,7 @@ async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: except StopAsyncIteration: continue - logger.info("Negotiating initial proposal done") + logger.info(f"Negotiating initial proposal done {initial.id}") yield offer_proposal finally: self._golem.add_autoclose_resource(demand) diff --git a/golem_core/managers/payment/pay_all.py b/golem_core/managers/payment/pay_all.py index e3fa36d0..6e6d5d1b 100644 --- a/golem_core/managers/payment/pay_all.py +++ b/golem_core/managers/payment/pay_all.py @@ -41,8 +41,8 @@ async def get_allocation(self) -> "Allocation": self._golem, Decimal(self._budget), self._network, self._driver ) self._golem.add_autoclose_resource(self._allocation) - logger.info("Creating allocation done") - logger.info("Getting allocation done") + logger.info(f"Creating allocation done {self._allocation.id}") + logger.info(f"Getting allocation done {self._allocation.id}") return self._allocation async def on_invoice_received(self, invoice_event: NewResource) -> None: @@ -50,22 +50,22 @@ async def on_invoice_received(self, invoice_event: NewResource) -> None: invoice = invoice_event.resource assert isinstance(invoice, Invoice) if (await invoice.get_data(force=True)).status == "RECEIVED": - logger.info("Accepting invoice...") + logger.info(f"Accepting invoice {invoice.id}") assert self._allocation is not None # TODO think of a better way await invoice.accept_full(self._allocation) await invoice.get_data(force=True) - logger.info("Accepting invoice done") + logger.info(f"Accepting invoice done {invoice.id}") async def on_debit_note_received(self, debit_note_event: NewResource) -> None: logger.info("Received debit note...") debit_note = debit_note_event.resource assert isinstance(debit_note, DebitNote) if (await debit_note.get_data(force=True)).status == "RECEIVED": - logger.info("Accepting debit note...") + logger.info(f"Accepting debit note {debit_note.id}") assert self._allocation is not None # TODO think of a better way await debit_note.accept_full(self._allocation) await debit_note.get_data(force=True) - logger.info("Accepting debit note done") + logger.info(f"Accepting debit note done {debit_note.id}") async def wait_for_invoices(self): logger.info("Waiting for invoices...") diff --git a/golem_core/managers/proposal/stack.py b/golem_core/managers/proposal/stack.py index f89b8651..dc780e1f 100644 --- a/golem_core/managers/proposal/stack.py +++ b/golem_core/managers/proposal/stack.py @@ -35,5 +35,5 @@ async def _consume_proposals(self) -> None: async def get_proposal(self) -> Proposal: logger.info("Getting proposals...") proposal = await self._proposals.get() - logger.info("Getting proposals done") + logger.info(f"Getting proposals done {proposal.id}") return proposal diff --git a/golem_core/managers/work/sequential.py b/golem_core/managers/work/sequential.py index f9189b16..9aada90b 100644 --- a/golem_core/managers/work/sequential.py +++ b/golem_core/managers/work/sequential.py @@ -27,18 +27,18 @@ def _apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkC return result async def do_work(self, work: Work) -> WorkResult: - logger.info("Running work...") + logger.info(f"Running work {work}") decorated_do_work = self._apply_work_decorators(self._do_work, work) result = await decorated_do_work(work) - logger.info("Running work done") + logger.info(f"Running work done {work}") return result async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: - logger.info("Running work sequence...") + logger.info(f"Running work sequence {work_list}") results = [] @@ -49,6 +49,6 @@ async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: logger.debug(f"Doing work sequence #{i} done") - logger.info(f"Running work sequence done") + logger.info(f"Running work sequence done {work_list}") return results From d6c41fe4c12178c427e0df21ff67c7450479527a Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 31 May 2023 10:47:01 +0200 Subject: [PATCH 036/123] Add better pyament wait for invoices method --- golem_core/core/events/base.py | 2 +- golem_core/managers/activity/single_use.py | 8 ++-- golem_core/managers/agreement/single_use.py | 6 +-- golem_core/managers/payment/pay_all.py | 43 ++++++++++++++++----- 4 files changed, 41 insertions(+), 18 deletions(-) diff --git a/golem_core/core/events/base.py b/golem_core/core/events/base.py index a1678edb..ff45f505 100644 --- a/golem_core/core/events/base.py +++ b/golem_core/core/events/base.py @@ -87,7 +87,7 @@ async def off(self, callback_handler: TCallbackHandler) -> None: try: self._callbacks[event_type].remove(callback_info) except (KeyError, ValueError): - raise ValueError(f"Given callback handler is not found in event bus!") + raise ValueError("Given callback handler is not found in event bus!") async def emit(self, event: TEvent) -> None: for event_type, callback_infos in self._callbacks.items(): diff --git a/golem_core/managers/activity/single_use.py b/golem_core/managers/activity/single_use.py index 6dabe29b..93a8ac4e 100644 --- a/golem_core/managers/activity/single_use.py +++ b/golem_core/managers/activity/single_use.py @@ -36,24 +36,24 @@ def __init__( async def _prepare_activity(self) -> Activity: while True: logger.info("Preparing activity...") - logger.debug(f"Getting agreement...") + logger.debug("Getting agreement...") agreement = await self._get_agreement() logger.debug(f"Getting agreement done with `{agreement}`") try: - logger.debug(f"Creating activity...") + logger.debug("Creating activity...") activity = await agreement.create_activity() logger.debug(f"Creating activity done with `{activity}`") logger.info(f"Preparing activity done {activity.id}") - logger.debug(f"Yielding activity...") + logger.debug("Yielding activity...") yield activity - logger.debug(f"Yielding activity done") + logger.debug("Yielding activity done") break except Exception as e: diff --git a/golem_core/managers/agreement/single_use.py b/golem_core/managers/agreement/single_use.py index 08676ec6..b9070c32 100644 --- a/golem_core/managers/agreement/single_use.py +++ b/golem_core/managers/agreement/single_use.py @@ -25,15 +25,15 @@ async def get_agreement(self) -> Agreement: logger.debug(f"Getting proposal done with {proposal}") try: - logger.info(f"Creating agreement...") + logger.info("Creating agreement...") agreement = await proposal.create_agreement() - logger.debug(f"Sending agreement to provider...") + logger.debug("Sending agreement to provider...") await agreement.confirm() - logger.debug(f"Waiting for provider approval...") + logger.debug("Waiting for provider approval...") await agreement.wait_for_approval() except Exception as e: diff --git a/golem_core/managers/payment/pay_all.py b/golem_core/managers/payment/pay_all.py index 6e6d5d1b..52a267ac 100644 --- a/golem_core/managers/payment/pay_all.py +++ b/golem_core/managers/payment/pay_all.py @@ -4,10 +4,11 @@ from typing import Optional from golem_core.core.golem_node.golem_node import PAYMENT_DRIVER, PAYMENT_NETWORK, GolemNode +from golem_core.core.market_api.resources.agreement import Agreement from golem_core.core.payment_api.resources.allocation import Allocation from golem_core.core.payment_api.resources.debit_note import DebitNote from golem_core.core.payment_api.resources.invoice import Invoice -from golem_core.core.resources.events import NewResource +from golem_core.core.resources.events import NewResource, ResourceClosed from golem_core.managers.base import PaymentManager logger = logging.getLogger(__name__) @@ -28,9 +29,17 @@ def __init__( self._allocation: Optional[Allocation] = None - self._golem.event_bus.resource_listen(self.on_invoice_received, [NewResource], [Invoice]) + self._golem.event_bus.resource_listen(self._on_invoice_received, [NewResource], [Invoice]) self._golem.event_bus.resource_listen( - self.on_debit_note_received, [NewResource], [DebitNote] + self._on_debit_note_received, [NewResource], [DebitNote] + ) + + self._opened_agreements_count: int = 0 + self._closed_agreements_count: int = 0 + self._payed_invoices_count: int = 0 + self._golem.event_bus.resource_listen(self._on_new_agreement, [NewResource], [Agreement]) + self._golem.event_bus.resource_listen( + self._on_agreement_closed, [ResourceClosed], [Agreement] ) async def get_allocation(self) -> "Allocation": @@ -45,7 +54,25 @@ async def get_allocation(self) -> "Allocation": logger.info(f"Getting allocation done {self._allocation.id}") return self._allocation - async def on_invoice_received(self, invoice_event: NewResource) -> None: + async def wait_for_invoices(self): + logger.info("Waiting for invoices...") + for _ in range(60): + await asyncio.sleep(1) + if ( + self._opened_agreements_count + == self._closed_agreements_count + == self._payed_invoices_count + ): + break + logger.info("Waiting for invoices done") + + async def _on_new_agreement(self, agreement_event: NewResource): + self._opened_agreements_count += 1 + + async def _on_agreement_closed(self, agreement_event: ResourceClosed): + self._closed_agreements_count += 1 + + async def _on_invoice_received(self, invoice_event: NewResource) -> None: logger.info("Received invoice...") invoice = invoice_event.resource assert isinstance(invoice, Invoice) @@ -54,9 +81,10 @@ async def on_invoice_received(self, invoice_event: NewResource) -> None: assert self._allocation is not None # TODO think of a better way await invoice.accept_full(self._allocation) await invoice.get_data(force=True) + self._payed_invoices_count += 1 logger.info(f"Accepting invoice done {invoice.id}") - async def on_debit_note_received(self, debit_note_event: NewResource) -> None: + async def _on_debit_note_received(self, debit_note_event: NewResource) -> None: logger.info("Received debit note...") debit_note = debit_note_event.resource assert isinstance(debit_note, DebitNote) @@ -66,8 +94,3 @@ async def on_debit_note_received(self, debit_note_event: NewResource) -> None: await debit_note.accept_full(self._allocation) await debit_note.get_data(force=True) logger.info(f"Accepting debit note done {debit_note.id}") - - async def wait_for_invoices(self): - logger.info("Waiting for invoices...") - await asyncio.sleep(30) - logger.info("Waiting for invoices done") From 0f6af8401f4b3fccb6e933ed951c026c9740628a Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 31 May 2023 15:02:08 +0200 Subject: [PATCH 037/123] Add batch work example --- examples/managers/basic_composition.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index db8aa8ab..b7c51cf1 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -1,6 +1,5 @@ import asyncio import logging.config -from functools import partial from typing import List from golem_core.core.golem_node.golem_node import GolemNode @@ -15,8 +14,8 @@ from golem_core.utils.logging import DEFAULT_LOGGING -async def work(context: WorkContext, label: str) -> str: - r = await context.run(f"echo {label}") +async def commands_work_example(context: WorkContext) -> str: + r = await context.run("echo 'hello golem'") await r.wait() result = "" for event in r.events: @@ -24,7 +23,15 @@ async def work(context: WorkContext, label: str) -> str: return result -# TODO add Batch example +async def batch_work_example(context: WorkContext): + batch = await context.create_batch() + batch.run("echo 'hello batch'") + batch.run("echo 'bye batch'") + batch_result = await batch() + result = "" + for event in batch_result: + result += event.stdout + return result async def main(): @@ -32,9 +39,8 @@ async def main(): payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") work_list = [ - partial(work, label="label-1"), - partial(work, label="label-2"), - partial(work, label="label-3"), + commands_work_example, + batch_work_example, ] async with GolemNode() as golem: From d6fd9889b3a74c38a42cbde3150cc1a54ab082ac Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 31 May 2023 18:29:26 +0200 Subject: [PATCH 038/123] finished new event_bus --- big_5_draft.py | 236 ------------------ examples/attach.py | 4 +- examples/core_example.py | 4 +- examples/managers/basic_composition.py | 1 + examples/task_api_draft/examples/yacat.py | 6 +- golem_core/core/activity_api/events.py | 30 ++- .../core/activity_api/resources/activity.py | 7 +- .../activity_api/resources/pooling_batch.py | 11 +- golem_core/core/events/__init__.py | 5 +- golem_core/core/events/base.py | 83 ++---- golem_core/core/events/event_bus.py | 210 +++++++++++++++- golem_core/core/golem_node/golem_node.py | 15 +- golem_core/core/market_api/__init__.py | 4 + golem_core/core/market_api/events.py | 37 +++ .../core/market_api/resources/agreement.py | 10 +- .../market_api/resources/demand/demand.py | 12 +- .../core/market_api/resources/proposal.py | 8 +- golem_core/core/network_api/events.py | 18 ++ .../core/network_api/resources/network.py | 4 +- golem_core/core/payment_api/__init__.py | 8 +- golem_core/core/payment_api/events.py | 37 +++ .../core/payment_api/resources/allocation.py | 11 +- .../core/payment_api/resources/debit_note.py | 11 +- .../core/payment_api/resources/invoice.py | 9 +- golem_core/core/resources/base.py | 9 +- golem_core/core/resources/events.py | 18 +- golem_core/managers/activity/single_use.py | 26 +- golem_core/managers/agreement/single_use.py | 26 +- golem_core/managers/negotiation/accept_all.py | 22 +- golem_core/managers/payment/default.py | 32 +-- golem_core/managers/payment/pay_all.py | 81 +++--- golem_core/managers/proposal/stack.py | 15 +- golem_core/managers/work/sequential.py | 9 +- golem_core/utils/logging.py | 9 + tests/integration/test_app_session_id.py | 2 +- tests/unit/test_event_bus.py | 171 +++++++++++++ 36 files changed, 763 insertions(+), 438 deletions(-) delete mode 100644 big_5_draft.py create mode 100644 golem_core/core/market_api/events.py create mode 100644 golem_core/core/network_api/events.py create mode 100644 golem_core/core/payment_api/events.py create mode 100644 tests/unit/test_event_bus.py diff --git a/big_5_draft.py b/big_5_draft.py deleted file mode 100644 index 8c6eeff0..00000000 --- a/big_5_draft.py +++ /dev/null @@ -1,236 +0,0 @@ -import asyncio -from typing import Callable, List, Optional - -from golem_core.core.market_api import RepositoryVmPayload -from golem_core.managers.activity.single_use import SingleUseActivityManager -from golem_core.managers.agreement.single_use import SingleUseAgreementManager -from golem_core.managers.base import WorkContext -from golem_core.managers.payment.pay_all import PayAllPaymentManager -from golem_core.managers.work.decorators import ( - redundancy_cancel_others_on_first_done, - retry, - work_decorator, -) -from golem_core.managers.work.sequential import SequentialWorkManager - - -class ConfirmAllNegotiationManager: - def __init__(self, get_allocation: "Callable", payload, event_bus): - self._event_bus = event_bus - self._get_allocation = get_allocation - self._allocation = self._get_allocation() - self._payload = payload - demand_builder = DemandBuilder() - demand_builder.add(self._payload) - demand_builder.add(self._allocation) - self._demand = demand_builder.create_demand() - - def negotiate(self): - for initial in self._demand.get_proposals(): # infinite loop - pending = initial.respond() - confirmed = pending.confirm() - self._event_bus.register(ProposalConfirmed(demand=self._demand, proposal=confirmed)) - - -def filter_blacklist(proposal: "Proposal") -> bool: - providers_blacklist: List[str] = ... - return proposal.provider_id in providers_blacklist - - -class FilterNegotiationManager: - INITIAL = "INITIAL" - PENDING = "PENDING" - - def __init__(self, get_allocation: "Callable", payload, event_bus): - self._event_bus = event_bus - self._get_allocation = get_allocation - self._allocation = self._get_allocation() - self._payload = payload - demand_builder = DemandBuilder() - demand_builder.add(self._payload) - demand_builder.add(self._allocation) - self._demand = demand_builder.create_demand() - self._filters = { - self.INITIAL: [], - self.PENDING: [], - } - - def add_filter(self, filter: "Filter", type: str): - self._filters[type].append(filter) - - def _filter(self, initial: "Proposal", type: str) -> bool: - for f in self._filters[type]: - if f(initial): - return True - return False - - def negotiate(self): - for initial in self._demand.get_proposals(): # infinite loop - if self._filter(initial, self.INITIAL): - continue - - pending = initial.respond() - if self._filter(pending, self.PENDING): - pending.reject() - continue - - confirmed = pending.confirm() - self._event_bus.register(ProposalConfirmed(demand=self._demand, proposal=confirmed)) - - -class AcceptableRangeNegotiationManager: - def __init__(self, get_allocation: "Callable", payload, event_bus): - self._event_bus = event_bus - self._get_allocation = get_allocation - self._allocation = self._get_allocation() - self._payload = payload - demand_builder = DemandBuilder() - demand_builder.add(self._payload) - demand_builder.add(self._allocation) - self._demand = demand_builder.create_demand() - - def negotiate(self): - for initial in self._demand.get_proposals(): # infinite loop - pending = self._negotiate_for_accepted_range(initial) - if pending is None: - continue - confirmed = pending.confirm() - self._event_bus.register(ProposalConfirmed(demand=self._demand, proposal=confirmed)) - - def _validate_values(self, proposal: "Proposal") -> bool: - """Checks if proposal's values are in accepted range - - e.g. - True - x_accepted range: [2,10] - proposal.x: 9 - - False - x_accepted range: [2,10] - proposal.x: 11 - """ - - def _middle_values(self, our: "Proposal", their: "Proposal") -> Optional["Proposal"]: - """Create new proposal with new values in accepted range based on given proposals. - - If middle values are outside of accepted range return None - - e.g. - New proposal - x_accepted range: [2,10] - our.x: 5 - their.x : 13 - new: (5+13)//2 -> 9 - - None - x_accepted range: [2,10] - our.x: 9 - their.x : 13 - new: (9+13)//2 -> 11 -> None - """ - - def _negotiate_for_accepted_range(self, our: "Proposal"): - their = our.respond() - while True: - if self._validate_values(their): - return their - our = self._middle_values(our, their) - if our is None: - return None - their = their.respond_with(our) - - -def blacklist_offers(blacklist: "List"): - def _blacklist_offers(func): - def wrapper(*args, **kwargs): - while True: - offer = func() - - if offer not in blacklist: - return offer - - return wrapper - - return _blacklist_offers - - -class LifoOfferManager: - _offers: List["Offer"] - - def __init__(self, event_bus) -> None: - self._event_bus = event_bus - self._event_bus.resource_listen(self.on_new_offer, ProposalConfirmed) - - def on_new_offer(self, offer: "Offer") -> None: - self._offers.append(offer) - - def get_offer(self) -> "Offer": - while True: - try: - return self._offers.pop() - except IndexError: - # wait for offers - # await sleep - pass - - -@work_decorator(redundancy_cancel_others_on_first_done(size=5)) -@work_decorator(retry(tries=5)) -async def work(context: WorkContext): - await context.run("echo hello world") - - -async def work_service(context: WorkContext): - await context.run("app --daemon") - - -async def work_service_fetch(context: WorkContext): - await context.run("app --daemon &") - - while await context.run("app check-if-running"): - await asyncio.sleep(1) - - -async def work_batch(context: WorkContext): - batch = await context.create_batch() - batch.run("echo 1") - batch.run("echo 2") - batch.run("echo 3") - - await batch() - - -async def main(): - payload = RepositoryVmPayload(image_hash="...") - budget = 1.0 - work_list = [ - work, - work, - work, - ] - event_bus = EventBus() - - payment_manager = PayAllPaymentManager(budget, event_bus) - - negotiation_manager = FilterNegotiationManager( - payment_manager.get_allocation, payload, event_bus - ) - negotiation_manager.add_filter(filter_blacklist, negotiation_manager.INITIAL) - negotiation_manager.negotiate() # run in async, this will generate ProposalConfirmed events - - offer_manager = LifoOfferManager(event_bus) # listen to ProposalConfirmed - - agreement_manager = SingleUseAgreementManager( - blacklist_offers(["banned_node_id"])(offer_manager.get_offer) - ) - - activity_manager = SingleUseActivityManager( - agreement_manager.get_agreement, - ) - - work_manager = SequentialWorkManager(activity_manager.do_work) - await work_manager.do_work_list(work_list) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/attach.py b/examples/attach.py index 1a67c256..aeb34a17 100644 --- a/examples/attach.py +++ b/examples/attach.py @@ -3,7 +3,7 @@ from golem_core.core.activity_api import commands from golem_core.core.golem_node import GolemNode -from golem_core.core.payment_api import DebitNote +from golem_core.core.payment_api import DebitNote, NewDebitNote from golem_core.core.resources import NewResource ACTIVITY_ID = sys.argv[1].strip() @@ -22,7 +22,7 @@ async def accept_debit_note(payment_event: NewResource) -> None: async def main() -> None: golem = GolemNode() - golem.event_bus.resource_listen(accept_debit_note, [NewResource], [DebitNote]) + await golem.event_bus.on(NewDebitNote, accept_debit_note) async with golem: activity = golem.activity(ACTIVITY_ID) diff --git a/examples/core_example.py b/examples/core_example.py index aa6afacd..cba778ca 100644 --- a/examples/core_example.py +++ b/examples/core_example.py @@ -116,10 +116,10 @@ async def example_5() -> None: golem = GolemNode() got_events = [] - async def on_event(event: ResourceEvent) -> None: + async def collect_resource_events(event: ResourceEvent) -> None: got_events.append(event) - golem.event_bus.resource_listen(on_event) + await golem.event_bus.on(ResourceEvent, collect_resource_events) async with golem: allocation = await golem.create_allocation(1) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index b7c51cf1..d3598599 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -51,6 +51,7 @@ async def main(): activity_manager = SingleUseActivityManager(golem, agreement_manager.get_agreement) work_manager = SequentialWorkManager(golem, activity_manager.do_work) + await payment_manager.start() await negotiation_manager.start_negotiation(payload) await proposal_manager.start_consuming_proposals() diff --git a/examples/task_api_draft/examples/yacat.py b/examples/task_api_draft/examples/yacat.py index 4e5d196f..43354957 100644 --- a/examples/task_api_draft/examples/yacat.py +++ b/examples/task_api_draft/examples/yacat.py @@ -24,6 +24,7 @@ ) from examples.task_api_draft.task_api.activity_pool import ActivityPool from golem_core.core.activity_api import Activity, PoolingBatch, default_prepare_activity +from golem_core.core.events import Event from golem_core.core.golem_node import GolemNode from golem_core.core.market_api import ( Proposal, @@ -32,6 +33,7 @@ default_negotiate, ) from golem_core.core.payment_api import DebitNote +from golem_core.core.payment_api.events import NewDebitNote from golem_core.core.resources import NewResource, ResourceClosed from golem_core.managers import DefaultPaymentManager from golem_core.pipeline import Buffer, Chain, Map, Sort, Zip @@ -171,9 +173,9 @@ async def main() -> None: asyncio.create_task(manage_activities()) golem = GolemNode() - golem.event_bus.listen(DefaultLogger().on_event) + await golem.event_bus.on(Event, DefaultLogger().on_event) golem.event_bus.resource_listen(count_batches, [NewResource], [PoolingBatch]) - golem.event_bus.resource_listen(gather_debit_note_log, [NewResource], [DebitNote]) + await golem.event_bus.on(NewDebitNote, gather_debit_note_log) golem.event_bus.resource_listen(update_new_activity_status, [NewResource], [Activity]) golem.event_bus.resource_listen(note_activity_destroyed, [ResourceClosed], [Activity]) diff --git a/golem_core/core/activity_api/events.py b/golem_core/core/activity_api/events.py index ae993eb1..c6c464e0 100644 --- a/golem_core/core/activity_api/events.py +++ b/golem_core/core/activity_api/events.py @@ -1,16 +1,34 @@ from typing import TYPE_CHECKING -from golem_core.core.resources import ResourceEvent +from golem_core.core.resources import ( + NewResource, + ResourceClosed, + ResourceDataChanged, + ResourceEvent, +) if TYPE_CHECKING: - from golem_core.core.activity_api.resources import PoolingBatch + pass -class BatchFinished(ResourceEvent): +class NewActivity(NewResource["Activity"]): + pass + + +class ActivityDataChanged(ResourceDataChanged["Activity"]): + pass + + +class ActivityClosed(ResourceClosed["Activity"]): + pass + + +class NewPoolingBatch(NewResource["PoolingBatch"]): + pass + + +class BatchFinished(ResourceEvent["PoolingBatch"]): """Emitted when the execution of a :any:`PoolingBatch` finishes. The same event is emitted for successful and failed batches. """ - - def __init__(self, resource: "PoolingBatch"): - super().__init__(resource) diff --git a/golem_core/core/activity_api/resources/activity.py b/golem_core/core/activity_api/resources/activity.py index 6286863f..7bb714b1 100644 --- a/golem_core/core/activity_api/resources/activity.py +++ b/golem_core/core/activity_api/resources/activity.py @@ -6,9 +6,10 @@ from ya_activity import models from golem_core.core.activity_api.commands import Command, Script +from golem_core.core.activity_api.events import ActivityClosed, NewActivity from golem_core.core.activity_api.resources.pooling_batch import PoolingBatch from golem_core.core.payment_api import DebitNote -from golem_core.core.resources import _NULL, ActivityApi, Resource, ResourceClosed, api_call_wrapper +from golem_core.core.resources import _NULL, ActivityApi, Resource, api_call_wrapper if TYPE_CHECKING: from golem_core.core.golem_node.golem_node import GolemNode @@ -24,6 +25,8 @@ class Activity(Resource[ActivityApi, _NULL, "Agreement", PoolingBatch, _NULL]): def __init__(self, node: "GolemNode", id_: str): super().__init__(node, id_) + asyncio.create_task(node.event_bus.emit(NewActivity(self))) + self._running_batch_counter = 0 self._busy_event = asyncio.Event() self._idle_event = asyncio.Event() @@ -85,7 +88,7 @@ async def destroy(self) -> None: used.""" await self.api.destroy_activity(self.id) self._destroyed_event.set() - self.node.event_bus.emit(ResourceClosed(self)) + await self.node.event_bus.emit(ActivityClosed(self)) @api_call_wrapper() async def execute(self, script: models.ExeScriptRequest) -> PoolingBatch: diff --git a/golem_core/core/activity_api/resources/pooling_batch.py b/golem_core/core/activity_api/resources/pooling_batch.py index 07cc42a0..77b205a3 100644 --- a/golem_core/core/activity_api/resources/pooling_batch.py +++ b/golem_core/core/activity_api/resources/pooling_batch.py @@ -4,7 +4,7 @@ from ya_activity import models -from golem_core.core.activity_api.events import BatchFinished +from golem_core.core.activity_api.events import BatchFinished, NewPoolingBatch from golem_core.core.activity_api.exceptions import ( BatchError, BatchTimeoutError, @@ -36,6 +36,7 @@ class PoolingBatch( def __init__(self, node: "GolemNode", id_: str): super().__init__(node, id_) + asyncio.create_task(node.event_bus.emit(NewPoolingBatch(self))) self.finished_event = asyncio.Event() self._futures: Optional[List[asyncio.Future[models.ExeScriptCommandResult]]] = None @@ -112,7 +113,7 @@ async def _collect_yagna_events(self) -> None: # This happens when activity_api is destroyed when we're waiting for batch results # (I'm not sure if always - for sure when provider destroys activity because # agreement timed out). Maybe some other scenarios are also possible. - self._set_finished() + await self._set_finished() def _collect_events_kwargs(self) -> Dict: return {"timeout": 5, "_request_timeout": 5.5} @@ -140,10 +141,10 @@ async def _process_event(self, event: models.ExeScriptCommandResult) -> None: self._futures[event.index].set_result(event) if event.is_batch_finished: - self._set_finished() + await self._set_finished() - def _set_finished(self) -> None: - self.node.event_bus.emit(BatchFinished(self)) + async def _set_finished(self) -> None: + await self.node.event_bus.emit(BatchFinished(self)) self.finished_event.set() self.parent.running_batch_counter -= 1 self.stop_collecting_events() diff --git a/golem_core/core/events/__init__.py b/golem_core/core/events/__init__.py index 42a81be0..30bce955 100644 --- a/golem_core/core/events/__init__.py +++ b/golem_core/core/events/__init__.py @@ -1,11 +1,12 @@ -from golem_core.core.events.base import Event, TEvent -from golem_core.core.events.event_bus import EventBus +from golem_core.core.events.base import Event, EventBus, TEvent +from golem_core.core.events.event_bus import InMemoryEventBus from golem_core.core.events.event_filters import AnyEventFilter, EventFilter __all__ = ( "Event", "TEvent", "EventBus", + "InMemoryEventBus", "EventFilter", "AnyEventFilter", ) diff --git a/golem_core/core/events/base.py b/golem_core/core/events/base.py index ff45f505..d1db7dd1 100644 --- a/golem_core/core/events/base.py +++ b/golem_core/core/events/base.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod -from collections import defaultdict -from dataclasses import dataclass -from typing import Awaitable, Callable, DefaultDict, List, Optional, Type, TypeVar +from typing import Awaitable, Callable, Optional, Type, TypeVar + +from golem_core.core.exceptions import BaseCoreException TEvent = TypeVar("TEvent", bound="Event") @@ -13,94 +13,45 @@ class Event(ABC): TCallbackHandler = TypeVar("TCallbackHandler") +class EventBusError(BaseCoreException): + pass + + class EventBus(ABC): @abstractmethod - async def on( - self, - event_type: Type[TEvent], - callback: Callable[[TEvent], Awaitable[None]], - filter_func: Optional[Callable[[TEvent], bool]] = None, - ) -> TCallbackHandler: + async def start(self) -> None: ... @abstractmethod - async def on_once( - self, event_type: Type[TEvent], callback: Callable[[TEvent], Awaitable[None]] - ) -> TCallbackHandler: + async def stop(self) -> None: ... @abstractmethod - async def off(self, callback_handler: TCallbackHandler) -> None: + def is_started(self) -> bool: ... @abstractmethod - async def emit(self, event: TEvent) -> None: - ... - - -@dataclass -class CallbackInfo: - callback: Callable[[TEvent], Awaitable[None]] - filter_func: Optional[Callable[[TEvent], bool]] - once: bool - - -class InMemoryEventBus(EventBus): - def __init__(self): - self._callbacks: DefaultDict[Type[TEvent], List[CallbackInfo]] = defaultdict(list) - async def on( self, event_type: Type[TEvent], callback: Callable[[TEvent], Awaitable[None]], filter_func: Optional[Callable[[TEvent], bool]] = None, ) -> TCallbackHandler: - callback_info = CallbackInfo( - callback=callback, - filter_func=filter_func, - once=False, - ) - - self._callbacks[event_type].append(callback_info) - - return (event_type, callback_info) + ... + @abstractmethod async def on_once( self, event_type: Type[TEvent], callback: Callable[[TEvent], Awaitable[None]], filter_func: Optional[Callable[[TEvent], bool]] = None, ) -> TCallbackHandler: - callback_info = CallbackInfo( - callback=callback, - filter_func=filter_func, - once=True, - ) - - self._callbacks[event_type].append(callback_info) - - return (event_type, callback_info) + ... + @abstractmethod async def off(self, callback_handler: TCallbackHandler) -> None: - event_type, callback_info = callback_handler - - try: - self._callbacks[event_type].remove(callback_info) - except (KeyError, ValueError): - raise ValueError("Given callback handler is not found in event bus!") + ... + @abstractmethod async def emit(self, event: TEvent) -> None: - for event_type, callback_infos in self._callbacks.items(): - if not isinstance(event, event_type): - continue - - callback_infos_copy = callback_infos[:] - - for callback_info in callback_infos_copy: - if callback_info.filter_func is not None and not callback_info.filter_func(event): - continue - - callback_info.callback(event) - - if callback_info.once: - callback_infos.remove(callback_info) + ... diff --git a/golem_core/core/events/event_bus.py b/golem_core/core/events/event_bus.py index af21356a..875dfafb 100644 --- a/golem_core/core/events/event_bus.py +++ b/golem_core/core/events/event_bus.py @@ -1,5 +1,7 @@ import asyncio +import logging from collections import defaultdict +from dataclasses import dataclass from typing import ( TYPE_CHECKING, Any, @@ -12,13 +14,18 @@ Type, ) -from golem_core.core.events.base import Event, TEvent +from golem_core.core.events.base import Event +from golem_core.core.events.base import EventBus as BaseEventBus +from golem_core.core.events.base import EventBusError, TCallbackHandler, TEvent from golem_core.core.events.event_filters import AnyEventFilter, EventFilter if TYPE_CHECKING: from golem_core.core.resources import Resource, ResourceEvent, TResourceEvent +logger = logging.getLogger(__name__) + + class EventBus: """Emit events, listen for events. @@ -127,3 +134,204 @@ async def _emit(self, event: Event) -> None: if tasks: await asyncio.gather(*tasks, return_exceptions=True) self.queue.task_done() + + +@dataclass +class _CallbackInfo: + callback: Callable[[TEvent], Awaitable[None]] + filter_func: Optional[Callable[[TEvent], bool]] + once: bool + + +class InMemoryEventBus(BaseEventBus): + def __init__(self): + self._callbacks: DefaultDict[Type[TEvent], List[_CallbackInfo]] = defaultdict(list) + self._event_queue = asyncio.Queue() + self._process_event_queue_loop_task: Optional[asyncio.Task] = None + + async def start(self): + logger.debug("Starting event bus...") + + if self.is_started(): + message = "Event bus is already started!" + logger.debug(f"Starting event bus failed with `{message}`") + raise EventBusError(message) + + self._process_event_queue_loop_task = asyncio.create_task(self._process_event_queue_loop()) + + logger.debug("Starting event bus done") + + async def stop(self): + logger.debug("Stopping event bus...") + + if not self.is_started(): + message = "Event bus is not started!" + logger.debug(f"Stopping event bus failed with `{message}`") + raise EventBusError(message) + + await self._event_queue.join() + + self._process_event_queue_loop_task.cancel() + self._process_event_queue_loop_task = None + + logger.debug("Stopping event bus done") + + def is_started(self) -> bool: + return self._process_event_queue_loop_task is not None + + async def on( + self, + event_type: Type[TEvent], + callback: Callable[[TEvent], Awaitable[None]], + filter_func: Optional[Callable[[TEvent], bool]] = None, + ) -> TCallbackHandler: + logger.debug( + f"Adding callback handler for `{event_type}` with callback `{callback}` and filter `{filter_func}`..." + ) + + callback_info = _CallbackInfo( + callback=callback, + filter_func=filter_func, + once=False, + ) + + self._callbacks[event_type].append(callback_info) + + callback_handler = (event_type, callback_info) + + logger.debug(f"Adding callback handler done with `{id(callback_handler)}`") + + return callback_handler + + async def on_once( + self, + event_type: Type[TEvent], + callback: Callable[[TEvent], Awaitable[None]], + filter_func: Optional[Callable[[TEvent], bool]] = None, + ) -> TCallbackHandler: + logger.debug( + f"Adding one-time callback handler for `{event_type}` with callback `{callback}` and filter `{filter_func}`..." + ) + + callback_info = _CallbackInfo( + callback=callback, + filter_func=filter_func, + once=True, + ) + + self._callbacks[event_type].append(callback_info) + + callback_handler = (event_type, callback_info) + + logger.debug(f"Adding one-time callback handler done with `{id(callback_handler)}`") + + return callback_handler + + async def off(self, callback_handler: TCallbackHandler) -> None: + logger.debug(f"Removing callback handler `{id(callback_handler)}`...") + + event_type, callback_info = callback_handler + + try: + self._callbacks[event_type].remove(callback_info) + except (KeyError, ValueError): + message = "Given callback handler is not found in event bus!" + logger.debug( + f"Removing callback handler `{id(callback_handler)}` failed with `{message}`" + ) + raise EventBusError(message) + + logger.debug(f"Removing callback handler `{id(callback_handler)}` done") + + async def emit(self, event: TEvent) -> None: + logger.debug(f"Emitting event `{event}`...") + + if not self.is_started(): + message = "Event bus is not started!" + logger.debug(f"Emitting event `{event}` failed with `message`") + raise EventBusError(message) + + await self._event_queue.put(event) + + logger.debug(f"Emitting event `{event}` done") + + async def _process_event_queue_loop(self): + while True: + logger.debug("Getting event from queue...") + + event = await self._event_queue.get() + + logger.debug(f"Getting event from queue done with `{event}`") + + logger.debug(f"Processing callbacks for event `{event}`...") + + for event_type, callback_infos in self._callbacks.items(): + await self._process_event(event, event_type, callback_infos) + + logger.debug(f"Processing callbacks for event `{event}` done") + + self._event_queue.task_done() + + async def _process_event( + self, event: Event, event_type: Type[Event], callback_infos: List[_CallbackInfo] + ): + logger.debug(f"Processing event `{event}` on event type `{event_type}`...") + + if not isinstance(event, event_type): + logger.debug( + f"Processing event `{event}` on event type `{event_type}` ignored as event is not a instance of event type" + ) + return + + callback_infos_to_remove = [] + + logger.debug(f"Processing callbacks for event {event}...") + + for callback_info in callback_infos: + logger.debug(f"Processing callback {callback_info}...") + + if callback_info.filter_func is not None: + logger.debug("Calling filter function...") + try: + if not callback_info.filter_func(event): + logger.debug("Calling filter function done, ignoring callback") + continue + except: + logger.exception( + f"Encountered an error in `{callback_info.filter_func}` filter function while handling `{event}`!" + ) + continue + else: + logger.debug("Calling filter function done, calling callback") + else: + logger.debug("Callback has no filter function") + + logger.debug(f"Calling {callback_info.callback}...") + try: + await callback_info.callback(event) + except Exception as e: + logger.debug(f"Calling {callback_info.callback} failed with `{e}") + + logger.exception( + f"Encountered an error in `{callback_info.callback}` callback while handling `{event}`!" + ) + continue + else: + logger.debug(f"Calling {callback_info.callback} done") + + if callback_info.once: + callback_infos_to_remove.append(callback_info) + + logger.debug(f"Callback {callback_info} marked as to be removed") + + logger.debug(f"Processing callbacks for event {event} done") + + if callback_infos_to_remove: + logger.debug(f"Removing callbacks `{callback_infos_to_remove}`...") + + for callback_info in callback_infos_to_remove: + callback_infos.remove(callback_info) + + logger.debug(f"Removing callbacks `{callback_infos_to_remove}` done") + + logger.debug(f"Processing event `{event}` on event type `{event_type}` done") diff --git a/golem_core/core/golem_node/golem_node.py b/golem_core/core/golem_node/golem_node.py index 03779355..7c2a0cec 100644 --- a/golem_core/core/golem_node/golem_node.py +++ b/golem_core/core/golem_node/golem_node.py @@ -7,7 +7,7 @@ from uuid import uuid4 from golem_core.core.activity_api import Activity, PoolingBatch -from golem_core.core.events import EventBus +from golem_core.core.events import EventBus, InMemoryEventBus from golem_core.core.golem_node.events import SessionStarted, ShutdownFinished, ShutdownStarted from golem_core.core.market_api import Agreement, Demand, DemandBuilder, Payload, Proposal from golem_core.core.market_api.resources.demand.demand_offer_base import defaults as dobm_defaults @@ -83,7 +83,7 @@ def __init__( # (This is done internally by the metaclass of the Resource) self._resources: DefaultDict[Type[Resource], Dict[str, Resource]] = defaultdict(dict) self._autoclose_resources: Set[Resource] = set() - self._event_bus = EventBus() + self._event_bus = InMemoryEventBus() self._invoice_event_collector = InvoiceEventCollector(self) self._debit_note_event_collector = DebitNoteEventCollector(self) @@ -108,7 +108,7 @@ async def __aexit__(self, *exc_info: Any) -> None: await self.aclose() async def start(self) -> None: - self._event_bus.start() + await self.event_bus.start() api_factory = ApiFactory(self._api_config) self._ya_market_api = api_factory.create_market_api_client() @@ -120,16 +120,17 @@ async def start(self) -> None: self._invoice_event_collector.start_collecting_events() self._debit_note_event_collector.start_collecting_events() - self.event_bus.emit(SessionStarted(self)) + await self.event_bus.emit(SessionStarted(self)) async def aclose(self) -> None: - self.event_bus.emit(ShutdownStarted(self)) + await self.event_bus.emit(ShutdownStarted(self)) self._set_no_more_children() self._stop_event_collectors() await self._close_autoclose_resources() await self._close_apis() - self.event_bus.emit(ShutdownFinished(self)) - await self._event_bus.stop() + await self.event_bus.emit(ShutdownFinished(self)) + + await self.event_bus.stop() def _stop_event_collectors(self) -> None: demands = self.all_resources(Demand) diff --git a/golem_core/core/market_api/__init__.py b/golem_core/core/market_api/__init__.py index 60e86983..d28ae76a 100644 --- a/golem_core/core/market_api/__init__.py +++ b/golem_core/core/market_api/__init__.py @@ -1,3 +1,4 @@ +from golem_core.core.market_api.events import AgreementClosed, AgreementDataChanged, NewAgreement from golem_core.core.market_api.exceptions import BaseMarketApiException from golem_core.core.market_api.pipeline import ( default_create_activity, @@ -64,4 +65,7 @@ "ConstraintException", "InvalidPropertiesError", "BaseMarketApiException", + "NewAgreement", + "AgreementDataChanged", + "AgreementClosed", ) diff --git a/golem_core/core/market_api/events.py b/golem_core/core/market_api/events.py new file mode 100644 index 00000000..eeaa1286 --- /dev/null +++ b/golem_core/core/market_api/events.py @@ -0,0 +1,37 @@ +from golem_core.core.resources import NewResource, ResourceClosed, ResourceDataChanged + + +class NewDemand(NewResource["Demand"]): + pass + + +class DemandDataChanged(ResourceDataChanged["Demand"]): + pass + + +class DemandClosed(ResourceClosed["Demand"]): + pass + + +class NewAgreement(NewResource["Agreement"]): + pass + + +class AgreementDataChanged(ResourceDataChanged["Agreement"]): + pass + + +class AgreementClosed(ResourceClosed["Agreement"]): + pass + + +class NewProposal(NewResource["Proposal"]): + pass + + +class ProposalDataChanged(ResourceDataChanged["Proposal"]): + pass + + +class ProposalClosed(ResourceClosed["Proposal"]): + pass diff --git a/golem_core/core/market_api/resources/agreement.py b/golem_core/core/market_api/resources/agreement.py index e0599988..af5501ec 100644 --- a/golem_core/core/market_api/resources/agreement.py +++ b/golem_core/core/market_api/resources/agreement.py @@ -7,8 +7,10 @@ from ya_market.exceptions import ApiException from golem_core.core.activity_api import Activity +from golem_core.core.market_api import AgreementClosed, NewAgreement from golem_core.core.payment_api import Invoice -from golem_core.core.resources import _NULL, Resource, ResourceClosed, api_call_wrapper +from golem_core.core.resources import _NULL, Resource, api_call_wrapper +from golem_core.core.resources.base import TModel if TYPE_CHECKING: from golem_core.core.market_api.resources.proposal import Proposal # noqa @@ -27,6 +29,10 @@ class Agreement(Resource[RequestorApi, models.Agreement, "Proposal", Activity, _ await agreement.terminate() """ + def __init__(self, node: "GolemNode", id_: str, data: Optional[TModel] = None): + super().__init__(node, id_, data) + asyncio.create_task(node.event_bus.emit(NewAgreement(self))) + @api_call_wrapper() async def confirm(self) -> None: """Confirm the agreement. @@ -87,7 +93,7 @@ async def terminate(self, reason: str = "") -> None: else: raise - self.node.event_bus.emit(ResourceClosed(self)) + await self.node.event_bus.emit(AgreementClosed(self)) @property def invoice(self) -> Optional[Invoice]: diff --git a/golem_core/core/market_api/resources/demand/demand.py b/golem_core/core/market_api/resources/demand/demand.py index 5ecd9f27..b210c213 100644 --- a/golem_core/core/market_api/resources/demand/demand.py +++ b/golem_core/core/market_api/resources/demand/demand.py @@ -1,17 +1,19 @@ -from typing import TYPE_CHECKING, AsyncIterator, Callable, Dict, List, Union +import asyncio +from typing import TYPE_CHECKING, AsyncIterator, Callable, Dict, List, Optional, Union from ya_market import RequestorApi from ya_market import models as models +from golem_core.core.market_api.events import DemandClosed, NewDemand from golem_core.core.market_api.resources.proposal import Proposal from golem_core.core.resources import ( _NULL, Resource, - ResourceClosed, ResourceNotFound, YagnaEventCollector, api_call_wrapper, ) +from golem_core.core.resources.base import TModel if TYPE_CHECKING: from golem_core.core.golem_node import GolemNode @@ -23,6 +25,10 @@ class Demand(Resource[RequestorApi, models.Demand, _NULL, Proposal, _NULL], Yagn Created with one of the :class:`Demand`-returning methods of the :any:`GolemNode`. """ + def __init__(self, node: "GolemNode", id_: str, data: Optional[TModel] = None): + super().__init__(node, id_, data) + asyncio.create_task(node.event_bus.emit(NewDemand(self))) + ###################### # EXTERNAL INTERFACE @api_call_wrapper(ignore=[404, 410]) @@ -34,7 +40,7 @@ async def unsubscribe(self) -> None: self.set_no_more_children() self.stop_collecting_events() await self.api.unsubscribe_demand(self.id) - self.node.event_bus.emit(ResourceClosed(self)) + await self.node.event_bus.emit(DemandClosed(self)) async def initial_proposals(self) -> AsyncIterator["Proposal"]: """Yield initial proposals matched to this demand.""" diff --git a/golem_core/core/market_api/resources/proposal.py b/golem_core/core/market_api/resources/proposal.py index 5c9ea2b2..fbf28238 100644 --- a/golem_core/core/market_api/resources/proposal.py +++ b/golem_core/core/market_api/resources/proposal.py @@ -1,12 +1,14 @@ +import asyncio from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, AsyncIterator, Optional, Union from ya_market import RequestorApi from ya_market import models as models +from golem_core.core.market_api.events import NewProposal from golem_core.core.market_api.resources.agreement import Agreement from golem_core.core.resources import Resource -from golem_core.core.resources.base import api_call_wrapper +from golem_core.core.resources.base import TModel, api_call_wrapper if TYPE_CHECKING: from golem_core.core.golem_node import GolemNode @@ -40,6 +42,10 @@ class Proposal( _demand: Optional["Demand"] = None + def __init__(self, node: "GolemNode", id_: str, data: Optional[TModel] = None): + super().__init__(node, id_, data) + asyncio.create_task(node.event_bus.emit(NewProposal(self))) + ############################## # State-related properties @property diff --git a/golem_core/core/network_api/events.py b/golem_core/core/network_api/events.py new file mode 100644 index 00000000..92f31c55 --- /dev/null +++ b/golem_core/core/network_api/events.py @@ -0,0 +1,18 @@ +from typing import TYPE_CHECKING + +from golem_core.core.resources import NewResource, ResourceClosed, ResourceDataChanged + +if TYPE_CHECKING: + pass + + +class NewNetwork(NewResource["Network"]): + pass + + +class NetworkDataChanged(ResourceDataChanged["Network"]): + pass + + +class NetworkClosed(ResourceClosed["Network"]): + pass diff --git a/golem_core/core/network_api/resources/network.py b/golem_core/core/network_api/resources/network.py index dbd3be00..33a4381d 100644 --- a/golem_core/core/network_api/resources/network.py +++ b/golem_core/core/network_api/resources/network.py @@ -4,6 +4,7 @@ from ya_net import RequestorApi, models +from golem_core.core.network_api.events import NewNetwork from golem_core.core.network_api.exceptions import NetworkFull from golem_core.core.resources import _NULL, Resource, ResourceClosed, api_call_wrapper @@ -42,6 +43,7 @@ class Network(Resource[RequestorApi, models.Network, _NULL, _NULL, _NULL]): def __init__(self, golem_node: "GolemNode", id_: str, data: models.Network): super().__init__(golem_node, id_, data) + asyncio.create_task(golem_node.event_bus.emit(NewNetwork(self))) self._create_node_lock = asyncio.Lock() self._ip_network: IpNetwork = ip_network(data.ip, strict=False) @@ -68,7 +70,7 @@ async def create( async def remove(self) -> None: """Remove the network.""" await self.api.remove_network(self.id) - self.node.event_bus.emit(ResourceClosed(self)) + await self.node.event_bus.emit(ResourceClosed(self)) @api_call_wrapper() async def create_node(self, provider_id: str, node_ip: Optional[str] = None) -> str: diff --git a/golem_core/core/payment_api/__init__.py b/golem_core/core/payment_api/__init__.py index 7a8a251b..54651731 100644 --- a/golem_core/core/payment_api/__init__.py +++ b/golem_core/core/payment_api/__init__.py @@ -1,3 +1,4 @@ +from golem_core.core.payment_api.events import DebitNoteClosed, DebitNoteDataChanged, NewDebitNote from golem_core.core.payment_api.exceptions import BasePaymentApiException, NoMatchingAccount from golem_core.core.payment_api.resources import ( Allocation, @@ -7,7 +8,7 @@ InvoiceEventCollector, ) -__all__ = [ +__all__ = ( "Allocation", "DebitNote", "Invoice", @@ -15,4 +16,7 @@ "InvoiceEventCollector", "BasePaymentApiException", "NoMatchingAccount", -] + "NewDebitNote", + "DebitNoteClosed", + "DebitNoteDataChanged", +) diff --git a/golem_core/core/payment_api/events.py b/golem_core/core/payment_api/events.py new file mode 100644 index 00000000..f42a3fe1 --- /dev/null +++ b/golem_core/core/payment_api/events.py @@ -0,0 +1,37 @@ +from golem_core.core.resources import NewResource, ResourceClosed, ResourceDataChanged + + +class NewAllocation(NewResource["Allocation"]): + pass + + +class AllocationDataChanged(ResourceDataChanged["Allocation"]): + pass + + +class AllocationClosed(ResourceClosed["Allocation"]): + pass + + +class NewDebitNote(NewResource["DebitNote"]): + pass + + +class DebitNoteDataChanged(ResourceDataChanged["DebitNote"]): + pass + + +class DebitNoteClosed(ResourceClosed["DebitNote"]): + pass + + +class NewInvoice(NewResource["Invoice"]): + pass + + +class InvoiceDataChanged(ResourceDataChanged["Invoice"]): + pass + + +class InvoiceClosed(ResourceClosed["Invoice"]): + pass diff --git a/golem_core/core/payment_api/resources/allocation.py b/golem_core/core/payment_api/resources/allocation.py index 9cdf08a1..81d87608 100644 --- a/golem_core/core/payment_api/resources/allocation.py +++ b/golem_core/core/payment_api/resources/allocation.py @@ -1,11 +1,14 @@ +import asyncio from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, List, Tuple +from typing import TYPE_CHECKING, List, Optional, Tuple from _decimal import Decimal from ya_payment import RequestorApi, models +from golem_core.core.payment_api.events import NewAllocation from golem_core.core.payment_api.exceptions import NoMatchingAccount from golem_core.core.resources import _NULL, Resource, ResourceClosed, api_call_wrapper +from golem_core.core.resources.base import TModel if TYPE_CHECKING: from golem_core.core.golem_node import GolemNode @@ -17,6 +20,10 @@ class Allocation(Resource[RequestorApi, models.Allocation, _NULL, _NULL, _NULL]) Created with one of the :class:`Allocation`-returning methods of the :any:`GolemNode`. """ + def __init__(self, node: "GolemNode", id_: str, data: Optional[TModel] = None): + super().__init__(node, id_, data) + asyncio.create_task(node.event_bus.emit(NewAllocation(self))) + @api_call_wrapper(ignore=[404, 410]) async def release(self) -> None: """Release the allocation. @@ -25,7 +32,7 @@ async def release(self) -> None: released allocation is not available anymore. """ await self.api.release_allocation(self.id) - self.node.event_bus.emit(ResourceClosed(self)) + await self.node.event_bus.emit(ResourceClosed(self)) @classmethod async def create_any_account( diff --git a/golem_core/core/payment_api/resources/debit_note.py b/golem_core/core/payment_api/resources/debit_note.py index 927fc5b0..e7f7a7f1 100644 --- a/golem_core/core/payment_api/resources/debit_note.py +++ b/golem_core/core/payment_api/resources/debit_note.py @@ -1,12 +1,17 @@ -from typing import TYPE_CHECKING, Union +import asyncio +from typing import TYPE_CHECKING, Optional, Union from _decimal import Decimal from ya_payment import RequestorApi, models +from golem_core.core.payment_api import NewDebitNote from golem_core.core.payment_api.resources.allocation import Allocation from golem_core.core.resources import _NULL, Resource, api_call_wrapper +from golem_core.core.resources.base import TModel if TYPE_CHECKING: + from golem_core.core.golem_node import GolemNode + from golem_core.core.activity_api import Activity # noqa @@ -16,6 +21,10 @@ class DebitNote(Resource[RequestorApi, models.DebitNote, "Activity", _NULL, _NUL Ususally created by a :any:`GolemNode` initialized with `collect_payment_events = True`. """ + def __init__(self, node: "GolemNode", id_: str, data: Optional[TModel] = None): + super().__init__(node, id_, data) + asyncio.create_task(node.event_bus.emit(NewDebitNote(self))) + async def accept_full(self, allocation: Allocation) -> None: """Accept full debit note amount using a given :any:`Allocation`.""" amount_str = (await self.get_data()).total_amount_due diff --git a/golem_core/core/payment_api/resources/invoice.py b/golem_core/core/payment_api/resources/invoice.py index 594b5ec0..f3584e1a 100644 --- a/golem_core/core/payment_api/resources/invoice.py +++ b/golem_core/core/payment_api/resources/invoice.py @@ -1,10 +1,13 @@ +import asyncio from decimal import Decimal -from typing import TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Optional, Union from ya_payment import RequestorApi, models +from golem_core.core.payment_api.events import NewInvoice from golem_core.core.payment_api.resources.allocation import Allocation from golem_core.core.resources import _NULL, Resource, api_call_wrapper +from golem_core.core.resources.base import TModel if TYPE_CHECKING: from golem_core.core.market_api.resources.agreement import Agreement # noqa @@ -16,6 +19,10 @@ class Invoice(Resource[RequestorApi, models.Invoice, "Agreement", _NULL, _NULL]) Ususally created by a :any:`GolemNode` initialized with `collect_payment_events = True`. """ + def __init__(self, node: "GolemNode", id_: str, data: Optional[TModel] = None): + super().__init__(node, id_, data) + asyncio.create_task(node.event_bus.emit(NewInvoice(self))) + async def accept_full(self, allocation: Allocation) -> None: """Accept full invoice amount using a given :any:`Allocation`.""" amount_str = (await self.get_data()).amount diff --git a/golem_core/core/resources/base.py b/golem_core/core/resources/base.py index 27681b3a..52f33d49 100644 --- a/golem_core/core/resources/base.py +++ b/golem_core/core/resources/base.py @@ -20,7 +20,7 @@ from ya_net import ApiException as NetApiException from ya_payment import ApiException as PaymentApiException -from golem_core.core.resources.events import NewResource, ResourceDataChanged +from golem_core.core.resources.events import ResourceDataChanged from golem_core.core.resources.exceptions import ResourceNotFound from golem_core.core.resources.low import TRequestorApi, get_requestor_api @@ -87,7 +87,10 @@ def __call__(cls, node: "GolemNode", id_: str, *args, **kwargs): if id_ not in node._resources[cls]: obj = super(ResourceMeta, cls).__call__(node, id_, *args, **kwargs) # type: ignore node._resources[cls][id_] = obj - node.event_bus.emit(NewResource(obj)) + + # FIXME: Is there any better solution for this? + # asyncio.create_task(node.event_bus.emit(NewResource(obj))) + return node._resources[cls][id_] @@ -235,7 +238,7 @@ async def get_data(self, force: bool = False) -> TModel: old_data = self._data self._data = await self._get_data() if old_data is not None and old_data != self._data: - self.node.event_bus.emit(ResourceDataChanged(self, old_data)) + await self.node.event_bus.emit(ResourceDataChanged(self, old_data)) assert self._data is not None # mypy return self._data diff --git a/golem_core/core/resources/events.py b/golem_core/core/resources/events.py index 12d6f81f..af76b0ba 100644 --- a/golem_core/core/resources/events.py +++ b/golem_core/core/resources/events.py @@ -1,23 +1,23 @@ from abc import ABC -from typing import TYPE_CHECKING, Any, Dict, Tuple, TypeVar +from typing import TYPE_CHECKING, Any, Dict, Generic, Tuple, TypeVar from golem_core.core.events.base import Event if TYPE_CHECKING: from golem_core.core.resources.base import Resource - TResourceEvent = TypeVar("TResourceEvent", bound="ResourceEvent") +TResource = TypeVar("TResource", bound="Resource") -class ResourceEvent(Event, ABC): +class ResourceEvent(Event, ABC, Generic[TResource]): """Base class for all events related to a particular :any:`Resource`.""" - def __init__(self, resource: "Resource"): + def __init__(self, resource: TResource): self._resource = resource @property - def resource(self) -> "Resource": + def resource(self) -> TResource: """Resource related to this :class:`ResourceEvent`.""" return self._resource @@ -25,7 +25,7 @@ def __repr__(self) -> str: return f"{type(self).__name__}({self.resource})" -class NewResource(ResourceEvent): +class NewResource(ResourceEvent[TResource], ABC, Generic[TResource]): """Emitted when a new :any:`Resource` object is created. There are three distinct scenarios possible: @@ -39,7 +39,7 @@ class NewResource(ResourceEvent): """ -class ResourceDataChanged(ResourceEvent): +class ResourceDataChanged(ResourceEvent[TResource], Generic[TResource]): """Emitted when `data` attribute of a :any:`Resource` changes. This event is **not** emitted when the data "would have changed if we @@ -54,7 +54,7 @@ class ResourceDataChanged(ResourceEvent): resource-changing call. """ - def __init__(self, resource: "Resource", old_data: Any): + def __init__(self, resource: TResource, old_data: Any): super().__init__(resource) self._old_data = old_data @@ -88,7 +88,7 @@ def __repr__(self) -> str: return f"{type(self).__name__}({self.resource}, {diff_str})" -class ResourceClosed(ResourceEvent): +class ResourceClosed(ResourceEvent[TResource], Generic[TResource]): """Emitted when a resource is deleted or rendered unusable. Usual case is when we delete a resource (e.g. :any:`Allocation.release()`), diff --git a/golem_core/managers/activity/single_use.py b/golem_core/managers/activity/single_use.py index 93a8ac4e..8cc719d4 100644 --- a/golem_core/managers/activity/single_use.py +++ b/golem_core/managers/activity/single_use.py @@ -35,7 +35,6 @@ def __init__( @asynccontextmanager async def _prepare_activity(self) -> Activity: while True: - logger.info("Preparing activity...") logger.debug("Getting agreement...") agreement = await self._get_agreement() @@ -48,9 +47,10 @@ async def _prepare_activity(self) -> Activity: activity = await agreement.create_activity() logger.debug(f"Creating activity done with `{activity}`") + logger.info(f"Activity `{activity}` created") - logger.info(f"Preparing activity done {activity.id}") logger.debug("Yielding activity...") + yield activity logger.debug("Yielding activity done") @@ -65,12 +65,12 @@ async def _prepare_activity(self) -> Activity: logger.debug(f"Releasing agreement by emitting `{event}`...") - self._event_bus.emit(event) + await self._event_bus.emit(event) logger.debug(f"Releasing agreement by emitting `{event}` done") async def do_work(self, work: Work) -> WorkResult: - logger.info(f"Doing work {work} on activity") + logger.debug(f"Doing work `{work}`...") async with self._prepare_activity() as activity: work_context = WorkContext(activity) @@ -83,33 +83,35 @@ async def do_work(self, work: Work) -> WorkResult: logger.debug("Calling `on_activity_start` done") try: - logger.debug("Calling `work`...") + logger.debug(f"Calling `{work}`...") work_result = await work(work_context) except Exception as e: - logger.debug(f"Calling `work` done with exception `{e}`") + logger.debug(f"Calling `{work}` done with exception `{e}`") work_result = WorkResult(exception=e) else: if isinstance(work_result, WorkResult): - logger.debug(f"Calling `work` done with explicit result `{work_result}`") + logger.debug(f"Calling `{work}` done with explicit result `{work_result}`") else: - logger.debug(f"Calling `work` done with implicit result `{work_result}`") + logger.debug(f"Calling `{work}` done with implicit result `{work_result}`") work_result = WorkResult(result=work_result) if self._on_activity_stop: - logger.debug("Calling `on_activity_stop`...") + logger.debug(f"Calling `on_activity_stop` on activity `{activity}`...") await self._on_activity_stop(work_context) - logger.debug("Calling `on_activity_stop` done") + logger.debug(f"Calling `on_activity_stop` on activity `{activity}` done") - if not activity.destroyed: + if activity.destroyed: + logger.info(f"Activity `{activity}` destroyed") + else: logger.warning( "SingleUseActivityManager expects that activity will be terminated" " after its work is finished. Looks like you forgot calling" " `context.terminate()` in custom `on_activity_end` callback." ) - logger.info(f"Doing work done {work} on activity {activity.id}") + logger.debug(f"Doing work `{work}` done") return work_result diff --git a/golem_core/managers/agreement/single_use.py b/golem_core/managers/agreement/single_use.py index b9070c32..f92007bb 100644 --- a/golem_core/managers/agreement/single_use.py +++ b/golem_core/managers/agreement/single_use.py @@ -15,7 +15,7 @@ def __init__(self, golem: GolemNode, get_proposal: Callable[[], Awaitable[Propos self._event_bus = golem.event_bus async def get_agreement(self) -> Agreement: - logger.info("Getting agreement...") + logger.debug("Getting agreement...") while True: logger.debug("Getting proposal...") @@ -25,7 +25,7 @@ async def get_agreement(self) -> Agreement: logger.debug(f"Getting proposal done with {proposal}") try: - logger.info("Creating agreement...") + logger.debug("Creating agreement...") agreement = await proposal.create_agreement() @@ -37,22 +37,28 @@ async def get_agreement(self) -> Agreement: await agreement.wait_for_approval() except Exception as e: - logger.debug(f"Creating agreement failed with {e}. Retrying...") + logger.debug(f"Creating agreement failed with `{e}`. Retrying...") else: - logger.info(f"Creating agreement done {agreement.id}") + logger.debug(f"Creating agreement done with `{agreement}`") + logger.info(f"Agreement `{agreement}` created") # TODO: Support removing callback on resource close - self._event_bus.resource_listen( - self._on_agreement_released, [AgreementReleased], [Agreement], [agreement.id] + await self._event_bus.on_once( + AgreementReleased, + self._terminate_agreement, + lambda e: e.resource.id == agreement.id, ) - logger.info(f"Getting agreement done {agreement.id}") + logger.debug(f"Getting agreement done with `{agreement}`") + return agreement - async def _on_agreement_released(self, event: AgreementReleased) -> None: - logger.debug("Calling `_on_agreement_released`...") + async def _terminate_agreement(self, event: AgreementReleased) -> None: + logger.debug("Calling `_terminate_agreement`...") agreement: Agreement = event.resource await agreement.terminate() - logger.debug("Calling `_on_agreement_released` done") + logger.debug("Calling `_terminate_agreement` done") + + logger.info(f"Agreement `{agreement}` closed") diff --git a/golem_core/managers/negotiation/accept_all.py b/golem_core/managers/negotiation/accept_all.py index a29822c2..4d13b29c 100644 --- a/golem_core/managers/negotiation/accept_all.py +++ b/golem_core/managers/negotiation/accept_all.py @@ -22,20 +22,27 @@ def __init__( self._eligible_proposals: asyncio.Queue[Proposal] = asyncio.Queue() async def get_proposal(self) -> Proposal: - logger.info("Getting proposal...") + logger.debug("Getting proposal...") + proposal = await self._eligible_proposals.get() - logger.info(f"Getting proposal done {proposal.id}") + + logger.debug(f"Getting proposal done with `{proposal.id}`") + return proposal async def start_negotiation(self, payload: Payload) -> None: logger.debug("Starting negotiations...") + self._negotiations.append(asyncio.create_task(self._negotiate_task(payload))) + logger.debug("Starting negotiations done") async def stop_negotiation(self) -> None: logger.debug("Stopping negotiations...") + for task in self._negotiations: task.cancel() + logger.debug("Stopping negotiations done") async def _negotiate_task(self, payload: Payload) -> None: @@ -45,7 +52,8 @@ async def _negotiate_task(self, payload: Payload) -> None: await self._eligible_proposals.put(proposal) async def _build_demand(self, allocation: Allocation, payload: Payload) -> Demand: - logger.info("Creating demand...") + logger.debug("Creating demand...") + demand_builder = DemandBuilder() await demand_builder.add( @@ -67,13 +75,15 @@ async def _build_demand(self, allocation: Allocation, payload: Payload) -> Deman demand = await demand_builder.create_demand(self._golem) demand.start_collecting_events() - logger.info(f"Creating demand done {demand.id}") + + logger.debug(f"Creating demand done with `{demand.id}`") + return demand async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: try: async for initial in demand.initial_proposals(): - logger.info(f"Negotiating initial proposal {initial.id}") + logger.debug(f"Negotiating initial proposal `{initial.id}`...") try: demand_proposal = await initial.respond() except Exception as err: @@ -88,7 +98,7 @@ async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: except StopAsyncIteration: continue - logger.info(f"Negotiating initial proposal done {initial.id}") + logger.debug(f"Negotiating initial proposal `{initial.id}` done") yield offer_proposal finally: self._golem.add_autoclose_resource(demand) diff --git a/golem_core/managers/payment/default.py b/golem_core/managers/payment/default.py index d9eaa697..5d11e636 100644 --- a/golem_core/managers/payment/default.py +++ b/golem_core/managers/payment/default.py @@ -2,11 +2,12 @@ from datetime import datetime, timedelta from typing import TYPE_CHECKING, Set -from golem_core.core.payment_api import DebitNote, Invoice -from golem_core.core.resources import NewResource +from golem_core.core.payment_api import NewDebitNote +from golem_core.core.payment_api.events import NewInvoice if TYPE_CHECKING: from golem_core.core.golem_node import GolemNode + from golem_core.core.market_api import NewAgreement from golem_core.core.payment_api import Allocation @@ -40,33 +41,34 @@ def __init__(self, node: "GolemNode", allocation: "Allocation"): """ # FIXME: Resolve local import due to cyclic imports - from golem_core.core.market_api.resources.agreement import Agreement + from golem_core.core.market_api import Agreement + self._node = node self.allocation = allocation self._agreements: Set[Agreement] = set() - node.event_bus.resource_listen(self.on_agreement, [NewResource], [Agreement]) - node.event_bus.resource_listen(self.on_invoice, [NewResource], [Invoice]) - node.event_bus.resource_listen(self.on_debit_note, [NewResource], [DebitNote]) + async def start(self): + # FIXME: Add event_bus.off - async def on_agreement(self, event: NewResource) -> None: + await self._node.event_bus.on(NewAgreement, self.on_agreement) + await self._node.event_bus.on(NewInvoice, self.on_invoice) + await self._node.event_bus.on(NewDebitNote, self.on_debit_note) + + async def on_agreement(self, event: "NewAgreement") -> None: # FIXME: Resolve local import due to cyclic imports - from golem_core.core.market_api.resources.agreement import Agreement - agreement = event.resource - assert isinstance(agreement, Agreement) - self._agreements.add(agreement) + self._agreements.add(event.resource) - async def on_invoice(self, event: NewResource) -> None: + async def on_invoice(self, event: NewInvoice) -> None: invoice = event.resource - assert isinstance(invoice, Invoice) + if (await invoice.get_data(force=True)).status == "RECEIVED": await invoice.accept_full(self.allocation) await invoice.get_data(force=True) - async def on_debit_note(self, event: NewResource) -> None: + async def on_debit_note(self, event: NewDebitNote) -> None: debit_note = event.resource - assert isinstance(debit_note, DebitNote) + if (await debit_note.get_data(force=True)).status == "RECEIVED": await debit_note.accept_full(self.allocation) await debit_note.get_data(force=True) diff --git a/golem_core/managers/payment/pay_all.py b/golem_core/managers/payment/pay_all.py index 52a267ac..4f638cce 100644 --- a/golem_core/managers/payment/pay_all.py +++ b/golem_core/managers/payment/pay_all.py @@ -4,11 +4,11 @@ from typing import Optional from golem_core.core.golem_node.golem_node import PAYMENT_DRIVER, PAYMENT_NETWORK, GolemNode -from golem_core.core.market_api.resources.agreement import Agreement +from golem_core.core.market_api import AgreementClosed, NewAgreement +from golem_core.core.payment_api.events import NewDebitNote, NewInvoice from golem_core.core.payment_api.resources.allocation import Allocation from golem_core.core.payment_api.resources.debit_note import DebitNote from golem_core.core.payment_api.resources.invoice import Invoice -from golem_core.core.resources.events import NewResource, ResourceClosed from golem_core.managers.base import PaymentManager logger = logging.getLogger(__name__) @@ -29,33 +29,39 @@ def __init__( self._allocation: Optional[Allocation] = None - self._golem.event_bus.resource_listen(self._on_invoice_received, [NewResource], [Invoice]) - self._golem.event_bus.resource_listen( - self._on_debit_note_received, [NewResource], [DebitNote] - ) + self._opened_agreements_count = 0 + self._closed_agreements_count = 0 + self._payed_invoices_count = 0 - self._opened_agreements_count: int = 0 - self._closed_agreements_count: int = 0 - self._payed_invoices_count: int = 0 - self._golem.event_bus.resource_listen(self._on_new_agreement, [NewResource], [Agreement]) - self._golem.event_bus.resource_listen( - self._on_agreement_closed, [ResourceClosed], [Agreement] - ) + async def start(self) -> None: + # TODO: Add stop with event_bus.off() + + await self._golem.event_bus.on(NewInvoice, self._pay_invoice_if_received) + await self._golem.event_bus.on(NewDebitNote, self._pay_debit_note_if_received) + + await self._golem.event_bus.on(NewAgreement, self._increment_opened_agreements) + await self._golem.event_bus.on(AgreementClosed, self._increment_closed_agreements) async def get_allocation(self) -> "Allocation": - logger.info("Getting allocation...") + logger.debug("Getting allocation...") + if self._allocation is None: - logger.info("Creating allocation...") + logger.debug("Creating allocation...") + self._allocation = await Allocation.create_any_account( self._golem, Decimal(self._budget), self._network, self._driver ) self._golem.add_autoclose_resource(self._allocation) - logger.info(f"Creating allocation done {self._allocation.id}") - logger.info(f"Getting allocation done {self._allocation.id}") + + logger.debug(f"Creating allocation done with `{self._allocation.id}`") + + logger.debug(f"Getting allocation done with `{self._allocation.id}`") + return self._allocation async def wait_for_invoices(self): logger.info("Waiting for invoices...") + for _ in range(60): await asyncio.sleep(1) if ( @@ -63,34 +69,47 @@ async def wait_for_invoices(self): == self._closed_agreements_count == self._payed_invoices_count ): - break - logger.info("Waiting for invoices done") + logger.info("Waiting for invoices done with all paid") + return + + # TODO: Add list of agreements without payment + logger.warning("Waiting for invoices failed with timeout!") - async def _on_new_agreement(self, agreement_event: NewResource): + async def _increment_opened_agreements(self, event: NewAgreement): self._opened_agreements_count += 1 - async def _on_agreement_closed(self, agreement_event: ResourceClosed): + async def _increment_closed_agreements(self, event: AgreementClosed): self._closed_agreements_count += 1 - async def _on_invoice_received(self, invoice_event: NewResource) -> None: - logger.info("Received invoice...") - invoice = invoice_event.resource + async def _pay_invoice_if_received(self, event: NewInvoice) -> None: + logger.debug("Received invoice") + + invoice = event.resource assert isinstance(invoice, Invoice) + if (await invoice.get_data(force=True)).status == "RECEIVED": - logger.info(f"Accepting invoice {invoice.id}") + logger.debug(f"Accepting invoice `{invoice.id}`...") + assert self._allocation is not None # TODO think of a better way await invoice.accept_full(self._allocation) await invoice.get_data(force=True) self._payed_invoices_count += 1 - logger.info(f"Accepting invoice done {invoice.id}") - async def _on_debit_note_received(self, debit_note_event: NewResource) -> None: - logger.info("Received debit note...") - debit_note = debit_note_event.resource + logger.debug(f"Accepting invoice `{invoice.id}` done") + logger.info(f"Invoice `{invoice.id}` accepted") + + async def _pay_debit_note_if_received(self, event: NewDebitNote) -> None: + logger.debug("Received debit note") + + debit_note = event.resource assert isinstance(debit_note, DebitNote) + if (await debit_note.get_data(force=True)).status == "RECEIVED": - logger.info(f"Accepting debit note {debit_note.id}") + logger.debug(f"Accepting DebitNote `{debit_note.id}`...") + assert self._allocation is not None # TODO think of a better way await debit_note.accept_full(self._allocation) await debit_note.get_data(force=True) - logger.info(f"Accepting debit note done {debit_note.id}") + + logger.debug(f"Accepting DebitNote `{debit_note.id}` done") + logger.debug(f"DebitNote `{debit_note.id}` accepted") diff --git a/golem_core/managers/proposal/stack.py b/golem_core/managers/proposal/stack.py index dc780e1f..500f03d3 100644 --- a/golem_core/managers/proposal/stack.py +++ b/golem_core/managers/proposal/stack.py @@ -17,23 +17,32 @@ def __init__(self, golem: GolemNode, get_proposal) -> None: async def start_consuming_proposals(self) -> None: logger.debug("Starting consuming proposals...") + self._tasks.append(asyncio.create_task(self._consume_proposals())) + logger.debug("Starting consuming proposals done") async def stop_consuming_proposals(self) -> None: logger.debug("Stopping consuming proposals...") + for task in self._tasks: task.cancel() + logger.debug("Stopping consuming proposals done") async def _consume_proposals(self) -> None: while True: proposal = await self._get_proposal() - logger.debug("Adding proposal to the stack") + + logger.debug(f"Adding proposal `{proposal}` on the stack") + await self._proposals.put(proposal) async def get_proposal(self) -> Proposal: - logger.info("Getting proposals...") + logger.debug("Getting proposal...") + proposal = await self._proposals.get() - logger.info(f"Getting proposals done {proposal.id}") + + logger.debug(f"Getting proposal done with `{proposal.id}`") + return proposal diff --git a/golem_core/managers/work/sequential.py b/golem_core/managers/work/sequential.py index 9aada90b..56311475 100644 --- a/golem_core/managers/work/sequential.py +++ b/golem_core/managers/work/sequential.py @@ -27,18 +27,19 @@ def _apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkC return result async def do_work(self, work: Work) -> WorkResult: - logger.info(f"Running work {work}") + logger.debug(f"Running work {work}") decorated_do_work = self._apply_work_decorators(self._do_work, work) result = await decorated_do_work(work) - logger.info(f"Running work done {work}") + logger.debug(f"Running work done {work}") + logger.info(f"Work `{work}` completed") return result async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: - logger.info(f"Running work sequence {work_list}") + logger.debug(f"Running work sequence {work_list}") results = [] @@ -49,6 +50,6 @@ async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: logger.debug(f"Doing work sequence #{i} done") - logger.info(f"Running work sequence done {work_list}") + logger.debug(f"Running work sequence done {work_list}") return results diff --git a/golem_core/utils/logging.py b/golem_core/utils/logging.py index 87dd7cac..ae783a3c 100644 --- a/golem_core/utils/logging.py +++ b/golem_core/utils/logging.py @@ -25,6 +25,15 @@ "golem_core": { "level": "INFO", }, + "golem_core.managers": { + "level": "INFO", + }, + "golem_core.managers.negotiation": { + "level": "INFO", + }, + "golem_core.managers.proposal": { + "level": "INFO", + }, }, } diff --git a/tests/integration/test_app_session_id.py b/tests/integration/test_app_session_id.py index 6e28adc5..a91a5b27 100644 --- a/tests/integration/test_app_session_id.py +++ b/tests/integration/test_app_session_id.py @@ -26,7 +26,7 @@ async def save_event(event: ResourceEvent) -> None: events.append(event) golem = GolemNode(**kwargs) - golem.event_bus.resource_listen(save_event) + await golem.event_bus.on(ResourceEvent, save_event) async with golem: other_golem = GolemNode(app_session_id="0") diff --git a/tests/unit/test_event_bus.py b/tests/unit/test_event_bus.py new file mode 100644 index 00000000..77a4c747 --- /dev/null +++ b/tests/unit/test_event_bus.py @@ -0,0 +1,171 @@ +import logging +import logging.config + +import pytest + +from golem_core.core.events.base import Event, EventBusError +from golem_core.core.events.event_bus import InMemoryEventBus +from golem_core.utils.logging import DEFAULT_LOGGING + + +class ExampleEvent(Event): + pass + + +class ParamExampleEvent(Event): + def __init__(self, val): + self.val = val + + +@pytest.fixture(autouse=True) +async def logs(): + logging.config.dictConfig(DEFAULT_LOGGING) + + +@pytest.fixture +async def event_bus(caplog): + event_bus = InMemoryEventBus() + + await event_bus.start() + + yield event_bus + + if event_bus.is_started(): + await event_bus.stop() + + +async def test_start_stop(): + event_bus = InMemoryEventBus() + + await event_bus.start() + + assert event_bus.is_started() + + with pytest.raises(EventBusError, match="already started"): + await event_bus.start() + + await event_bus.stop() + + assert not event_bus.is_started() + + with pytest.raises(EventBusError, match="not started"): + await event_bus.stop() + + +async def test_on_off(mocker): + event_bus = InMemoryEventBus() + + callback_mock = mocker.Mock() + + callback_handler = await event_bus.on(ExampleEvent, callback_mock) + + await event_bus.off(callback_handler) + + with pytest.raises(EventBusError, match="callback handler is not found"): + await event_bus.off(callback_handler) + + +async def test_emit_raises_while_not_started(): + event_bus = InMemoryEventBus() + + assert not event_bus.is_started() + + with pytest.raises(EventBusError, match="not started"): + await event_bus.emit(ExampleEvent()) + + +async def test_emit_multiple(mocker, event_bus): + callback_mock = mocker.Mock() + + await event_bus.on(ExampleEvent, callback_mock) + + event1 = ExampleEvent() + event2 = ExampleEvent() + event3 = ExampleEvent() + + await event_bus.emit(event1) + await event_bus.emit(event2) + await event_bus.emit(event3) + + await event_bus.stop() # Waits for all callbacks to be called + + assert callback_mock.mock_calls == [ + mocker.call(event1), + mocker.call(event2), + mocker.call(event3), + ] + + +async def test_emit_once(mocker, event_bus): + callback_mock = mocker.Mock() + + callback_handler = await event_bus.on_once(ExampleEvent, callback_mock) + + event1 = ExampleEvent() + event2 = ExampleEvent() + event3 = ExampleEvent() + + await event_bus.emit(event1) + await event_bus.emit(event2) + await event_bus.emit(event3) + + await event_bus.stop() # Waits for all callbacks to be called + + assert callback_mock.mock_calls == [ + mocker.call(event1), + ] + + with pytest.raises(EventBusError, match="callback handler is not found"): + await event_bus.off(callback_handler) + + +async def test_emit_with_filter(mocker, event_bus): + callback_mock = mocker.Mock() + callback_mock_once = mocker.Mock() + + await event_bus.on(ParamExampleEvent, callback_mock, lambda e: e.val == 2) + await event_bus.on_once(ParamExampleEvent, callback_mock_once, lambda e: e.val == 3) + + event1 = ParamExampleEvent(1) + event2 = ParamExampleEvent(2) + event3 = ParamExampleEvent(3) + + await event_bus.emit(event1) + await event_bus.emit(event2) + await event_bus.emit(event3) + + await event_bus.stop() # Waits for all callbacks to be called + + assert callback_mock.mock_calls == [ + mocker.call(event2), + ] + + assert callback_mock_once.mock_calls == [ + mocker.call(event3), + ] + + +async def test_emit_with_errors(mocker, caplog, event_bus): + callback_mock = mocker.Mock(side_effect=Exception) + + await event_bus.on(ExampleEvent, callback_mock) + await event_bus.on(ParamExampleEvent, callback_mock, lambda e: e.not_existing) + + event1 = ExampleEvent() + event2 = ParamExampleEvent(2) + + caplog.at_level(logging.ERROR) + + caplog.clear() + await event_bus.emit(event1) + await event_bus.stop() # Waits for all callbacks to be called + + assert "callback while handling" in caplog.text + + await event_bus.start() + + caplog.clear() + await event_bus.emit(event2) + await event_bus.stop() # Waits for all callbacks to be called + + assert "filter function while handling" in caplog.text From c9ae2d14445d2ce6baa8f918480d87b12e7ac853 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 1 Jun 2023 11:07:19 +0200 Subject: [PATCH 039/123] Added proposal info log --- golem_core/managers/proposal/stack.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/golem_core/managers/proposal/stack.py b/golem_core/managers/proposal/stack.py index 500f03d3..959f3cb8 100644 --- a/golem_core/managers/proposal/stack.py +++ b/golem_core/managers/proposal/stack.py @@ -45,4 +45,6 @@ async def get_proposal(self) -> Proposal: logger.debug(f"Getting proposal done with `{proposal.id}`") + logger.info(f"Proposal `{proposal}` picked") + return proposal From 22eaa4ca5346a8c9c625c941e16dbe00cba714a5 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 5 Jun 2023 12:32:28 +0200 Subject: [PATCH 040/123] Format --- examples/managers/basic_composition.py | 6 ++++++ golem_core/core/payment_api/resources/debit_note.py | 3 +-- golem_core/managers/negotiation/accept_all.py | 1 - 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index d3598599..5cace16d 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -10,6 +10,10 @@ from golem_core.managers.negotiation import AcceptAllNegotiationManager from golem_core.managers.payment.pay_all import PayAllPaymentManager from golem_core.managers.proposal import StackProposalManager +from golem_core.managers.work.decorators import ( + redundancy_cancel_others_on_first_done, + work_decorator, +) from golem_core.managers.work.sequential import SequentialWorkManager from golem_core.utils.logging import DEFAULT_LOGGING @@ -23,6 +27,8 @@ async def commands_work_example(context: WorkContext) -> str: return result +# @work_decorator(redundancy_cancel_others_on_first_done(size=2)) +@work_decorator(redundancy_cancel_others_on_first_done(size=2)) async def batch_work_example(context: WorkContext): batch = await context.create_batch() batch.run("echo 'hello batch'") diff --git a/golem_core/core/payment_api/resources/debit_note.py b/golem_core/core/payment_api/resources/debit_note.py index e7f7a7f1..6159328e 100644 --- a/golem_core/core/payment_api/resources/debit_note.py +++ b/golem_core/core/payment_api/resources/debit_note.py @@ -10,9 +10,8 @@ from golem_core.core.resources.base import TModel if TYPE_CHECKING: - from golem_core.core.golem_node import GolemNode - from golem_core.core.activity_api import Activity # noqa + from golem_core.core.golem_node import GolemNode class DebitNote(Resource[RequestorApi, models.DebitNote, "Activity", _NULL, _NULL]): diff --git a/golem_core/managers/negotiation/accept_all.py b/golem_core/managers/negotiation/accept_all.py index 4d13b29c..2e3341b5 100644 --- a/golem_core/managers/negotiation/accept_all.py +++ b/golem_core/managers/negotiation/accept_all.py @@ -93,7 +93,6 @@ async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: continue try: - # TODO IDK how to call `confirm` on a proposal in golem-core offer_proposal = await demand_proposal.responses().__anext__() except StopAsyncIteration: continue From 5c5695243278fb4d134e0da9b169c6bfd67f4620 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 5 Jun 2023 12:43:18 +0200 Subject: [PATCH 041/123] Add context managers to managers --- examples/managers/basic_composition.py | 29 +++++++++---------- golem_core/managers/activity/single_use.py | 6 ++++ golem_core/managers/agreement/single_use.py | 6 ++++ golem_core/managers/negotiation/accept_all.py | 17 +++++++++-- golem_core/managers/payment/pay_all.py | 7 +++++ golem_core/managers/proposal/stack.py | 7 +++++ golem_core/managers/work/sequential.py | 6 ++++ 7 files changed, 59 insertions(+), 19 deletions(-) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index 5cace16d..7117b8e8 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -49,25 +49,22 @@ async def main(): batch_work_example, ] - async with GolemNode() as golem: - payment_manager = PayAllPaymentManager(golem, budget=1.0) - negotiation_manager = AcceptAllNegotiationManager(golem, payment_manager.get_allocation) - proposal_manager = StackProposalManager(golem, negotiation_manager.get_proposal) - agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) - activity_manager = SingleUseActivityManager(golem, agreement_manager.get_agreement) - work_manager = SequentialWorkManager(golem, activity_manager.do_work) - - await payment_manager.start() - await negotiation_manager.start_negotiation(payload) - await proposal_manager.start_consuming_proposals() - + async with GolemNode() as golem, PayAllPaymentManager( + golem, budget=1.0 + ) as payment_manager, AcceptAllNegotiationManager( + golem, payment_manager.get_allocation, payload + ) as negotiation_manager, StackProposalManager( + golem, negotiation_manager.get_proposal + ) as proposal_manager, SingleUseAgreementManager( + golem, proposal_manager.get_proposal + ) as agreement_manager, SingleUseActivityManager( + golem, agreement_manager.get_agreement + ) as activity_manager, SequentialWorkManager( + golem, activity_manager.do_work + ) as work_manager: results: List[WorkResult] = await work_manager.do_work_list(work_list) print(f"\nWORK MANAGER RESULTS:{[result.result for result in results]}\n") - await proposal_manager.stop_consuming_proposals() - await negotiation_manager.stop_negotiation() - await payment_manager.wait_for_invoices() - if __name__ == "__main__": asyncio.run(main()) diff --git a/golem_core/managers/activity/single_use.py b/golem_core/managers/activity/single_use.py index 8cc719d4..44d576c1 100644 --- a/golem_core/managers/activity/single_use.py +++ b/golem_core/managers/activity/single_use.py @@ -32,6 +32,12 @@ def __init__( self._on_activity_start = on_activity_start self._on_activity_stop = on_activity_stop + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + pass + @asynccontextmanager async def _prepare_activity(self) -> Activity: while True: diff --git a/golem_core/managers/agreement/single_use.py b/golem_core/managers/agreement/single_use.py index f92007bb..185a599d 100644 --- a/golem_core/managers/agreement/single_use.py +++ b/golem_core/managers/agreement/single_use.py @@ -14,6 +14,12 @@ def __init__(self, golem: GolemNode, get_proposal: Callable[[], Awaitable[Propos self._get_proposal = get_proposal self._event_bus = golem.event_bus + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + pass + async def get_agreement(self) -> Agreement: logger.debug("Getting agreement...") diff --git a/golem_core/managers/negotiation/accept_all.py b/golem_core/managers/negotiation/accept_all.py index 2e3341b5..036d3e74 100644 --- a/golem_core/managers/negotiation/accept_all.py +++ b/golem_core/managers/negotiation/accept_all.py @@ -14,13 +14,24 @@ class AcceptAllNegotiationManager(ProposalNegotiationManager): def __init__( - self, golem: GolemNode, get_allocation: Callable[[], Awaitable[Allocation]] + self, + golem: GolemNode, + get_allocation: Callable[[], Awaitable[Allocation]], + payload: Payload, ) -> None: self._golem = golem self._get_allocation = get_allocation + self._payload = payload self._negotiations: List[asyncio.Task] = [] self._eligible_proposals: asyncio.Queue[Proposal] = asyncio.Queue() + async def __aenter__(self): + await self.start_negotiation() + return self + + async def __aexit__(self, exc_type, exc, tb): + await self.stop_negotiation() + async def get_proposal(self) -> Proposal: logger.debug("Getting proposal...") @@ -30,10 +41,10 @@ async def get_proposal(self) -> Proposal: return proposal - async def start_negotiation(self, payload: Payload) -> None: + async def start_negotiation(self) -> None: logger.debug("Starting negotiations...") - self._negotiations.append(asyncio.create_task(self._negotiate_task(payload))) + self._negotiations.append(asyncio.create_task(self._negotiate_task(self._payload))) logger.debug("Starting negotiations done") diff --git a/golem_core/managers/payment/pay_all.py b/golem_core/managers/payment/pay_all.py index 4f638cce..a75c9e67 100644 --- a/golem_core/managers/payment/pay_all.py +++ b/golem_core/managers/payment/pay_all.py @@ -33,6 +33,13 @@ def __init__( self._closed_agreements_count = 0 self._payed_invoices_count = 0 + async def __aenter__(self): + await self.start() + return self + + async def __aexit__(self, exc_type, exc, tb): + await self.wait_for_invoices() + async def start(self) -> None: # TODO: Add stop with event_bus.off() diff --git a/golem_core/managers/proposal/stack.py b/golem_core/managers/proposal/stack.py index 959f3cb8..2ea7d69d 100644 --- a/golem_core/managers/proposal/stack.py +++ b/golem_core/managers/proposal/stack.py @@ -15,6 +15,13 @@ def __init__(self, golem: GolemNode, get_proposal) -> None: self._proposals: asyncio.Queue[Proposal] = asyncio.Queue() self._tasks: List[asyncio.Task] = [] + async def __aenter__(self): + await self.start_consuming_proposals() + return self + + async def __aexit__(self, exc_type, exc, tb): + await self.stop_consuming_proposals() + async def start_consuming_proposals(self) -> None: logger.debug("Starting consuming proposals...") diff --git a/golem_core/managers/work/sequential.py b/golem_core/managers/work/sequential.py index 56311475..2c248646 100644 --- a/golem_core/managers/work/sequential.py +++ b/golem_core/managers/work/sequential.py @@ -12,6 +12,12 @@ class SequentialWorkManager: def __init__(self, golem: GolemNode, do_work: DoWorkCallable): self._do_work = do_work + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + pass + def _apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: logger.debug(f"Applying decorators on `{work}`...") From 3bdd45530e0789d7273f37b91b0746e087aebd50 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 5 Jun 2023 12:47:01 +0200 Subject: [PATCH 042/123] Improve CM formating --- examples/managers/basic_composition.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index 7117b8e8..6c87f458 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -49,19 +49,17 @@ async def main(): batch_work_example, ] - async with GolemNode() as golem, PayAllPaymentManager( - golem, budget=1.0 - ) as payment_manager, AcceptAllNegotiationManager( - golem, payment_manager.get_allocation, payload - ) as negotiation_manager, StackProposalManager( - golem, negotiation_manager.get_proposal - ) as proposal_manager, SingleUseAgreementManager( - golem, proposal_manager.get_proposal - ) as agreement_manager, SingleUseActivityManager( - golem, agreement_manager.get_agreement - ) as activity_manager, SequentialWorkManager( - golem, activity_manager.do_work - ) as work_manager: + async with ( + GolemNode() as golem, + PayAllPaymentManager(golem, budget=1.0) as payment_manager, + AcceptAllNegotiationManager( + golem, payment_manager.get_allocation, payload + ) as negotiation_manager, + StackProposalManager(golem, negotiation_manager.get_proposal) as proposal_manager, + SingleUseAgreementManager(golem, proposal_manager.get_proposal) as agreement_manager, + SingleUseActivityManager(golem, agreement_manager.get_agreement) as activity_manager, + SequentialWorkManager(golem, activity_manager.do_work) as work_manager, + ): results: List[WorkResult] = await work_manager.do_work_list(work_list) print(f"\nWORK MANAGER RESULTS:{[result.result for result in results]}\n") From 01e3084961fa449eaef7d831c0ea7103679ba68d Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 5 Jun 2023 12:55:29 +0200 Subject: [PATCH 043/123] Fix formatting for 3.8 --- examples/managers/basic_composition.py | 30 ++++++++++++-------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index 6c87f458..64d6671e 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -10,10 +10,6 @@ from golem_core.managers.negotiation import AcceptAllNegotiationManager from golem_core.managers.payment.pay_all import PayAllPaymentManager from golem_core.managers.proposal import StackProposalManager -from golem_core.managers.work.decorators import ( - redundancy_cancel_others_on_first_done, - work_decorator, -) from golem_core.managers.work.sequential import SequentialWorkManager from golem_core.utils.logging import DEFAULT_LOGGING @@ -28,7 +24,7 @@ async def commands_work_example(context: WorkContext) -> str: # @work_decorator(redundancy_cancel_others_on_first_done(size=2)) -@work_decorator(redundancy_cancel_others_on_first_done(size=2)) +# @work_decorator(redundancy_cancel_others_on_first_done(size=2)) async def batch_work_example(context: WorkContext): batch = await context.create_batch() batch.run("echo 'hello batch'") @@ -49,17 +45,19 @@ async def main(): batch_work_example, ] - async with ( - GolemNode() as golem, - PayAllPaymentManager(golem, budget=1.0) as payment_manager, - AcceptAllNegotiationManager( - golem, payment_manager.get_allocation, payload - ) as negotiation_manager, - StackProposalManager(golem, negotiation_manager.get_proposal) as proposal_manager, - SingleUseAgreementManager(golem, proposal_manager.get_proposal) as agreement_manager, - SingleUseActivityManager(golem, agreement_manager.get_agreement) as activity_manager, - SequentialWorkManager(golem, activity_manager.do_work) as work_manager, - ): + async with GolemNode() as golem, PayAllPaymentManager( + golem, budget=1.0 + ) as payment_manager, AcceptAllNegotiationManager( + golem, payment_manager.get_allocation, payload + ) as negotiation_manager, StackProposalManager( + golem, negotiation_manager.get_proposal + ) as proposal_manager, SingleUseAgreementManager( + golem, proposal_manager.get_proposal + ) as agreement_manager, SingleUseActivityManager( + golem, agreement_manager.get_agreement + ) as activity_manager, SequentialWorkManager( + golem, activity_manager.do_work + ) as work_manager: results: List[WorkResult] = await work_manager.do_work_list(work_list) print(f"\nWORK MANAGER RESULTS:{[result.result for result in results]}\n") From a4683ae309b5415f3edca66dd706321b1aaa7aa9 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 5 Jun 2023 13:38:12 +0200 Subject: [PATCH 044/123] Even better formatting of context manager --- examples/managers/basic_composition.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index 64d6671e..9e4baa39 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -45,19 +45,18 @@ async def main(): batch_work_example, ] - async with GolemNode() as golem, PayAllPaymentManager( - golem, budget=1.0 - ) as payment_manager, AcceptAllNegotiationManager( + golem = GolemNode() + + payment_manager = PayAllPaymentManager(golem, budget=1.0) + negotiation_manager = AcceptAllNegotiationManager( golem, payment_manager.get_allocation, payload - ) as negotiation_manager, StackProposalManager( - golem, negotiation_manager.get_proposal - ) as proposal_manager, SingleUseAgreementManager( - golem, proposal_manager.get_proposal - ) as agreement_manager, SingleUseActivityManager( - golem, agreement_manager.get_agreement - ) as activity_manager, SequentialWorkManager( - golem, activity_manager.do_work - ) as work_manager: + ) + proposal_manager = StackProposalManager(golem, negotiation_manager.get_proposal) + agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) + activity_manager = SingleUseActivityManager(golem, agreement_manager.get_agreement) + work_manager = SequentialWorkManager(golem, activity_manager.do_work) + + async with golem, payment_manager, negotiation_manager, proposal_manager: results: List[WorkResult] = await work_manager.do_work_list(work_list) print(f"\nWORK MANAGER RESULTS:{[result.result for result in results]}\n") From 37359f7b283abcc87735c5d6a1b4786b4ccf1a02 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 5 Jun 2023 13:44:40 +0200 Subject: [PATCH 045/123] improve CM implementation --- examples/managers/basic_composition.py | 2 +- golem_core/managers/activity/single_use.py | 6 ------ golem_core/managers/agreement/single_use.py | 6 ------ golem_core/managers/base.py | 13 ++++++++++++- golem_core/managers/negotiation/accept_all.py | 11 ++--------- golem_core/managers/payment/pay_all.py | 10 +++------- golem_core/managers/proposal/stack.py | 11 ++--------- golem_core/managers/work/sequential.py | 10 ++-------- 8 files changed, 22 insertions(+), 47 deletions(-) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index 9e4baa39..b8ef31d8 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -46,7 +46,7 @@ async def main(): ] golem = GolemNode() - + payment_manager = PayAllPaymentManager(golem, budget=1.0) negotiation_manager = AcceptAllNegotiationManager( golem, payment_manager.get_allocation, payload diff --git a/golem_core/managers/activity/single_use.py b/golem_core/managers/activity/single_use.py index 44d576c1..8cc719d4 100644 --- a/golem_core/managers/activity/single_use.py +++ b/golem_core/managers/activity/single_use.py @@ -32,12 +32,6 @@ def __init__( self._on_activity_start = on_activity_start self._on_activity_stop = on_activity_stop - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc, tb): - pass - @asynccontextmanager async def _prepare_activity(self) -> Activity: while True: diff --git a/golem_core/managers/agreement/single_use.py b/golem_core/managers/agreement/single_use.py index 185a599d..f92007bb 100644 --- a/golem_core/managers/agreement/single_use.py +++ b/golem_core/managers/agreement/single_use.py @@ -14,12 +14,6 @@ def __init__(self, golem: GolemNode, get_proposal: Callable[[], Awaitable[Propos self._get_proposal = get_proposal self._event_bus = golem.event_bus - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc, tb): - pass - async def get_agreement(self) -> Agreement: logger.debug("Getting agreement...") diff --git a/golem_core/managers/base.py b/golem_core/managers/base.py index 55b96883..6f95d7d9 100644 --- a/golem_core/managers/base.py +++ b/golem_core/managers/base.py @@ -88,7 +88,18 @@ class ManagerEvent(ResourceEvent, ABC): class Manager(ABC): - ... + async def __aenter__(self): + await self.start() + return self + + async def __aexit__(self, exc_type, exc, tb): + await self.stop() + + async def start(self): + ... + + async def stop(self): + ... class PaymentManager(Manager, ABC): diff --git a/golem_core/managers/negotiation/accept_all.py b/golem_core/managers/negotiation/accept_all.py index 036d3e74..0855d0ad 100644 --- a/golem_core/managers/negotiation/accept_all.py +++ b/golem_core/managers/negotiation/accept_all.py @@ -25,13 +25,6 @@ def __init__( self._negotiations: List[asyncio.Task] = [] self._eligible_proposals: asyncio.Queue[Proposal] = asyncio.Queue() - async def __aenter__(self): - await self.start_negotiation() - return self - - async def __aexit__(self, exc_type, exc, tb): - await self.stop_negotiation() - async def get_proposal(self) -> Proposal: logger.debug("Getting proposal...") @@ -41,14 +34,14 @@ async def get_proposal(self) -> Proposal: return proposal - async def start_negotiation(self) -> None: + async def start(self) -> None: logger.debug("Starting negotiations...") self._negotiations.append(asyncio.create_task(self._negotiate_task(self._payload))) logger.debug("Starting negotiations done") - async def stop_negotiation(self) -> None: + async def stop(self) -> None: logger.debug("Stopping negotiations...") for task in self._negotiations: diff --git a/golem_core/managers/payment/pay_all.py b/golem_core/managers/payment/pay_all.py index a75c9e67..3e11e80a 100644 --- a/golem_core/managers/payment/pay_all.py +++ b/golem_core/managers/payment/pay_all.py @@ -33,13 +33,6 @@ def __init__( self._closed_agreements_count = 0 self._payed_invoices_count = 0 - async def __aenter__(self): - await self.start() - return self - - async def __aexit__(self, exc_type, exc, tb): - await self.wait_for_invoices() - async def start(self) -> None: # TODO: Add stop with event_bus.off() @@ -49,6 +42,9 @@ async def start(self) -> None: await self._golem.event_bus.on(NewAgreement, self._increment_opened_agreements) await self._golem.event_bus.on(AgreementClosed, self._increment_closed_agreements) + async def stop(self) -> None: + await self.wait_for_invoices() + async def get_allocation(self) -> "Allocation": logger.debug("Getting allocation...") diff --git a/golem_core/managers/proposal/stack.py b/golem_core/managers/proposal/stack.py index 2ea7d69d..63cc88e2 100644 --- a/golem_core/managers/proposal/stack.py +++ b/golem_core/managers/proposal/stack.py @@ -15,21 +15,14 @@ def __init__(self, golem: GolemNode, get_proposal) -> None: self._proposals: asyncio.Queue[Proposal] = asyncio.Queue() self._tasks: List[asyncio.Task] = [] - async def __aenter__(self): - await self.start_consuming_proposals() - return self - - async def __aexit__(self, exc_type, exc, tb): - await self.stop_consuming_proposals() - - async def start_consuming_proposals(self) -> None: + async def start(self) -> None: logger.debug("Starting consuming proposals...") self._tasks.append(asyncio.create_task(self._consume_proposals())) logger.debug("Starting consuming proposals done") - async def stop_consuming_proposals(self) -> None: + async def stop(self) -> None: logger.debug("Stopping consuming proposals...") for task in self._tasks: diff --git a/golem_core/managers/work/sequential.py b/golem_core/managers/work/sequential.py index 2c248646..efd3844e 100644 --- a/golem_core/managers/work/sequential.py +++ b/golem_core/managers/work/sequential.py @@ -3,21 +3,15 @@ from typing import List from golem_core.core.golem_node.golem_node import GolemNode -from golem_core.managers.base import DoWorkCallable, Work, WorkResult +from golem_core.managers.base import DoWorkCallable, Work, WorkManager, WorkResult logger = logging.getLogger(__name__) -class SequentialWorkManager: +class SequentialWorkManager(WorkManager): def __init__(self, golem: GolemNode, do_work: DoWorkCallable): self._do_work = do_work - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc, tb): - pass - def _apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: logger.debug(f"Applying decorators on `{work}`...") From ad1917c53386acf41e8ccaf6e6b41910fb65261a Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 5 Jun 2023 16:07:08 +0200 Subject: [PATCH 046/123] Fix work manager decorators --- examples/managers/basic_composition.py | 12 ++++++++++-- golem_core/managers/base.py | 4 ++-- golem_core/managers/work/decorators.py | 4 ++-- golem_core/managers/work/sequential.py | 3 +-- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index b8ef31d8..0c73906f 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -1,5 +1,6 @@ import asyncio import logging.config +from random import randint from typing import List from golem_core.core.golem_node.golem_node import GolemNode @@ -10,6 +11,11 @@ from golem_core.managers.negotiation import AcceptAllNegotiationManager from golem_core.managers.payment.pay_all import PayAllPaymentManager from golem_core.managers.proposal import StackProposalManager +from golem_core.managers.work.decorators import ( + redundancy_cancel_others_on_first_done, + retry, + work_decorator, +) from golem_core.managers.work.sequential import SequentialWorkManager from golem_core.utils.logging import DEFAULT_LOGGING @@ -23,9 +29,11 @@ async def commands_work_example(context: WorkContext) -> str: return result -# @work_decorator(redundancy_cancel_others_on_first_done(size=2)) -# @work_decorator(redundancy_cancel_others_on_first_done(size=2)) +@work_decorator(redundancy_cancel_others_on_first_done(size=2)) +@work_decorator(retry(tries=5)) async def batch_work_example(context: WorkContext): + if randint(0, 1): + raise Exception("Random fail") batch = await context.create_batch() batch.run("echo 'hello batch'") batch.run("echo 'bye batch'") diff --git a/golem_core/managers/base.py b/golem_core/managers/base.py index 6f95d7d9..c3b99346 100644 --- a/golem_core/managers/base.py +++ b/golem_core/managers/base.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, Awaitable, Callable, Dict, List, Optional, Union from golem_core.core.activity_api import Activity, Script, commands @@ -67,7 +67,7 @@ async def create_batch(self) -> Batch: class WorkResult: result: Optional[Any] = None exception: Optional[Exception] = None - extras: Optional[Dict] = None + extras: Dict = field(default_factory=dict) WorkDecorator = Callable[["DoWorkCallable"], "DoWorkCallable"] diff --git a/golem_core/managers/work/decorators.py b/golem_core/managers/work/decorators.py index 809875cb..fe8db5fe 100644 --- a/golem_core/managers/work/decorators.py +++ b/golem_core/managers/work/decorators.py @@ -16,7 +16,7 @@ def _work_decorator(work: Work): return _work_decorator -def retry(tries: int = 3): +def retry(tries: int): def _retry(do_work: DoWorkCallable) -> DoWorkCallable: @wraps(do_work) async def wrapper(work: Work) -> WorkResult: @@ -45,7 +45,7 @@ async def wrapper(work: Work) -> WorkResult: return _retry -def redundancy_cancel_others_on_first_done(size: int = 3): +def redundancy_cancel_others_on_first_done(size: int): def _redundancy(do_work: DoWorkCallable): @wraps(do_work) async def wrapper(work: Work) -> WorkResult: diff --git a/golem_core/managers/work/sequential.py b/golem_core/managers/work/sequential.py index efd3844e..10ad6edc 100644 --- a/golem_core/managers/work/sequential.py +++ b/golem_core/managers/work/sequential.py @@ -1,5 +1,4 @@ import logging -from functools import partial from typing import List from golem_core.core.golem_node.golem_node import GolemNode @@ -20,7 +19,7 @@ def _apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkC result = do_work for dec in work._work_decorators: - result = partial(dec, result) + result = dec(result) logger.debug(f"Applying decorators on `{work}` done") From 9d709f0267bf204d7c28c11fc5b2defdd4b72564 Mon Sep 17 00:00:00 2001 From: approxit Date: Mon, 5 Jun 2023 14:01:20 +0200 Subject: [PATCH 047/123] negotiation plugins draft --- .../core/market_api/resources/proposal.py | 6 +- golem_core/managers/base.py | 11 +- golem_core/managers/negotiation/accept_all.py | 115 +++++++++++++----- golem_core/managers/proposal/stack.py | 4 +- 4 files changed, 97 insertions(+), 39 deletions(-) diff --git a/golem_core/core/market_api/resources/proposal.py b/golem_core/core/market_api/resources/proposal.py index fbf28238..48c4a07b 100644 --- a/golem_core/core/market_api/resources/proposal.py +++ b/golem_core/core/market_api/resources/proposal.py @@ -143,7 +143,7 @@ async def reject(self, reason: str = "") -> None: ) @api_call_wrapper() - async def respond(self) -> "Proposal": + async def respond(self, properties, constraints) -> "Proposal": """Respond to a proposal with a counter-proposal. Invalid on our responses. @@ -154,7 +154,9 @@ async def respond(self) -> "Proposal": https://github.com/golemfactory/golem-core-python/issues/18 """ - data = await self._response_data() + data = models.DemandOfferBase( + properties=properties, constraints=constraints + ) new_proposal_id = await self.api.counter_proposal_demand( self.demand.id, self.id, data, _request_timeout=5 ) diff --git a/golem_core/managers/base.py b/golem_core/managers/base.py index c3b99346..0fc73391 100644 --- a/golem_core/managers/base.py +++ b/golem_core/managers/base.py @@ -3,9 +3,10 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional, Union from golem_core.core.activity_api import Activity, Script, commands -from golem_core.core.market_api import Agreement, Proposal +from golem_core.core.market_api import Agreement, Proposal, DemandBuilder from golem_core.core.payment_api import Allocation from golem_core.core.resources import ResourceEvent +from golem_core.exceptions import BaseGolemException class Batch: @@ -87,6 +88,10 @@ class ManagerEvent(ResourceEvent, ABC): pass +class ManagerException(BaseGolemException): + pass + + class Manager(ABC): async def __aenter__(self): await self.start() @@ -108,13 +113,13 @@ async def get_allocation(self) -> Allocation: ... -class ProposalNegotiationManager(Manager, ABC): +class NegotiationManager(Manager, ABC): @abstractmethod async def get_proposal(self) -> Proposal: ... -class ProposalAggregationManager(Manager, ABC): +class ProposalManager(Manager, ABC): @abstractmethod async def get_proposal(self) -> Proposal: ... diff --git a/golem_core/managers/negotiation/accept_all.py b/golem_core/managers/negotiation/accept_all.py index 0855d0ad..e4f379bb 100644 --- a/golem_core/managers/negotiation/accept_all.py +++ b/golem_core/managers/negotiation/accept_all.py @@ -1,18 +1,29 @@ import asyncio import logging +from abc import ABC, abstractmethod +from copy import deepcopy from datetime import datetime, timezone -from typing import AsyncIterator, Awaitable, Callable, List +from typing import AsyncIterator, Awaitable, Callable, List, Optional from golem_core.core.golem_node.golem_node import DEFAULT_EXPIRATION_TIMEOUT, SUBNET, GolemNode from golem_core.core.market_api import Demand, DemandBuilder, Payload, Proposal from golem_core.core.market_api.resources.demand.demand_offer_base import defaults as dobm_defaults from golem_core.core.payment_api import Allocation -from golem_core.managers.base import ProposalNegotiationManager +from golem_core.managers.base import NegotiationManager, ManagerException logger = logging.getLogger(__name__) -class AcceptAllNegotiationManager(ProposalNegotiationManager): +class IgnoreProposal(Exception): + pass + +class NegotiationPlugin(ABC): + @abstractmethod + async def __call__(self, demand_builder: DemandBuilder, proposal: Proposal) -> DemandBuilder: + ... + + +class AcceptAllNegotiationManager(NegotiationManager): def __init__( self, golem: GolemNode, @@ -22,38 +33,63 @@ def __init__( self._golem = golem self._get_allocation = get_allocation self._payload = payload - self._negotiations: List[asyncio.Task] = [] + + self._negotiation_loop_task: Optional[asyncio.Task] = None + self._plugins: List[NegotiationPlugin] = [] self._eligible_proposals: asyncio.Queue[Proposal] = asyncio.Queue() + def register_plugin(self, plugin: NegotiationPlugin): + self._plugins.append(plugin) + + def unregister_plugin(self, plugin: NegotiationPlugin): + self._plugins.remove(plugin) + async def get_proposal(self) -> Proposal: logger.debug("Getting proposal...") proposal = await self._eligible_proposals.get() - logger.debug(f"Getting proposal done with `{proposal.id}`") + logger.debug(f"Getting proposal done with `{proposal}`") return proposal async def start(self) -> None: logger.debug("Starting negotiations...") - self._negotiations.append(asyncio.create_task(self._negotiate_task(self._payload))) + if self.is_negotiation_started(): + message = "Negotiation is already started!" + logger.debug(f"Starting negotiations failed with `{message}`") + raise ManagerException(message) + + self._negotiation_loop_task = asyncio.create_task(self._negotiation_loop(payload)) logger.debug("Starting negotiations done") async def stop(self) -> None: logger.debug("Stopping negotiations...") - for task in self._negotiations: - task.cancel() + if not self.is_negotiation_started(): + message = "Negotiation is already stopped!" + logger.debug(f"Stopping negotiations failed with `{message}`") + raise ManagerException(message) + + self._negotiation_loop_task.cancel() + self._negotiation_loop_task = None logger.debug("Stopping negotiations done") - async def _negotiate_task(self, payload: Payload) -> None: + def is_negotiation_started(self) -> bool: + return self._negotiation_loop_task is not None + + async def _negotiation_loop(self, payload: Payload) -> None: allocation = await self._get_allocation() demand = await self._build_demand(allocation, payload) - async for proposal in self._negotiate(demand): - await self._eligible_proposals.put(proposal) + + try: + async for proposal in self._negotiate(demand): + await self._eligible_proposals.put(proposal) + finally: + await demand.unsubscribe() async def _build_demand(self, allocation: Allocation, payload: Payload) -> Demand: logger.debug("Creating demand...") @@ -80,28 +116,43 @@ async def _build_demand(self, allocation: Allocation, payload: Payload) -> Deman demand = await demand_builder.create_demand(self._golem) demand.start_collecting_events() - logger.debug(f"Creating demand done with `{demand.id}`") + logger.debug(f"Creating demand done with `{demand}`") return demand async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: - try: - async for initial in demand.initial_proposals(): - logger.debug(f"Negotiating initial proposal `{initial.id}`...") - try: - demand_proposal = await initial.respond() - except Exception as err: - logger.debug( - f"Unable to respond to initial proposal {initial.id}.Got {type(err)}\n{err}" - ) - continue - - try: - offer_proposal = await demand_proposal.responses().__anext__() - except StopAsyncIteration: - continue - - logger.debug(f"Negotiating initial proposal `{initial.id}` done") - yield offer_proposal - finally: - self._golem.add_autoclose_resource(demand) + async for initial in demand.initial_proposals(): + logger.debug(f"Negotiating initial proposal `{initial}`...") + + try: + demand_proposal = await initial.respond() + except Exception as e: + logger.debug(f"Negotiating initial proposal `{initial}` failed with `{e}`") + continue + + try: + offer_proposal = await demand_proposal.responses().__anext__() + except StopAsyncIteration: + continue + + logger.debug(f"Negotiating initial proposal `{initial}` done") + + yield offer_proposal + + async def _negotiate_with_plugins(self, offer_proposal: Proposal) -> Optional[Proposal]: + while True: + original_demand_builder = DemandBuilder.from_proposal(offer_proposal) + demand_builder = deepcopy(original_demand_builder) + + try: + for plugin in self._plugins: + demand_builder = await plugin(demand_builder, offer_proposal) + except IgnoreProposal: + return None + + if offer_proposal.initial or demand_builder != original_demand_builder: + demand_proposal = await offer_proposal.respond(demand_builder.properties, demand_builder.constraints) + offer_proposal = await demand_proposal.responses().__anext__() + continue + + return offer_proposal diff --git a/golem_core/managers/proposal/stack.py b/golem_core/managers/proposal/stack.py index 63cc88e2..7139296a 100644 --- a/golem_core/managers/proposal/stack.py +++ b/golem_core/managers/proposal/stack.py @@ -4,12 +4,12 @@ from golem_core.core.golem_node.golem_node import GolemNode from golem_core.core.market_api import Proposal -from golem_core.managers.base import ProposalAggregationManager +from golem_core.managers.base import ProposalManager logger = logging.getLogger(__name__) -class StackProposalManager(ProposalAggregationManager): +class StackProposalManager(ProposalManager): def __init__(self, golem: GolemNode, get_proposal) -> None: self._get_proposal = get_proposal self._proposals: asyncio.Queue[Proposal] = asyncio.Queue() From 6ec7002423a2efc2d5089ac4c74f622da1b474b8 Mon Sep 17 00:00:00 2001 From: approxit Date: Tue, 6 Jun 2023 14:00:20 +0200 Subject: [PATCH 048/123] negotiation plugins PoC --- examples/managers/basic_composition.py | 18 +- .../resources/demand/demand_builder.py | 24 ++- .../core/market_api/resources/proposal.py | 19 +- golem_core/managers/base.py | 8 +- golem_core/managers/negotiation/__init__.py | 4 +- golem_core/managers/negotiation/accept_all.py | 158 --------------- golem_core/managers/negotiation/plugins.py | 30 +++ golem_core/managers/negotiation/sequential.py | 191 ++++++++++++++++++ golem_core/managers/payment/pay_all.py | 8 + golem_core/managers/proposal/stack.py | 10 +- golem_core/managers/work/sequential.py | 8 +- 11 files changed, 293 insertions(+), 185 deletions(-) delete mode 100644 golem_core/managers/negotiation/accept_all.py create mode 100644 golem_core/managers/negotiation/plugins.py create mode 100644 golem_core/managers/negotiation/sequential.py diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index 0c73906f..2c245001 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -8,7 +8,8 @@ from golem_core.managers.activity.single_use import SingleUseActivityManager from golem_core.managers.agreement.single_use import SingleUseAgreementManager from golem_core.managers.base import WorkContext, WorkResult -from golem_core.managers.negotiation import AcceptAllNegotiationManager +from golem_core.managers.negotiation import SequentialNegotiationManager +from golem_core.managers.negotiation.plugins import BlacklistProviderId from golem_core.managers.payment.pay_all import PayAllPaymentManager from golem_core.managers.proposal import StackProposalManager from golem_core.managers.work.decorators import ( @@ -56,8 +57,19 @@ async def main(): golem = GolemNode() payment_manager = PayAllPaymentManager(golem, budget=1.0) - negotiation_manager = AcceptAllNegotiationManager( - golem, payment_manager.get_allocation, payload + negotiation_manager = SequentialNegotiationManager( + golem, + payment_manager.get_allocation, + payload, + plugins=[ + BlacklistProviderId( + [ + "0x3b0f605fcb0690458064c10346af0c5f6b7202a5", + "0x7ad8ce2f95f69be197d136e308303d2395e68379", + "0x40f401ead13eabe677324bf50605c68caabb22c7", + ] + ), + ], ) proposal_manager = StackProposalManager(golem, negotiation_manager.get_proposal) agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) diff --git a/golem_core/core/market_api/resources/demand/demand_builder.py b/golem_core/core/market_api/resources/demand/demand_builder.py index 10a2bdb7..dba02188 100644 --- a/golem_core/core/market_api/resources/demand/demand_builder.py +++ b/golem_core/core/market_api/resources/demand/demand_builder.py @@ -1,5 +1,6 @@ import abc -from typing import TYPE_CHECKING, Any, Dict, List +from copy import deepcopy +from typing import TYPE_CHECKING, Any, Dict, List, Optional from golem_core.core.market_api.resources.demand.demand import Demand from golem_core.core.market_api.resources.demand.demand_offer_base.model import ( @@ -32,13 +33,22 @@ class DemandBuilder: ``` """ - def __init__(self): - self._properties: Dict[str, Any] = {} - self._constraints: List[str] = [] + def __init__( + self, properties: Optional[Dict[str, Any]] = None, constraints: Optional[List[str]] = None + ): + self._properties: Dict[str, Any] = properties if properties is not None else {} + self._constraints: List[str] = constraints if constraints is not None else [] def __repr__(self): return repr({"properties": self._properties, "constraints": self._constraints}) + def __eq__(self, other): + return ( + isinstance(other, DemandBuilder) + and self._properties == other.properties + and self.constraints == other.constraints + ) + @property def properties(self) -> Dict: """List of properties for this demand.""" @@ -77,6 +87,12 @@ async def create_demand(self, node: "GolemNode") -> "Demand": node, self.properties, self.constraints ) + @classmethod + async def from_demand(cls, demand: "Demand") -> "DemandBuilder": + demand_data = deepcopy(await demand.get_data()) + + return cls(demand_data.properties, [demand_data.constraints]) + class DemandBuilderDecorator(abc.ABC): """An interface that specifies classes that can add properties and constraints through a \ diff --git a/golem_core/core/market_api/resources/proposal.py b/golem_core/core/market_api/resources/proposal.py index 48c4a07b..07a98baa 100644 --- a/golem_core/core/market_api/resources/proposal.py +++ b/golem_core/core/market_api/resources/proposal.py @@ -1,6 +1,6 @@ import asyncio from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, AsyncIterator, Optional, Union +from typing import TYPE_CHECKING, AsyncIterator, Dict, Optional, Union from ya_market import RequestorApi from ya_market import models as models @@ -143,20 +143,24 @@ async def reject(self, reason: str = "") -> None: ) @api_call_wrapper() - async def respond(self, properties, constraints) -> "Proposal": + async def respond( + self, properties: Optional[Dict] = None, constraints: Optional[str] = None + ) -> "Proposal": """Respond to a proposal with a counter-proposal. Invalid on our responses. - TODO: all the negotiation logic should be reflected in params of this method, - but negotiations are not implemented yet. Related issues: + Related issues: https://github.com/golemfactory/golem-core-python/issues/17 https://github.com/golemfactory/golem-core-python/issues/18 """ + if properties is None and constraints is None: + data = await self._response_data() + elif properties is not None and constraints is not None: + data = models.DemandOfferBase(properties=properties, constraints=constraints) + else: + raise ValueError("Both `properties` and `constraints` arguments must be provided!") - data = models.DemandOfferBase( - properties=properties, constraints=constraints - ) new_proposal_id = await self.api.counter_proposal_demand( self.demand.id, self.id, data, _request_timeout=5 ) @@ -167,7 +171,6 @@ async def respond(self, properties, constraints) -> "Proposal": return new_proposal async def _response_data(self) -> models.DemandOfferBase: - # FIXME: this is a mock demand_data = await self.demand.get_data() data = models.DemandOfferBase( properties=demand_data.properties, constraints=demand_data.constraints diff --git a/golem_core/managers/base.py b/golem_core/managers/base.py index 0fc73391..93eef0d0 100644 --- a/golem_core/managers/base.py +++ b/golem_core/managers/base.py @@ -3,7 +3,7 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional, Union from golem_core.core.activity_api import Activity, Script, commands -from golem_core.core.market_api import Agreement, Proposal, DemandBuilder +from golem_core.core.market_api import Agreement, DemandBuilder, Proposal from golem_core.core.payment_api import Allocation from golem_core.core.resources import ResourceEvent from golem_core.exceptions import BaseGolemException @@ -139,3 +139,9 @@ async def do_work(self, work: Work) -> WorkResult: class WorkManager(Manager, ABC): ... + + +class NegotiationPlugin(ABC): + @abstractmethod + async def __call__(self, demand_builder: DemandBuilder, proposal: Proposal) -> DemandBuilder: + ... diff --git a/golem_core/managers/negotiation/__init__.py b/golem_core/managers/negotiation/__init__.py index a47383ee..55ec2fd6 100644 --- a/golem_core/managers/negotiation/__init__.py +++ b/golem_core/managers/negotiation/__init__.py @@ -1,3 +1,3 @@ -from golem_core.managers.negotiation.accept_all import AcceptAllNegotiationManager +from golem_core.managers.negotiation.sequential import SequentialNegotiationManager -__all__ = ("AcceptAllNegotiationManager",) +__all__ = ("SequentialNegotiationManager",) diff --git a/golem_core/managers/negotiation/accept_all.py b/golem_core/managers/negotiation/accept_all.py deleted file mode 100644 index e4f379bb..00000000 --- a/golem_core/managers/negotiation/accept_all.py +++ /dev/null @@ -1,158 +0,0 @@ -import asyncio -import logging -from abc import ABC, abstractmethod -from copy import deepcopy -from datetime import datetime, timezone -from typing import AsyncIterator, Awaitable, Callable, List, Optional - -from golem_core.core.golem_node.golem_node import DEFAULT_EXPIRATION_TIMEOUT, SUBNET, GolemNode -from golem_core.core.market_api import Demand, DemandBuilder, Payload, Proposal -from golem_core.core.market_api.resources.demand.demand_offer_base import defaults as dobm_defaults -from golem_core.core.payment_api import Allocation -from golem_core.managers.base import NegotiationManager, ManagerException - -logger = logging.getLogger(__name__) - - -class IgnoreProposal(Exception): - pass - -class NegotiationPlugin(ABC): - @abstractmethod - async def __call__(self, demand_builder: DemandBuilder, proposal: Proposal) -> DemandBuilder: - ... - - -class AcceptAllNegotiationManager(NegotiationManager): - def __init__( - self, - golem: GolemNode, - get_allocation: Callable[[], Awaitable[Allocation]], - payload: Payload, - ) -> None: - self._golem = golem - self._get_allocation = get_allocation - self._payload = payload - - self._negotiation_loop_task: Optional[asyncio.Task] = None - self._plugins: List[NegotiationPlugin] = [] - self._eligible_proposals: asyncio.Queue[Proposal] = asyncio.Queue() - - def register_plugin(self, plugin: NegotiationPlugin): - self._plugins.append(plugin) - - def unregister_plugin(self, plugin: NegotiationPlugin): - self._plugins.remove(plugin) - - async def get_proposal(self) -> Proposal: - logger.debug("Getting proposal...") - - proposal = await self._eligible_proposals.get() - - logger.debug(f"Getting proposal done with `{proposal}`") - - return proposal - - async def start(self) -> None: - logger.debug("Starting negotiations...") - - if self.is_negotiation_started(): - message = "Negotiation is already started!" - logger.debug(f"Starting negotiations failed with `{message}`") - raise ManagerException(message) - - self._negotiation_loop_task = asyncio.create_task(self._negotiation_loop(payload)) - - logger.debug("Starting negotiations done") - - async def stop(self) -> None: - logger.debug("Stopping negotiations...") - - if not self.is_negotiation_started(): - message = "Negotiation is already stopped!" - logger.debug(f"Stopping negotiations failed with `{message}`") - raise ManagerException(message) - - self._negotiation_loop_task.cancel() - self._negotiation_loop_task = None - - logger.debug("Stopping negotiations done") - - def is_negotiation_started(self) -> bool: - return self._negotiation_loop_task is not None - - async def _negotiation_loop(self, payload: Payload) -> None: - allocation = await self._get_allocation() - demand = await self._build_demand(allocation, payload) - - try: - async for proposal in self._negotiate(demand): - await self._eligible_proposals.put(proposal) - finally: - await demand.unsubscribe() - - async def _build_demand(self, allocation: Allocation, payload: Payload) -> Demand: - logger.debug("Creating demand...") - - demand_builder = DemandBuilder() - - await demand_builder.add( - dobm_defaults.Activity( - expiration=datetime.now(timezone.utc) + DEFAULT_EXPIRATION_TIMEOUT, - multi_activity=True, - ) - ) - await demand_builder.add(dobm_defaults.NodeInfo(subnet_tag=SUBNET)) - - await demand_builder.add(payload) - - ( - allocation_properties, - allocation_constraints, - ) = await allocation.demand_properties_constraints() - demand_builder.add_constraints(*allocation_constraints) - demand_builder.add_properties({p.key: p.value for p in allocation_properties}) - - demand = await demand_builder.create_demand(self._golem) - demand.start_collecting_events() - - logger.debug(f"Creating demand done with `{demand}`") - - return demand - - async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: - async for initial in demand.initial_proposals(): - logger.debug(f"Negotiating initial proposal `{initial}`...") - - try: - demand_proposal = await initial.respond() - except Exception as e: - logger.debug(f"Negotiating initial proposal `{initial}` failed with `{e}`") - continue - - try: - offer_proposal = await demand_proposal.responses().__anext__() - except StopAsyncIteration: - continue - - logger.debug(f"Negotiating initial proposal `{initial}` done") - - yield offer_proposal - - async def _negotiate_with_plugins(self, offer_proposal: Proposal) -> Optional[Proposal]: - while True: - original_demand_builder = DemandBuilder.from_proposal(offer_proposal) - demand_builder = deepcopy(original_demand_builder) - - try: - for plugin in self._plugins: - demand_builder = await plugin(demand_builder, offer_proposal) - except IgnoreProposal: - return None - - if offer_proposal.initial or demand_builder != original_demand_builder: - demand_proposal = await offer_proposal.respond(demand_builder.properties, demand_builder.constraints) - offer_proposal = await demand_proposal.responses().__anext__() - continue - - return offer_proposal diff --git a/golem_core/managers/negotiation/plugins.py b/golem_core/managers/negotiation/plugins.py new file mode 100644 index 00000000..998c5463 --- /dev/null +++ b/golem_core/managers/negotiation/plugins.py @@ -0,0 +1,30 @@ +import logging +from typing import Sequence + +from golem_core.core.market_api import DemandBuilder, Proposal +from golem_core.managers.base import NegotiationPlugin +from golem_core.managers.negotiation.sequential import RejectProposal + +logger = logging.getLogger(__name__) + + +class BlacklistProviderId(NegotiationPlugin): + def __init__(self, blacklist: Sequence[str]) -> None: + self._blacklist = blacklist + + async def __call__(self, demand_builder: DemandBuilder, proposal: Proposal) -> DemandBuilder: + logger.debug("Calling blacklist plugin...") + + provider_id = proposal.data.issuer_id + + if provider_id in self._blacklist: + logger.debug( + f"Calling blacklist plugin done with provider `{provider_id}` is blacklisted" + ) + raise RejectProposal(f"Provider ID `{provider_id}` is blacklisted by the requestor") + + logger.debug( + f"Calling blacklist plugin done with provider `{provider_id}` is not blacklisted" + ) + + return demand_builder diff --git a/golem_core/managers/negotiation/sequential.py b/golem_core/managers/negotiation/sequential.py new file mode 100644 index 00000000..a8ad1229 --- /dev/null +++ b/golem_core/managers/negotiation/sequential.py @@ -0,0 +1,191 @@ +import asyncio +import logging +from copy import deepcopy +from datetime import datetime, timezone +from typing import AsyncIterator, Awaitable, Callable, List, Optional, Sequence + +from golem_core.core.golem_node.golem_node import DEFAULT_EXPIRATION_TIMEOUT, SUBNET, GolemNode +from golem_core.core.market_api import Demand, DemandBuilder, Payload, Proposal +from golem_core.core.market_api.resources.demand.demand_offer_base import defaults as dobm_defaults +from golem_core.core.payment_api import Allocation +from golem_core.managers.base import ManagerException, NegotiationManager, NegotiationPlugin + +logger = logging.getLogger(__name__) + + +class RejectProposal(Exception): + pass + + +class SequentialNegotiationManager(NegotiationManager): + def __init__( + self, + golem: GolemNode, + get_allocation: Callable[[], Awaitable[Allocation]], + payload: Payload, + plugins: Optional[Sequence[NegotiationPlugin]] = None, + ) -> None: + self._golem = golem + self._get_allocation = get_allocation + self._payload = payload + + self._negotiation_loop_task: Optional[asyncio.Task] = None + self._plugins: List[NegotiationPlugin] = list(plugins) if plugins is not None else [] + self._eligible_proposals: asyncio.Queue[Proposal] = asyncio.Queue() + + def register_plugin(self, plugin: NegotiationPlugin): + self._plugins.append(plugin) + + def unregister_plugin(self, plugin: NegotiationPlugin): + self._plugins.remove(plugin) + + async def get_proposal(self) -> Proposal: + logger.debug("Getting proposal...") + + proposal = await self._eligible_proposals.get() + + logger.debug(f"Getting proposal done with `{proposal}`") + + return proposal + + async def start(self) -> None: + logger.debug("Starting...") + + if self.is_started_started(): + message = "Already started!" + logger.debug(f"Starting failed with `{message}`") + raise ManagerException(message) + + self._negotiation_loop_task = asyncio.create_task(self._negotiation_loop(self._payload)) + + logger.debug("Starting done") + + async def stop(self) -> None: + logger.debug("Stopping...") + + if not self.is_started_started(): + message = "Already stopped!" + logger.debug(f"Stopping failed with `{message}`") + raise ManagerException(message) + + self._negotiation_loop_task.cancel() + self._negotiation_loop_task = None + + logger.debug("Stopping done") + + def is_started_started(self) -> bool: + return self._negotiation_loop_task is not None + + async def _negotiation_loop(self, payload: Payload) -> None: + allocation = await self._get_allocation() + demand_builder = await self._prepare_demand_builder(allocation, payload) + + demand = await demand_builder.create_demand(self._golem) + demand.start_collecting_events() + + try: + async for proposal in self._negotiate(demand_builder, demand): + await self._eligible_proposals.put(proposal) + finally: + await demand.unsubscribe() + + async def _prepare_demand_builder( + self, allocation: Allocation, payload: Payload + ) -> DemandBuilder: + logger.debug("Preparing demand...") + + # FIXME: Code looks duplicated as GolemNode.create_demand does the same + + demand_builder = DemandBuilder() + + await demand_builder.add( + dobm_defaults.Activity( + expiration=datetime.now(timezone.utc) + DEFAULT_EXPIRATION_TIMEOUT, + multi_activity=True, + ) + ) + await demand_builder.add(dobm_defaults.NodeInfo(subnet_tag=SUBNET)) + + await demand_builder.add(payload) + + ( + allocation_properties, + allocation_constraints, + ) = await allocation.demand_properties_constraints() + demand_builder.add_constraints(*allocation_constraints) + demand_builder.add_properties({p.key: p.value for p in allocation_properties}) + + logger.debug(f"Preparing demand done`") + + return demand_builder + + async def _negotiate( + self, demand_builder: DemandBuilder, demand: Demand + ) -> AsyncIterator[Proposal]: + async for initial_offer_proposal in demand.initial_proposals(): + offer_proposal = await self._negotiate_proposal(demand_builder, initial_offer_proposal) + + if offer_proposal is None: + logger.debug( + f"Negotiating proposal `{initial_offer_proposal}` done and proposal was rejected" + ) + continue + + yield offer_proposal + + async def _negotiate_proposal( + self, demand_builder: DemandBuilder, offer_proposal: Proposal + ) -> Optional[Proposal]: + logger.debug(f"Negotiating proposal `{offer_proposal}`...") + + while True: + demand_builder_after_plugins = deepcopy(demand_builder) + + try: + logger.debug(f"Applying plugins on `{offer_proposal}`...") + + for plugin in self._plugins: + demand_builder_after_plugins = await plugin( + demand_builder_after_plugins, offer_proposal + ) + + except RejectProposal as e: + logger.debug( + f"Applying plugins on `{offer_proposal}` done and proposal was rejected" + ) + + if not offer_proposal.initial: + await offer_proposal.reject(str(e)) + + return None + else: + logger.debug(f"Applying plugins on `{offer_proposal}` done") + + if offer_proposal.initial or demand_builder_after_plugins != demand_builder: + logger.debug("Sending demand proposal...") + + demand_proposal = await offer_proposal.respond( + demand_builder_after_plugins.properties, + demand_builder_after_plugins.constraints, + ) + + logger.debug("Sending demand proposal done") + + logger.debug("Waiting for response...") + + new_offer_proposal = await demand_proposal.responses().__anext__() + + logger.debug(f"Waiting for response done with `{new_offer_proposal}`") + + logger.debug( + f"Proposal `{offer_proposal}` received counter proposal `{new_offer_proposal}`" + ) + offer_proposal = new_offer_proposal + + continue + else: + break + + logger.debug(f"Negotiating proposal `{offer_proposal}` done") + + return offer_proposal diff --git a/golem_core/managers/payment/pay_all.py b/golem_core/managers/payment/pay_all.py index 3e11e80a..be1cbdfe 100644 --- a/golem_core/managers/payment/pay_all.py +++ b/golem_core/managers/payment/pay_all.py @@ -34,6 +34,8 @@ def __init__( self._payed_invoices_count = 0 async def start(self) -> None: + logger.debug("Starting...") + # TODO: Add stop with event_bus.off() await self._golem.event_bus.on(NewInvoice, self._pay_invoice_if_received) @@ -42,9 +44,15 @@ async def start(self) -> None: await self._golem.event_bus.on(NewAgreement, self._increment_opened_agreements) await self._golem.event_bus.on(AgreementClosed, self._increment_closed_agreements) + logger.debug("Starting done") + async def stop(self) -> None: + logger.debug("Stopping...") + await self.wait_for_invoices() + logger.debug("Stopping done") + async def get_allocation(self) -> "Allocation": logger.debug("Getting allocation...") diff --git a/golem_core/managers/proposal/stack.py b/golem_core/managers/proposal/stack.py index 7139296a..0ee8fc57 100644 --- a/golem_core/managers/proposal/stack.py +++ b/golem_core/managers/proposal/stack.py @@ -16,19 +16,19 @@ def __init__(self, golem: GolemNode, get_proposal) -> None: self._tasks: List[asyncio.Task] = [] async def start(self) -> None: - logger.debug("Starting consuming proposals...") + logger.debug("Starting...") self._tasks.append(asyncio.create_task(self._consume_proposals())) - logger.debug("Starting consuming proposals done") + logger.debug("Starting done") async def stop(self) -> None: - logger.debug("Stopping consuming proposals...") + logger.debug("Stopping...") for task in self._tasks: task.cancel() - logger.debug("Stopping consuming proposals done") + logger.debug("Stopping done") async def _consume_proposals(self) -> None: while True: @@ -43,7 +43,7 @@ async def get_proposal(self) -> Proposal: proposal = await self._proposals.get() - logger.debug(f"Getting proposal done with `{proposal.id}`") + logger.debug(f"Getting proposal done with `{proposal}`") logger.info(f"Proposal `{proposal}` picked") diff --git a/golem_core/managers/work/sequential.py b/golem_core/managers/work/sequential.py index 10ad6edc..f28dd635 100644 --- a/golem_core/managers/work/sequential.py +++ b/golem_core/managers/work/sequential.py @@ -26,19 +26,19 @@ def _apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkC return result async def do_work(self, work: Work) -> WorkResult: - logger.debug(f"Running work {work}") + logger.debug(f"Running work `{work}`...") decorated_do_work = self._apply_work_decorators(self._do_work, work) result = await decorated_do_work(work) - logger.debug(f"Running work done {work}") + logger.debug(f"Running work `{work}` done") logger.info(f"Work `{work}` completed") return result async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: - logger.debug(f"Running work sequence {work_list}") + logger.debug(f"Running work sequence `{work_list}`...") results = [] @@ -49,6 +49,6 @@ async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: logger.debug(f"Doing work sequence #{i} done") - logger.debug(f"Running work sequence done {work_list}") + logger.debug(f"Running work sequence `{work_list}` done") return results From 4736c63e83eff676d6f93bf05aeeabd77dc3437c Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 7 Jun 2023 13:15:34 +0200 Subject: [PATCH 049/123] Add managers ssh example --- README.md | 2 +- examples/cli_example.sh | 2 +- examples/managers/ssh.py | 92 ++++++++++++++++++++++++ golem_core/core/golem_node/golem_node.py | 2 +- golem_core/managers/base.py | 12 ++-- golem_core/managers/network/__init__.py | 0 golem_core/managers/network/single.py | 50 +++++++++++++ golem_core/managers/payment/pay_all.py | 1 + 8 files changed, 154 insertions(+), 7 deletions(-) create mode 100644 examples/managers/ssh.py create mode 100644 golem_core/managers/network/__init__.py create mode 100644 golem_core/managers/network/single.py diff --git a/README.md b/README.md index b5fe7c85..3b5e807e 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ $ python -m golem_core find-node --runtime vm --timeout 1m # stops after 60 seco $ python -m golem_core allocation list $ python -m golem_core allocation new 1 -$ python -m golem_core allocation new 2 --driver erc20 --network rinkeby +$ python -m golem_core allocation new 2 --driver erc20 --network goerli $ python -m golem_core allocation clean ``` diff --git a/examples/cli_example.sh b/examples/cli_example.sh index 0451839a..05f8dc46 100644 --- a/examples/cli_example.sh +++ b/examples/cli_example.sh @@ -13,7 +13,7 @@ printf "\n*** ALLOCATION NEW ***\n" python3 -m golem_core allocation new 1 printf "\n*** ALLOCATION NEW ***\n" -python3 -m golem_core allocation new 2 --driver erc20 --network rinkeby +python3 -m golem_core allocation new 2 --driver erc20 --network goerli printf "\n*** ALLOCATION LIST ***\n" python3 -m golem_core allocation list diff --git a/examples/managers/ssh.py b/examples/managers/ssh.py new file mode 100644 index 00000000..d25187ca --- /dev/null +++ b/examples/managers/ssh.py @@ -0,0 +1,92 @@ +import asyncio +import logging.config +import random +import string +from uuid import uuid4 + +from golem_core.core.golem_node.golem_node import GolemNode +from golem_core.core.market_api import RepositoryVmPayload +from golem_core.managers.activity.single_use import SingleUseActivityManager +from golem_core.managers.agreement.single_use import SingleUseAgreementManager +from golem_core.managers.base import WorkContext, WorkResult +from golem_core.managers.negotiation import SequentialNegotiationManager +from golem_core.managers.network.single import SingleNetworkManager +from golem_core.managers.payment.pay_all import PayAllPaymentManager +from golem_core.managers.proposal import StackProposalManager +from golem_core.managers.work.sequential import SequentialWorkManager +from golem_core.utils.logging import DEFAULT_LOGGING + + +def on_activity_start(get_network_deploy_args): + async def _on_activity_start(context: WorkContext): + deploy_args = { + "net": [await get_network_deploy_args(context._activity.parent.parent.data.issuer_id)] + } + batch = await context.create_batch() + batch.deploy(deploy_args) + batch.start() + await batch() + + return _on_activity_start + + +def work(app_key, get_provider_uri): + async def _work(context: WorkContext) -> str: + password = "".join(random.choice(string.ascii_letters + string.digits) for _ in range(8)) + batch = await context.create_batch() + batch.run("syslogd") + batch.run("ssh-keygen -A") + batch.run(f'echo -e "{password}\n{password}" | passwd') + batch.run("/usr/sbin/sshd") + batch_result = await batch() + result = "" + for event in batch_result: + result += f"{event.stdout}" + + print( + "Connect with:\n" + f" ssh -o ProxyCommand='websocat asyncstdio: {await get_provider_uri(context._activity.parent.parent.data.issuer_id, 'ws')} --binary " + f'-H=Authorization:"Bearer {app_key}"\' root@{uuid4().hex} ' + ) + print(f"PASSWORD: {password}") + + for _ in range(3): + await asyncio.sleep(1) + return result + + return _work + + +async def main(): + logging.config.dictConfig(DEFAULT_LOGGING) + payload = RepositoryVmPayload( + "1e06505997e8bd1b9e1a00bd10d255fc6a390905e4d6840a22a79902", + capabilities=["vpn"], + ) + network_ip = "192.168.0.1/24" + + golem = GolemNode() + + network_manager = SingleNetworkManager(golem, network_ip) + payment_manager = PayAllPaymentManager(golem, budget=1.0) + negotiation_manager = SequentialNegotiationManager( + golem, payment_manager.get_allocation, payload + ) + proposal_manager = StackProposalManager(golem, negotiation_manager.get_proposal) + agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) + activity_manager = SingleUseActivityManager( + golem, + agreement_manager.get_agreement, + on_activity_start=on_activity_start(network_manager.get_deploy_args), + ) + work_manager = SequentialWorkManager(golem, activity_manager.do_work) + + async with golem, network_manager, payment_manager, negotiation_manager, proposal_manager: + result: WorkResult = await work_manager.do_work( + work(golem._api_config.app_key, network_manager.get_provider_uri) + ) + print(f"\nWORK MANAGER RESULTS:{result.result}\n") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/golem_core/core/golem_node/golem_node.py b/golem_core/core/golem_node/golem_node.py index 7c2a0cec..55ea6303 100644 --- a/golem_core/core/golem_node/golem_node.py +++ b/golem_core/core/golem_node/golem_node.py @@ -22,7 +22,7 @@ from golem_core.core.resources import ApiConfig, ApiFactory, Resource, TResource PAYMENT_DRIVER: str = os.getenv("YAGNA_PAYMENT_DRIVER", "erc20").lower() -PAYMENT_NETWORK: str = os.getenv("YAGNA_PAYMENT_NETWORK", "rinkeby").lower() +PAYMENT_NETWORK: str = os.getenv("YAGNA_PAYMENT_NETWORK", "goerli").lower() SUBNET: str = os.getenv("YAGNA_SUBNET", "public") DEFAULT_EXPIRATION_TIMEOUT = timedelta(minutes=30) diff --git a/golem_core/managers/base.py b/golem_core/managers/base.py index 93eef0d0..6e6cf685 100644 --- a/golem_core/managers/base.py +++ b/golem_core/managers/base.py @@ -14,8 +14,8 @@ def __init__(self, activity) -> None: self._script = Script() self._activity = activity - def deploy(self): - self._script.add_command(commands.Deploy()) + def deploy(self, deploy_args: Optional[commands.ArgsDict] = None): + self._script.add_command(commands.Deploy(deploy_args)) def start(self): self._script.add_command(commands.Start()) @@ -38,8 +38,8 @@ class WorkContext: def __init__(self, activity: Activity): self._activity = activity - async def deploy(self): - pooling_batch = await self._activity.execute_commands(commands.Deploy()) + async def deploy(self, deploy_args: Optional[commands.ArgsDict] = None): + pooling_batch = await self._activity.execute_commands(commands.Deploy(deploy_args)) await pooling_batch.wait() async def start(self): @@ -107,6 +107,10 @@ async def stop(self): ... +class NetworkManager(Manager, ABC): + ... + + class PaymentManager(Manager, ABC): @abstractmethod async def get_allocation(self) -> Allocation: diff --git a/golem_core/managers/network/__init__.py b/golem_core/managers/network/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/golem_core/managers/network/single.py b/golem_core/managers/network/single.py new file mode 100644 index 00000000..a7d44dd0 --- /dev/null +++ b/golem_core/managers/network/single.py @@ -0,0 +1,50 @@ +import asyncio +import logging +from typing import Dict +from urllib.parse import urlparse + +from golem_core.core.golem_node.golem_node import GolemNode +from golem_core.core.market_api.events import NewAgreement +from golem_core.core.network_api import Network +from golem_core.core.network_api.resources.network import DeployArgsType +from golem_core.managers.base import NetworkManager + +logger = logging.getLogger(__name__) + + +class SingleNetworkManager(NetworkManager): + def __init__(self, golem: GolemNode, ip: str) -> None: + self._golem = golem + self._ip = ip + self._nodes: Dict[str, str] = {} + + async def start(self): + self._network = await Network.create(self._golem, self._ip, None, None) + await self._network.add_requestor_ip(None) + self._golem.add_autoclose_resource(self._network) + + await self._golem.event_bus.on(NewAgreement, self._add_provider_to_network) + + async def get_node_id(self, provider_id: str) -> str: + while True: + node_ip = self._nodes.get(provider_id) + if node_ip: + return node_ip + await asyncio.sleep(0.1) + + async def get_deploy_args(self, provider_id: str) -> DeployArgsType: + node_ip = await self.get_node_id(provider_id) + return self._network.deploy_args(node_ip) + + async def get_provider_uri(self, provider_id: str, protocol: str = "http") -> str: + node_ip = await self.get_node_id(provider_id) + url = self._network.node._api_config.net_url + net_api_ws = urlparse(url)._replace(scheme=protocol).geturl() + connection_uri = f"{net_api_ws}/net/{self._network.id}/tcp/{node_ip}/22" + return connection_uri + + async def _add_provider_to_network(self, event: NewAgreement): + await event.resource.get_data() + provider_id = event.resource.data.offer.provider_id + logger.info(f"Adding provider {provider_id} to network") + self._nodes[provider_id] = await self._network.create_node(provider_id) diff --git a/golem_core/managers/payment/pay_all.py b/golem_core/managers/payment/pay_all.py index be1cbdfe..0effb6d0 100644 --- a/golem_core/managers/payment/pay_all.py +++ b/golem_core/managers/payment/pay_all.py @@ -54,6 +54,7 @@ async def stop(self) -> None: logger.debug("Stopping done") async def get_allocation(self) -> "Allocation": + # TODO handle NoMatchingAccount logger.debug("Getting allocation...") if self._allocation is None: From e7d5c5944526f3ad1b4d8d6e9dcaf48947a74d8f Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 7 Jun 2023 13:18:01 +0200 Subject: [PATCH 050/123] Added TODO to ssh manager example --- examples/managers/ssh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/managers/ssh.py b/examples/managers/ssh.py index d25187ca..a3f3c1eb 100644 --- a/examples/managers/ssh.py +++ b/examples/managers/ssh.py @@ -80,7 +80,7 @@ async def main(): on_activity_start=on_activity_start(network_manager.get_deploy_args), ) work_manager = SequentialWorkManager(golem, activity_manager.do_work) - + # TODO use different managers so it allows to finish work func without destroying activity async with golem, network_manager, payment_manager, negotiation_manager, proposal_manager: result: WorkResult = await work_manager.do_work( work(golem._api_config.app_key, network_manager.get_provider_uri) From 2f4e3c38ba8d91f15f2780927ad64bcbd441d9e6 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 7 Jun 2023 13:38:57 +0200 Subject: [PATCH 051/123] Fix old examples with new event bus interaface --- examples/exception_handling/exception_handling.py | 3 ++- examples/rate_providers/rate_providers.py | 3 ++- examples/score_based_providers/score_based_providers.py | 3 ++- examples/service.py | 3 ++- examples/task_api_draft/examples/pipeline_example.py | 3 ++- examples/task_api_draft/task_api/execute_tasks.py | 3 ++- golem_core/utils/logging.py | 4 ++-- 7 files changed, 14 insertions(+), 8 deletions(-) diff --git a/examples/exception_handling/exception_handling.py b/examples/exception_handling/exception_handling.py index d1f70cdc..6faee3a5 100644 --- a/examples/exception_handling/exception_handling.py +++ b/examples/exception_handling/exception_handling.py @@ -2,6 +2,7 @@ from typing import Callable, Tuple from golem_core.core.activity_api import Activity, BatchError, BatchTimeoutError, commands +from golem_core.core.events.base import Event from golem_core.core.golem_node import GolemNode from golem_core.core.market_api import ( RepositoryVmPayload, @@ -41,7 +42,7 @@ async def on_exception(func: Callable, args: Tuple, e: Exception) -> None: async def main() -> None: golem = GolemNode() - golem.event_bus.listen(DefaultLogger().on_event) + await golem.event_bus.on(Event, DefaultLogger().on_event) async with golem: allocation = await golem.create_allocation(1.0) diff --git a/examples/rate_providers/rate_providers.py b/examples/rate_providers/rate_providers.py index ca7ef166..d12d8672 100644 --- a/examples/rate_providers/rate_providers.py +++ b/examples/rate_providers/rate_providers.py @@ -5,6 +5,7 @@ from typing import Any, AsyncIterator, Callable, Dict, Optional, Tuple from golem_core.core.activity_api import Activity, commands +from golem_core.core.events.base import Event from golem_core.core.golem_node import GolemNode from golem_core.core.market_api import ( Proposal, @@ -82,7 +83,7 @@ async def on_exception(func: Callable, args: Tuple, e: Exception) -> None: async def main() -> None: golem = GolemNode() - golem.event_bus.listen(DefaultLogger().on_event) + await golem.event_bus.on(Event, DefaultLogger().on_event) async with golem: allocation = await golem.create_allocation(1.0) diff --git a/examples/score_based_providers/score_based_providers.py b/examples/score_based_providers/score_based_providers.py index 55b12c99..fa747323 100644 --- a/examples/score_based_providers/score_based_providers.py +++ b/examples/score_based_providers/score_based_providers.py @@ -3,6 +3,7 @@ from typing import Callable, Dict, Optional, Tuple from golem_core.core.activity_api import Activity, commands +from golem_core.core.events.base import Event from golem_core.core.golem_node import GolemNode from golem_core.core.market_api import ( Proposal, @@ -96,7 +97,7 @@ async def on_exception(func: Callable, args: Tuple, e: Exception) -> None: async def main() -> None: golem = GolemNode() - golem.event_bus.listen(DefaultLogger().on_event) + await golem.event_bus.on(Event, DefaultLogger().on_event) async with golem: allocation = await golem.create_allocation(1) diff --git a/examples/service.py b/examples/service.py index 916984a3..4591e39f 100644 --- a/examples/service.py +++ b/examples/service.py @@ -6,6 +6,7 @@ from uuid import uuid4 from golem_core.core.activity_api import Activity, commands +from golem_core.core.events.base import Event from golem_core.core.golem_node import GolemNode from golem_core.core.market_api import ( RepositoryVmPayload, @@ -56,7 +57,7 @@ async def _create_ssh_connection(activity: Activity) -> Tuple[str, str]: async def main() -> None: golem = GolemNode() - golem.event_bus.listen(DefaultLogger().on_event) + await golem.event_bus.on(Event, DefaultLogger().on_event) async with golem: network = await golem.create_network("192.168.0.1/24") diff --git a/examples/task_api_draft/examples/pipeline_example.py b/examples/task_api_draft/examples/pipeline_example.py index 7453140c..95c83790 100644 --- a/examples/task_api_draft/examples/pipeline_example.py +++ b/examples/task_api_draft/examples/pipeline_example.py @@ -5,6 +5,7 @@ from examples.task_api_draft.task_api.activity_pool import ActivityPool from golem_core.core.activity_api import Activity, commands +from golem_core.core.events.base import Event from golem_core.core.golem_node import GolemNode from golem_core.core.market_api import ( Proposal, @@ -68,7 +69,7 @@ async def on_exception(func: Callable, args: Tuple, e: Exception) -> None: async def main() -> None: golem = GolemNode() - golem.event_bus.listen(DefaultLogger().on_event) + await golem.event_bus.on(Event, DefaultLogger().on_event) async with golem: allocation = await golem.create_allocation(1) diff --git a/examples/task_api_draft/task_api/execute_tasks.py b/examples/task_api_draft/task_api/execute_tasks.py index 0568257e..070a8987 100644 --- a/examples/task_api_draft/task_api/execute_tasks.py +++ b/examples/task_api_draft/task_api/execute_tasks.py @@ -3,6 +3,7 @@ from typing import AsyncIterator, Awaitable, Callable, Iterable, Optional, Tuple, TypeVar from golem_core.core.activity_api import Activity, default_prepare_activity +from golem_core.core.events.base import Event from golem_core.core.golem_node import GolemNode from golem_core.core.market_api import ( Demand, @@ -143,7 +144,7 @@ async def execute_tasks( task_stream = TaskDataStream(task_data) golem = GolemNode() - golem.event_bus.listen(DefaultLogger().on_event) + await golem.event_bus.on(Event, DefaultLogger().on_event) async with golem: allocation = await golem.create_allocation(budget) diff --git a/golem_core/utils/logging.py b/golem_core/utils/logging.py index ae783a3c..332e9ec2 100644 --- a/golem_core/utils/logging.py +++ b/golem_core/utils/logging.py @@ -56,7 +56,7 @@ class DefaultLogger: Usage:: golem = GolemNode() - golem.event_bus.listen(DefaultLogger().on_event) + await golem.event_bus.on(Event, DefaultLogger().on_event) Or:: @@ -96,5 +96,5 @@ def _prepare_logger(self) -> logging.Logger: return logger async def on_event(self, event: "Event") -> None: - """Handle event produced by :any:`EventBus.listen`.""" + """Handle event produced by :any:`EventBus.on`.""" self.logger.info(event) From 9fb625657e985d5694ead69e20acb830cb36480e Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 7 Jun 2023 13:47:27 +0200 Subject: [PATCH 052/123] Fix resource_listen with new on func --- examples/task_api_draft/examples/yacat.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/examples/task_api_draft/examples/yacat.py b/examples/task_api_draft/examples/yacat.py index 43354957..5b305c42 100644 --- a/examples/task_api_draft/examples/yacat.py +++ b/examples/task_api_draft/examples/yacat.py @@ -174,10 +174,16 @@ async def main() -> None: golem = GolemNode() await golem.event_bus.on(Event, DefaultLogger().on_event) - golem.event_bus.resource_listen(count_batches, [NewResource], [PoolingBatch]) + await golem.event_bus.on( + NewResource, count_batches, lambda e: isinstance(e.resource, PoolingBatch) + ) await golem.event_bus.on(NewDebitNote, gather_debit_note_log) - golem.event_bus.resource_listen(update_new_activity_status, [NewResource], [Activity]) - golem.event_bus.resource_listen(note_activity_destroyed, [ResourceClosed], [Activity]) + await golem.event_bus.on( + NewResource, update_new_activity_status, lambda e: isinstance(e.resource, Activity) + ) + await golem.event_bus.on( + ResourceClosed, note_activity_destroyed, lambda e: isinstance(e.resource, Activity) + ) async with golem: allocation = await golem.create_allocation(amount=1) From b06534440145f5d088a45d162bb42f64f53975b2 Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 7 Jun 2023 21:49:46 +0200 Subject: [PATCH 053/123] new properties and constraints in progress --- .../market_api/resources/demand/demand.py | 12 ++ .../resources/demand/demand_builder.py | 53 ++----- .../demand/demand_offer_base/model.py | 149 +++--------------- .../core/market_api/resources/proposal.py | 23 +++ golem_core/core/props_cons/__init__.py | 0 golem_core/core/props_cons/base.py | 45 ++++++ golem_core/core/props_cons/constraints.py | 79 ++++++++++ .../core/props_cons/parsers/__init__.py | 0 golem_core/core/props_cons/parsers/base.py | 9 ++ .../core/props_cons/parsers/textx/__init__.py | 0 .../core/props_cons/parsers/textx/parser.py | 7 + golem_core/core/props_cons/properties.py | 19 +++ golem_core/managers/base.py | 5 +- golem_core/managers/negotiation/plugins.py | 12 +- golem_core/managers/negotiation/sequential.py | 15 +- 15 files changed, 252 insertions(+), 176 deletions(-) create mode 100644 golem_core/core/props_cons/__init__.py create mode 100644 golem_core/core/props_cons/base.py create mode 100644 golem_core/core/props_cons/constraints.py create mode 100644 golem_core/core/props_cons/parsers/__init__.py create mode 100644 golem_core/core/props_cons/parsers/base.py create mode 100644 golem_core/core/props_cons/parsers/textx/__init__.py create mode 100644 golem_core/core/props_cons/parsers/textx/parser.py create mode 100644 golem_core/core/props_cons/properties.py diff --git a/golem_core/core/market_api/resources/demand/demand.py b/golem_core/core/market_api/resources/demand/demand.py index b210c213..e15961be 100644 --- a/golem_core/core/market_api/resources/demand/demand.py +++ b/golem_core/core/market_api/resources/demand/demand.py @@ -1,4 +1,5 @@ import asyncio +from dataclasses import dataclass from typing import TYPE_CHECKING, AsyncIterator, Callable, Dict, List, Optional, Union from ya_market import RequestorApi @@ -6,6 +7,8 @@ from golem_core.core.market_api.events import DemandClosed, NewDemand from golem_core.core.market_api.resources.proposal import Proposal +from golem_core.core.props_cons.constraints import Constraints +from golem_core.core.props_cons.properties import Properties from golem_core.core.resources import ( _NULL, Resource, @@ -19,6 +22,15 @@ from golem_core.core.golem_node import GolemNode +@dataclass +class DemandData: + properties: Properties + constraints: Constraints + demand_id: str + requestor_id: str + timestamp: str + + class Demand(Resource[RequestorApi, models.Demand, _NULL, Proposal, _NULL], YagnaEventCollector): """A single demand on the Golem Network. diff --git a/golem_core/core/market_api/resources/demand/demand_builder.py b/golem_core/core/market_api/resources/demand/demand_builder.py index dba02188..48ccaa29 100644 --- a/golem_core/core/market_api/resources/demand/demand_builder.py +++ b/golem_core/core/market_api/resources/demand/demand_builder.py @@ -1,12 +1,11 @@ -import abc from copy import deepcopy -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from ctypes import Union +from typing import TYPE_CHECKING, Optional from golem_core.core.market_api.resources.demand.demand import Demand -from golem_core.core.market_api.resources.demand.demand_offer_base.model import ( - DemandOfferBaseModel, - join_str_constraints, -) +from golem_core.core.market_api.resources.demand.demand_offer_base.model import DemandOfferBaseModel +from golem_core.core.props_cons.constraints import Constraint, ConstraintGroup, Constraints +from golem_core.core.props_cons.properties import Properties if TYPE_CHECKING: # pragma: no cover from golem_core.core.golem_node import GolemNode @@ -34,10 +33,10 @@ class DemandBuilder: """ def __init__( - self, properties: Optional[Dict[str, Any]] = None, constraints: Optional[List[str]] = None + self, properties: Optional[Properties] = None, constraints: Optional[Constraints] = None ): - self._properties: Dict[str, Any] = properties if properties is not None else {} - self._constraints: List[str] = constraints if constraints is not None else [] + self._properties: Properties = properties if properties is not None else Properties() + self._constraints: Constraints = constraints if constraints is not None else Constraints() def __repr__(self): return repr({"properties": self._properties, "constraints": self._constraints}) @@ -46,18 +45,18 @@ def __eq__(self, other): return ( isinstance(other, DemandBuilder) and self._properties == other.properties - and self.constraints == other.constraints + and self._constraints == other.constraints ) @property - def properties(self) -> Dict: - """List of properties for this demand.""" + def properties(self) -> Properties: + """Collection of acumulated Properties.""" return self._properties @property - def constraints(self) -> str: - """Constraints definition for this demand.""" - return join_str_constraints(self._constraints) + def constraints(self) -> Constraints: + """Collection of acumulated Constraints.""" + return self._constraints async def add(self, model: DemandOfferBaseModel): """Add properties and constraints from the given model to this demand definition.""" @@ -67,19 +66,13 @@ async def add(self, model: DemandOfferBaseModel): self.add_properties(properties) self.add_constraints(constraints) - def add_properties(self, props: Dict): + def add_properties(self, props: Properties): """Add properties from the given dictionary to this demand definition.""" self._properties.update(props) - def add_constraints(self, *constraints: str): + def add_constraints(self, constraints: Union[Constraint, ConstraintGroup]): """Add a constraint from given args to the demand definition.""" - self._constraints.extend(constraints) - - async def decorate(self, *decorators: "DemandBuilderDecorator"): - """Decorate demand definition with given demand decorators.""" - - for decorator in decorators: - await decorator.decorate_demand_builder(self) + self._constraints.items.extend(constraints) async def create_demand(self, node: "GolemNode") -> "Demand": """Create demand and subscribe to its events.""" @@ -92,15 +85,3 @@ async def from_demand(cls, demand: "Demand") -> "DemandBuilder": demand_data = deepcopy(await demand.get_data()) return cls(demand_data.properties, [demand_data.constraints]) - - -class DemandBuilderDecorator(abc.ABC): - """An interface that specifies classes that can add properties and constraints through a \ - DemandBuilder.""" - - @abc.abstractmethod - async def decorate_demand_builder(self, demand_builder: DemandBuilder) -> None: - """Decorate given DemandBuilder. - - Intended to be overriden to customize given DemandBuilder decoration. - """ diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/model.py b/golem_core/core/market_api/resources/demand/demand_offer_base/model.py index 49fac626..c2e63d33 100644 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/model.py +++ b/golem_core/core/market_api/resources/demand/demand_offer_base/model.py @@ -1,15 +1,13 @@ import abc import dataclasses -import datetime import enum -import inspect -from typing import Any, Dict, Final, Iterable, List, Literal, Tuple, Type, TypeVar +from typing import Any, Dict, Final, List, Tuple, Type, TypeVar from golem_core.core.market_api.resources.demand.demand_offer_base.exceptions import ( - ConstraintException, InvalidPropertiesError, ) -from golem_core.utils.typing import match_type_union_aware +from golem_core.core.props_cons.constraints import Constraint, ConstraintOperator, Constraints +from golem_core.core.props_cons.properties import Properties TDemandOfferBaseModel = TypeVar("TDemandOfferBaseModel", bound="DemandOfferBaseModel") @@ -17,9 +15,6 @@ PROP_OPERATOR: Final[str] = "operator" PROP_MODEL_FIELD_TYPE: Final[str] = "model_field_type" -ConstraintOperator = Literal["=", ">=", "<="] -ConstraintGroupOperator = Literal["&", "|", "!"] - class DemandOfferBaseModelFieldType(enum.Enum): constraint = "constraint" @@ -37,87 +32,29 @@ class DemandOfferBaseModel(abc.ABC): def __init__(self, **kwargs): # pragma: no cover pass - async def serialize(self) -> Tuple[Dict[str, Any], str]: - """Return a tuple of serialized properties and constraints. - - Intended to be overriden with additional logic that requires async context. - """ - return self._serialize_properties(), self._serialize_constraints() + async def build_properties_and_constraints(self) -> Tuple[Properties, Constraints]: + return self._build_properties(), self._build_constraints() - def _serialize_properties(self) -> Dict[str, Any]: - """Return a serialized collection of property values.""" - return { - field.metadata[PROP_KEY]: self._serialize_property(getattr(self, field.name), field) - for field in self._get_fields(DemandOfferBaseModelFieldType.property) - if getattr(self, field.name) is not None - } + def _build_properties(self) -> Properties: + """Return a collaction of properties declated in model.""" + return Properties( + { + field.metadata[PROP_KEY]: getattr(self, field.name) + for field in self._get_fields(DemandOfferBaseModelFieldType.property) + } + ) - def _serialize_constraints(self) -> str: + def _build_constraints(self) -> Constraints: """Return a serialized collection of constraint values.""" - return join_str_constraints( - self._serialize_constraint(getattr(self, field.name), field) + return Constraints( + Constraint( + property_path=field.metadata[PROP_KEY], + operator=field.metadata[PROP_OPERATOR], + value=getattr(self, field.name), + ) for field in self._get_fields(DemandOfferBaseModelFieldType.constraint) - if getattr(self, field.name) is not None ) - @classmethod - def _serialize_property(cls, value: Any, field: dataclasses.Field) -> Any: - """Return serialized property value.""" - return cls.serialize_value(value) - - @classmethod - def _serialize_constraint(cls, value: Any, field: dataclasses.Field) -> str: - """Return serialized constraint value.""" - if isinstance(value, (list, tuple)): - if value: - return join_str_constraints([cls._serialize_constraint(v, field) for v in value]) - - return "" - - serialized_value = cls.serialize_value(value) - - return "({key}{operator}{value})".format( - key=field.metadata[PROP_KEY], - operator=field.metadata[PROP_OPERATOR], - value=serialized_value, - ) - - @classmethod - def serialize_value(cls, value: Any) -> Any: - """Return value in primitive format compatible with Golem's property and constraint syntax. - - Intended to be overriden with additional type serialisation methods. - """ - - if isinstance(value, (list, tuple)): - return type(value)(cls.serialize_value(v) for v in value) - - if isinstance(value, datetime.datetime): - return int(value.timestamp() * 1000) - - if isinstance(value, enum.Enum): - return value.value - - return value - - @classmethod - def deserialize_value(cls, value: Any, field: dataclasses.Field) -> Any: - """Return proper value for field from given primitive. - - Intended to be overriden with additional type serialisation methods. - """ - if matched_type := match_type_union_aware( - field.type, lambda t: inspect.isclass(t) and issubclass(t, datetime.datetime) - ): - return matched_type.fromtimestamp(int(float(value) * 0.001), datetime.timezone.utc) - - if matched_type := match_type_union_aware( - field.type, lambda t: inspect.isclass(t) and issubclass(t, enum.Enum) - ): - return matched_type(value) - - return value - @classmethod def _get_fields(cls, field_type: DemandOfferBaseModelFieldType) -> List[dataclasses.Field]: """Return a list of fields based on given type.""" @@ -227,54 +164,8 @@ def constraint( ) -def join_str_constraints( - constraints: Iterable[str], operator: ConstraintGroupOperator = "&" -) -> str: - """Join a list of constraints using the given opererator. - - The semantics here reflect LDAP filters: https://ldap.com/ldap-filters/ - - :param constraints: list of strings representing individual constraints - (which may include previously joined constraint groups) - :param operator: constraint group operator, one of "&", "|", "!", which represent - "and", "or" and "not" operations on those constraints. - "!" requires that the list contains one and only one constraint. - Defaults to "&" (and) if not given. - :return: string representation of the compound constraint. - - example: - ```python - >>> from dataclasses import dataclass - >>> from golem_core.core.market_api import join_str_constraints - >>> - >>> min_bar = '(bar>=42)' - >>> max_bar = '(bar<=128)' - >>> print(join_str_constraints([min_bar, max_bar])) - (&(bar>=42) - (bar<=128)) - ``` - """ - constraints = [c for c in constraints if c] - - if operator == "!": - if len(constraints) == 1: - return f"({operator}{constraints[0]})" - else: - raise ConstraintException(f"{operator} requires exactly one component.") - - if not constraints: - return f"({operator})" - - if len(constraints) == 1: - return f"{constraints[0]}" - - rules = "\n\t".join(constraints) - return f"({operator}{rules})" - - __all__ = ( "DemandOfferBaseModel", "prop", "constraint", - "join_str_constraints", ) diff --git a/golem_core/core/market_api/resources/proposal.py b/golem_core/core/market_api/resources/proposal.py index 07a98baa..b027da72 100644 --- a/golem_core/core/market_api/resources/proposal.py +++ b/golem_core/core/market_api/resources/proposal.py @@ -1,5 +1,7 @@ import asyncio +from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from enum import StrEnum from typing import TYPE_CHECKING, AsyncIterator, Dict, Optional, Union from ya_market import RequestorApi @@ -7,6 +9,8 @@ from golem_core.core.market_api.events import NewProposal from golem_core.core.market_api.resources.agreement import Agreement +from golem_core.core.props_cons.constraints import Constraints +from golem_core.core.props_cons.properties import Properties from golem_core.core.resources import Resource from golem_core.core.resources.base import TModel, api_call_wrapper @@ -15,6 +19,25 @@ from golem_core.core.market_api.resources.demand import Demand +class ProposalState(StrEnum): + INITIDAL = "Initial" + DRAFT = "Draft" + REJECTED = "Rejected" + ACCPETED = "Accepted" + EXPIRED = "Expired" + + +@dataclass +class ProposalData: + properties: Properties + constraints: Constraints + proposal_id: str + issuer_id: str + state: ProposalState + timestamp: datetime + prev_proposal_id: Optional[str] + + class Proposal( Resource[ RequestorApi, diff --git a/golem_core/core/props_cons/__init__.py b/golem_core/core/props_cons/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/golem_core/core/props_cons/base.py b/golem_core/core/props_cons/base.py new file mode 100644 index 00000000..b25d03e4 --- /dev/null +++ b/golem_core/core/props_cons/base.py @@ -0,0 +1,45 @@ +import datetime +import enum +import inspect +from dataclasses import Field +from typing import Any + +from golem_core.utils.typing import match_type_union_aware + +PropertyPath = str +PropertyValue = Any + + +class PropsConstrsSerializer: + @classmethod + def serialize_value(cls, value: Any) -> Any: + """Return value in primitive format compatible with Golem's property and constraint syntax.""" + + if isinstance(value, (list, tuple)): + return type(value)(cls.serialize_value(v) for v in value) + + if isinstance(value, datetime.datetime): + return int(value.timestamp() * 1000) + + if isinstance(value, enum.Enum): + return value.value + + return value + + @classmethod + def deserialize_value(cls, value: Any, field: Field) -> Any: + """Return proper value for field from given primitive. + + Intended to be overriden with additional type serialisation methods. + """ + if matched_type := match_type_union_aware( + field.type, lambda t: inspect.isclass(t) and issubclass(t, datetime.datetime) + ): + return matched_type.fromtimestamp(int(float(value) * 0.001), datetime.timezone.utc) + + if matched_type := match_type_union_aware( + field.type, lambda t: inspect.isclass(t) and issubclass(t, enum.Enum) + ): + return matched_type(value) + + return value diff --git a/golem_core/core/props_cons/constraints.py b/golem_core/core/props_cons/constraints.py new file mode 100644 index 00000000..22391088 --- /dev/null +++ b/golem_core/core/props_cons/constraints.py @@ -0,0 +1,79 @@ +from abc import ABC, abstractmethod +from ctypes import Union +from dataclasses import dataclass, field +from enum import StrEnum +from typing import Any, MutableSequence + +from golem_core.core.props_cons.base import PropertyPath + + +class ConstraintException(Exception): + pass + + +class ConstraintOperator(StrEnum): + EQUALS = "=" + GRATER_OR_EQUALS = ">=" + LESS_OR_EQUALS = "<=" + + +class ConstraintGroupOperator(StrEnum): + AND = "&" + OR = "|" + NOT = "!" + + +@dataclass +class MarketDemandOfferSyntaxElement(ABC): + def __post_init__(self) -> None: + self._validate() + + def serialize(self) -> str: + self._validate() + + return self._serialize() + + @abstractmethod + def _serialize(self) -> str: + ... + + def _validate(self) -> None: + ... + + +@dataclass +class Constraint(MarketDemandOfferSyntaxElement): + property_path: PropertyPath + operator: ConstraintOperator + value: Any + + def _serialize(self) -> str: + return f"({self.property_path}{self.operator}{self.value})" + + +@dataclass +class ConstraintGroup(MarketDemandOfferSyntaxElement): + items: MutableSequence[Union["ConstraintGroup", Constraint]] = field(default_factory=list) + operator: ConstraintGroupOperator = "&" + + def _validate(self) -> None: + if self.operator == "!" and 2 <= len(self.items): + return ConstraintException("ConstraintGroup with `!` operator can contain only 1 item!") + + def _serialize(self) -> str: + items_len = len(self.items) + + if items_len == 0: + return f"({self.operator})" + + if items_len == 1: + return self.items[0].serialize() + + items = "\n\t".join(item.serialize() for item in self.items) + + return f"({self.operator}{items})" + + +@dataclass +class Constraints(ConstraintGroup): + pass diff --git a/golem_core/core/props_cons/parsers/__init__.py b/golem_core/core/props_cons/parsers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/golem_core/core/props_cons/parsers/base.py b/golem_core/core/props_cons/parsers/base.py new file mode 100644 index 00000000..7aa6f2fe --- /dev/null +++ b/golem_core/core/props_cons/parsers/base.py @@ -0,0 +1,9 @@ +from abc import ABC, abstractmethod + +from golem_core.core.props_cons.constraints import Constraints + + +class DemandOfferSyntaxParser(ABC): + @abstractmethod + def parse(self, syntax: str) -> Constraints: + ... diff --git a/golem_core/core/props_cons/parsers/textx/__init__.py b/golem_core/core/props_cons/parsers/textx/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/golem_core/core/props_cons/parsers/textx/parser.py b/golem_core/core/props_cons/parsers/textx/parser.py new file mode 100644 index 00000000..111c6312 --- /dev/null +++ b/golem_core/core/props_cons/parsers/textx/parser.py @@ -0,0 +1,7 @@ +from golem_core.core.props_cons.constraints import Constraints +from golem_core.core.props_cons.parsers.base import DemandOfferSyntaxParser + + +class TextXDemandOfferSyntaxParser(DemandOfferSyntaxParser): + def parse(self, syntax: str) -> Constraints: + return Constraints() diff --git a/golem_core/core/props_cons/properties.py b/golem_core/core/props_cons/properties.py new file mode 100644 index 00000000..68547c5b --- /dev/null +++ b/golem_core/core/props_cons/properties.py @@ -0,0 +1,19 @@ +from copy import deepcopy +from typing import Any, Mapping + +from golem_core.core.props_cons.base import PropsConstrsSerializer + + +class Properties(PropsConstrsSerializer, dict): + """Low level wrapper class for Golem's Market API properties manipulation.""" + + def __init__(self, mapping, /) -> None: + mapping_deep_copy = deepcopy(mapping) + super(mapping_deep_copy) + + def serialize(self) -> Mapping[str, Any]: + """Serialize complex objects into format handled by Market API properties specification.""" + return {key: self._serialize_property(value) for key, value in self.items()} + + def _serialize_property(self, value: Any) -> Any: + return self._serialize_value(value) diff --git a/golem_core/managers/base.py b/golem_core/managers/base.py index 6e6cf685..ddf6e047 100644 --- a/golem_core/managers/base.py +++ b/golem_core/managers/base.py @@ -3,7 +3,8 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional, Union from golem_core.core.activity_api import Activity, Script, commands -from golem_core.core.market_api import Agreement, DemandBuilder, Proposal +from golem_core.core.market_api import Agreement, Proposal +from golem_core.core.market_api.resources.proposal import ProposalData from golem_core.core.payment_api import Allocation from golem_core.core.resources import ResourceEvent from golem_core.exceptions import BaseGolemException @@ -147,5 +148,5 @@ class WorkManager(Manager, ABC): class NegotiationPlugin(ABC): @abstractmethod - async def __call__(self, demand_builder: DemandBuilder, proposal: Proposal) -> DemandBuilder: + async def __call__(self, demand_data: ProposalData, offer_data: ProposalData) -> None: ... diff --git a/golem_core/managers/negotiation/plugins.py b/golem_core/managers/negotiation/plugins.py index 998c5463..6bf22998 100644 --- a/golem_core/managers/negotiation/plugins.py +++ b/golem_core/managers/negotiation/plugins.py @@ -1,7 +1,7 @@ import logging from typing import Sequence -from golem_core.core.market_api import DemandBuilder, Proposal +from golem_core.core.market_api.resources.proposal import ProposalData from golem_core.managers.base import NegotiationPlugin from golem_core.managers.negotiation.sequential import RejectProposal @@ -12,10 +12,12 @@ class BlacklistProviderId(NegotiationPlugin): def __init__(self, blacklist: Sequence[str]) -> None: self._blacklist = blacklist - async def __call__(self, demand_builder: DemandBuilder, proposal: Proposal) -> DemandBuilder: + async def __call__( + self, demand_proposal_data: ProposalData, offer_proposal_data: ProposalData + ) -> None: logger.debug("Calling blacklist plugin...") - provider_id = proposal.data.issuer_id + provider_id = offer_proposal_data.issuer_id if provider_id in self._blacklist: logger.debug( @@ -27,4 +29,6 @@ async def __call__(self, demand_builder: DemandBuilder, proposal: Proposal) -> D f"Calling blacklist plugin done with provider `{provider_id}` is not blacklisted" ) - return demand_builder + +class MidAgreementPayment(NegotiationPlugin): + pass diff --git a/golem_core/managers/negotiation/sequential.py b/golem_core/managers/negotiation/sequential.py index a8ad1229..6320e8ee 100644 --- a/golem_core/managers/negotiation/sequential.py +++ b/golem_core/managers/negotiation/sequential.py @@ -7,6 +7,7 @@ from golem_core.core.golem_node.golem_node import DEFAULT_EXPIRATION_TIMEOUT, SUBNET, GolemNode from golem_core.core.market_api import Demand, DemandBuilder, Payload, Proposal from golem_core.core.market_api.resources.demand.demand_offer_base import defaults as dobm_defaults +from golem_core.core.market_api.resources.proposal import ProposalData from golem_core.core.payment_api import Allocation from golem_core.managers.base import ManagerException, NegotiationManager, NegotiationPlugin @@ -84,7 +85,7 @@ async def _negotiation_loop(self, payload: Payload) -> None: demand.start_collecting_events() try: - async for proposal in self._negotiate(demand_builder, demand): + async for proposal in self._negotiate(demand): await self._eligible_proposals.put(proposal) finally: await demand.unsubscribe() @@ -119,11 +120,11 @@ async def _prepare_demand_builder( return demand_builder - async def _negotiate( - self, demand_builder: DemandBuilder, demand: Demand - ) -> AsyncIterator[Proposal]: + async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: + demand_data = self._get_demand_data_from_demand(demand) + async for initial_offer_proposal in demand.initial_proposals(): - offer_proposal = await self._negotiate_proposal(demand_builder, initial_offer_proposal) + offer_proposal = await self._negotiate_proposal(demand_data, initial_offer_proposal) if offer_proposal is None: logger.debug( @@ -189,3 +190,7 @@ async def _negotiate_proposal( logger.debug(f"Negotiating proposal `{offer_proposal}` done") return offer_proposal + + def _get_proposal_data_from_demand(self, demand: Demand) -> ProposalData: + # FIXME: Unnecessary serialisation from DemandBuilder to Demand, and from Demand to ProposalData + return ProposalData(demand.data.properties) From 0198123204cffe5dbfb03b5fb213924e523b59cd Mon Sep 17 00:00:00 2001 From: approxit Date: Fri, 9 Jun 2023 16:52:52 +0200 Subject: [PATCH 054/123] initial constraints parser --- golem_core/core/golem_node/golem_node.py | 2 + .../resources/demand/demand_builder.py | 70 ++++++----- .../demand/demand_offer_base/model.py | 2 +- .../core/payment_api/resources/allocation.py | 14 ++- golem_core/core/props_cons/base.py | 2 +- golem_core/core/props_cons/constraints.py | 17 ++- .../core/props_cons/parsers/textx/parser.py | 20 ++- .../core/props_cons/parsers/textx/syntax.tx | 39 ++++++ golem_core/managers/negotiation/sequential.py | 12 +- pyproject.toml | 2 + tests/unit/parsers/__init__.py | 0 tests/unit/parsers/test_parsers.py | 114 ++++++++++++++++++ 12 files changed, 246 insertions(+), 48 deletions(-) create mode 100644 golem_core/core/props_cons/parsers/textx/syntax.tx create mode 100644 tests/unit/parsers/__init__.py create mode 100644 tests/unit/parsers/test_parsers.py diff --git a/golem_core/core/golem_node/golem_node.py b/golem_core/core/golem_node/golem_node.py index 55ea6303..0992196a 100644 --- a/golem_core/core/golem_node/golem_node.py +++ b/golem_core/core/golem_node/golem_node.py @@ -19,6 +19,7 @@ Invoice, InvoiceEventCollector, ) +from golem_core.core.props_cons.parsers.textx.parser import TextXDemandOfferSyntaxParser from golem_core.core.resources import ApiConfig, ApiFactory, Resource, TResource PAYMENT_DRIVER: str = os.getenv("YAGNA_PAYMENT_DRIVER", "erc20").lower() @@ -84,6 +85,7 @@ def __init__( self._resources: DefaultDict[Type[Resource], Dict[str, Resource]] = defaultdict(dict) self._autoclose_resources: Set[Resource] = set() self._event_bus = InMemoryEventBus() + self._demand_offer_syntax_parser = TextXDemandOfferSyntaxParser() self._invoice_event_collector = InvoiceEventCollector(self) self._debit_note_event_collector = DebitNoteEventCollector(self) diff --git a/golem_core/core/market_api/resources/demand/demand_builder.py b/golem_core/core/market_api/resources/demand/demand_builder.py index 48ccaa29..456d4300 100644 --- a/golem_core/core/market_api/resources/demand/demand_builder.py +++ b/golem_core/core/market_api/resources/demand/demand_builder.py @@ -1,9 +1,12 @@ -from copy import deepcopy from ctypes import Union -from typing import TYPE_CHECKING, Optional +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Iterable, Optional +from golem_core.core.golem_node.golem_node import DEFAULT_EXPIRATION_TIMEOUT, SUBNET from golem_core.core.market_api.resources.demand.demand import Demand +from golem_core.core.market_api.resources.demand.demand_offer_base import defaults as dobm_defaults from golem_core.core.market_api.resources.demand.demand_offer_base.model import DemandOfferBaseModel +from golem_core.core.payment_api.resources.allocation import Allocation from golem_core.core.props_cons.constraints import Constraint, ConstraintGroup, Constraints from golem_core.core.props_cons.properties import Properties @@ -35,53 +38,66 @@ class DemandBuilder: def __init__( self, properties: Optional[Properties] = None, constraints: Optional[Constraints] = None ): - self._properties: Properties = properties if properties is not None else Properties() - self._constraints: Constraints = constraints if constraints is not None else Constraints() + self.properties: Properties = properties if properties is not None else Properties() + self.constraints: Constraints = constraints if constraints is not None else Constraints() def __repr__(self): - return repr({"properties": self._properties, "constraints": self._constraints}) + return repr({"properties": self.properties, "constraints": self.constraints}) def __eq__(self, other): return ( isinstance(other, DemandBuilder) - and self._properties == other.properties - and self._constraints == other.constraints + and self.properties == other.properties + and self.constraints == other.constraints ) - @property - def properties(self) -> Properties: - """Collection of acumulated Properties.""" - return self._properties - - @property - def constraints(self) -> Constraints: - """Collection of acumulated Constraints.""" - return self._constraints - async def add(self, model: DemandOfferBaseModel): """Add properties and constraints from the given model to this demand definition.""" - properties, constraints = await model.serialize() + properties, constraints = await model.build_properties_and_constraints() self.add_properties(properties) self.add_constraints(constraints) def add_properties(self, props: Properties): """Add properties from the given dictionary to this demand definition.""" - self._properties.update(props) + self.properties.update(props) - def add_constraints(self, constraints: Union[Constraint, ConstraintGroup]): + def add_constraints(self, *constraints: Union[Constraint, ConstraintGroup]): """Add a constraint from given args to the demand definition.""" - self._constraints.items.extend(constraints) + self.constraints.items.extend(constraints) + + async def add_default_parameters( + self, + subnet: Optional[str] = SUBNET, + expiration: Optional[datetime] = None, + allocations: Iterable[Allocation] = (), + ) -> None: + """Subscribe a new demand. + + :param payload: Details of the demand + :param subnet: Subnet tag + :param expiration: Timestamp when all agreements based on this demand will expire + TODO: is this correct? + :param allocations: Allocations that will be included in the description of this demand. + :param autoclose: Unsubscribe demand on :func:`__aexit__` + :param autostart: Immediately start collecting yagna events for this :any:`Demand`. + Without autostart events for this demand will start being collected after a call to + :func:`Demand.start_collecting_events`. + """ + if expiration is None: + expiration = datetime.now(timezone.utc) + DEFAULT_EXPIRATION_TIMEOUT + + await self.add(dobm_defaults.Activity(expiration=expiration, multi_activity=True)) + await self.add(dobm_defaults.NodeInfo(subnet_tag=subnet)) + + for allocation in allocations: + properties, constraints = await allocation.get_properties_and_constraints_for_demand() + self.add_constraints(constraints) + self.add_properties(properties) async def create_demand(self, node: "GolemNode") -> "Demand": """Create demand and subscribe to its events.""" return await Demand.create_from_properties_constraints( node, self.properties, self.constraints ) - - @classmethod - async def from_demand(cls, demand: "Demand") -> "DemandBuilder": - demand_data = deepcopy(await demand.get_data()) - - return cls(demand_data.properties, [demand_data.constraints]) diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/model.py b/golem_core/core/market_api/resources/demand/demand_offer_base/model.py index c2e63d33..0c1cba92 100644 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/model.py +++ b/golem_core/core/market_api/resources/demand/demand_offer_base/model.py @@ -48,7 +48,7 @@ def _build_constraints(self) -> Constraints: """Return a serialized collection of constraint values.""" return Constraints( Constraint( - property_path=field.metadata[PROP_KEY], + property_name=field.metadata[PROP_KEY], operator=field.metadata[PROP_OPERATOR], value=getattr(self, field.name), ) diff --git a/golem_core/core/payment_api/resources/allocation.py b/golem_core/core/payment_api/resources/allocation.py index 81d87608..ade26ea7 100644 --- a/golem_core/core/payment_api/resources/allocation.py +++ b/golem_core/core/payment_api/resources/allocation.py @@ -7,6 +7,8 @@ from golem_core.core.payment_api.events import NewAllocation from golem_core.core.payment_api.exceptions import NoMatchingAccount +from golem_core.core.props_cons.constraints import Constraints +from golem_core.core.props_cons.properties import Properties from golem_core.core.resources import _NULL, Resource, ResourceClosed, api_call_wrapper from golem_core.core.resources.base import TModel @@ -86,6 +88,16 @@ async def create(cls, node: "GolemNode", data: models.Allocation) -> "Allocation return cls(node, created.allocation_id, created) @api_call_wrapper() - async def demand_properties_constraints(self) -> Tuple[List[models.MarketProperty], List[str]]: + async def get_properties_and_constraints_for_demand(self) -> Tuple[Properties, Constraints]: data = await self.api.get_demand_decorations([self.id]) + + properties = Properties({ + prop.name: prop.value + for prop in data.properties + }) + + print(data.constraints) + + constraints = self._node._demand_offer_syntax_parser.parse() + return data.properties, data.constraints diff --git a/golem_core/core/props_cons/base.py b/golem_core/core/props_cons/base.py index b25d03e4..b4dba702 100644 --- a/golem_core/core/props_cons/base.py +++ b/golem_core/core/props_cons/base.py @@ -6,7 +6,7 @@ from golem_core.utils.typing import match_type_union_aware -PropertyPath = str +PropertyName = str PropertyValue = Any diff --git a/golem_core/core/props_cons/constraints.py b/golem_core/core/props_cons/constraints.py index 22391088..7fdff7c5 100644 --- a/golem_core/core/props_cons/constraints.py +++ b/golem_core/core/props_cons/constraints.py @@ -1,23 +1,22 @@ from abc import ABC, abstractmethod -from ctypes import Union from dataclasses import dataclass, field -from enum import StrEnum -from typing import Any, MutableSequence +from enum import Enum +from typing import Any, MutableSequence, Union -from golem_core.core.props_cons.base import PropertyPath +from golem_core.core.props_cons.base import PropertyName class ConstraintException(Exception): pass -class ConstraintOperator(StrEnum): +class ConstraintOperator(Enum): EQUALS = "=" GRATER_OR_EQUALS = ">=" LESS_OR_EQUALS = "<=" -class ConstraintGroupOperator(StrEnum): +class ConstraintGroupOperator(Enum): AND = "&" OR = "|" NOT = "!" @@ -43,12 +42,12 @@ def _validate(self) -> None: @dataclass class Constraint(MarketDemandOfferSyntaxElement): - property_path: PropertyPath + property_name: PropertyName operator: ConstraintOperator value: Any def _serialize(self) -> str: - return f"({self.property_path}{self.operator}{self.value})" + return f"({self.property_name}{self.operator}{self.value})" @dataclass @@ -58,7 +57,7 @@ class ConstraintGroup(MarketDemandOfferSyntaxElement): def _validate(self) -> None: if self.operator == "!" and 2 <= len(self.items): - return ConstraintException("ConstraintGroup with `!` operator can contain only 1 item!") + raise ConstraintException("ConstraintGroup with `!` operator can contain only 1 item!") def _serialize(self) -> str: items_len = len(self.items) diff --git a/golem_core/core/props_cons/parsers/textx/parser.py b/golem_core/core/props_cons/parsers/textx/parser.py index 111c6312..4257797a 100644 --- a/golem_core/core/props_cons/parsers/textx/parser.py +++ b/golem_core/core/props_cons/parsers/textx/parser.py @@ -1,7 +1,23 @@ -from golem_core.core.props_cons.constraints import Constraints +from pathlib import Path + +from textx import metamodel_from_file + +from golem_core.core.props_cons.constraints import Constraint, ConstraintGroup, Constraints from golem_core.core.props_cons.parsers.base import DemandOfferSyntaxParser class TextXDemandOfferSyntaxParser(DemandOfferSyntaxParser): + def __init__(self): + self._metamodel = metamodel_from_file(Path(__file__).with_name("syntax.tx")) + self._metamodel.register_obj_processors( + { + "ConstraintGroup": lambda e: ConstraintGroup(e.items, e.operator), + "Constraint": lambda e: Constraint(e.property_path, e.operator, e.value), + "ProeprtyValueList": lambda e: e.items, + } + ) + def parse(self, syntax: str) -> Constraints: - return Constraints() + model = self._metamodel.model_from_str(syntax) + + return model.constraints diff --git a/golem_core/core/props_cons/parsers/textx/syntax.tx b/golem_core/core/props_cons/parsers/textx/syntax.tx new file mode 100644 index 00000000..cc9b6ba9 --- /dev/null +++ b/golem_core/core/props_cons/parsers/textx/syntax.tx @@ -0,0 +1,39 @@ +Model: + constraints=ConstraintGroup +; + +ConstraintGroup: + "(" operator=ConstraintGroupOperator items*=ConstraintGroupOrSingle ")" +; + +Constraint: + "(" property_path=PropertyName operator=ConstraintOperator value=PropertyValue ")" +; + +ConstraintGroupOrSingle: + ConstraintGroup | Constraint +; + +ConstraintGroupOperator: + "&" | "|" | "!" +; + +ConstraintOperator: + "=" | ">=" | "<=" +; + +PropertyName: + /\w+(\.\w+)*/ +; + +ProeprtyValueList: + "[" items*=PropertyValue[","] "]" +; + +PropertyValue: + PropertyValueStr | ProeprtyValueList +; + +PropertyValueStr: + /[a-zA-Z0-9_.\/:;]+/ +; \ No newline at end of file diff --git a/golem_core/managers/negotiation/sequential.py b/golem_core/managers/negotiation/sequential.py index 6320e8ee..40f63576 100644 --- a/golem_core/managers/negotiation/sequential.py +++ b/golem_core/managers/negotiation/sequential.py @@ -57,7 +57,7 @@ async def start(self) -> None: logger.debug(f"Starting failed with `{message}`") raise ManagerException(message) - self._negotiation_loop_task = asyncio.create_task(self._negotiation_loop(self._payload)) + self._negotiation_loop_task = asyncio.create_task(self._negotiation_loop()) logger.debug("Starting done") @@ -77,9 +77,9 @@ async def stop(self) -> None: def is_started_started(self) -> bool: return self._negotiation_loop_task is not None - async def _negotiation_loop(self, payload: Payload) -> None: + async def _negotiation_loop(self) -> None: allocation = await self._get_allocation() - demand_builder = await self._prepare_demand_builder(allocation, payload) + demand_builder = await self._prepare_demand_builder(allocation) demand = await demand_builder.create_demand(self._golem) demand.start_collecting_events() @@ -90,9 +90,7 @@ async def _negotiation_loop(self, payload: Payload) -> None: finally: await demand.unsubscribe() - async def _prepare_demand_builder( - self, allocation: Allocation, payload: Payload - ) -> DemandBuilder: + async def _prepare_demand_builder(self, allocation: Allocation) -> DemandBuilder: logger.debug("Preparing demand...") # FIXME: Code looks duplicated as GolemNode.create_demand does the same @@ -107,7 +105,7 @@ async def _prepare_demand_builder( ) await demand_builder.add(dobm_defaults.NodeInfo(subnet_tag=SUBNET)) - await demand_builder.add(payload) + await demand_builder.add(self._payload) ( allocation_properties, diff --git a/pyproject.toml b/pyproject.toml index cbec423c..ebd87ea3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,8 @@ jsonrpc-base = "^1.0.3" srvresolver = "^0.3.5" semantic-version = "^2.8" async-exit-stack = "1.0.1" +textx = "^3.1.1" +setuptools = "*" # textx external dependency [tool.poetry.group.dev.dependencies] pytest = "^7" diff --git a/tests/unit/parsers/__init__.py b/tests/unit/parsers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/parsers/test_parsers.py b/tests/unit/parsers/test_parsers.py new file mode 100644 index 00000000..441ffc94 --- /dev/null +++ b/tests/unit/parsers/test_parsers.py @@ -0,0 +1,114 @@ +import pytest + +from golem_core.core.props_cons.constraints import Constraint, ConstraintException, ConstraintGroup +from golem_core.core.props_cons.parsers.textx.parser import TextXDemandOfferSyntaxParser + + +@pytest.fixture(scope="module") +def demand_offer_parser(): + return TextXDemandOfferSyntaxParser() + + +@pytest.mark.parametrize( + "input_string, output", + ( + ("(&)", ConstraintGroup(operator="&")), + ("(|)", ConstraintGroup(operator="|")), + ("(!)", ConstraintGroup(operator="!")), + ), +) +def test_empty_group(demand_offer_parser, input_string, output): + result = demand_offer_parser.parse(input_string) + + assert result == output + + +@pytest.mark.parametrize( + "input_string, output", + ( + ("(& (foo=1))", ConstraintGroup([Constraint("foo", "=", "1")])), + ("(& (float.value=1.5))", ConstraintGroup([Constraint("float.value", "=", "1.5")])), + ("(& (foo=bar))", ConstraintGroup([Constraint("foo", "=", "bar")])), + ( + "(& (foo=more.complex.value))", + ConstraintGroup([Constraint("foo", "=", "more.complex.value")]), + ), + ( + "(& (foo=http://google.com))", + ConstraintGroup([Constraint("foo", "=", "http://google.com")]), + ), + ("(& (foo=[1, 2, 3]))", ConstraintGroup([Constraint("foo", "=", ["1", "2", "3"])])), + ("(& (foo=[a, b, c]))", ConstraintGroup([Constraint("foo", "=", ["a", "b", "c"])])), + ("(& (some.nested.param=1))", ConstraintGroup([Constraint("some.nested.param", "=", "1")])), + ("(& (foo<=1))", ConstraintGroup([Constraint("foo", "<=", "1")])), + ("(& (foo>=1))", ConstraintGroup([Constraint("foo", ">=", "1")])), + ), +) +def test_single_constraint(demand_offer_parser, input_string, output): + result = demand_offer_parser.parse(input_string) + + assert result == output + + +@pytest.mark.parametrize( + "input_string, output", + ( + ( + "(& (foo=1) (bar=1))", + ConstraintGroup([Constraint("foo", "=", "1"), Constraint("bar", "=", "1")]), + ), + ( + "(& (even=1) (more=2) (values=3))", + ConstraintGroup( + [ + Constraint("even", "=", "1"), + Constraint("more", "=", "2"), + Constraint("values", "=", "3"), + ] + ), + ), + ( + "(| (foo=1) (bar=2))", + ConstraintGroup([Constraint("foo", "=", "1"), Constraint("bar", "=", "2")], "|"), + ), + ), +) +def test_multiple_constraints(demand_offer_parser, input_string, output): + result = demand_offer_parser.parse(input_string) + + assert result == output + + +def test_error_not_operator_with_multiple_items(demand_offer_parser): + with pytest.raises(ConstraintException): + result = demand_offer_parser.parse("(! (foo=1) (bar=1))") + + +@pytest.mark.parametrize( + "input_string, output", + ( + ( + "(& (& (foo=1) (bar=2)) (baz=3))", + ConstraintGroup( + [ + ConstraintGroup([Constraint("foo", "=", "1"), Constraint("bar", "=", "2")]), + Constraint("baz", "=", "3"), + ] + ), + ), + ( + "(| (& (foo=1) (bar=2)) (& (bat=3) (man=4)))", + ConstraintGroup( + [ + ConstraintGroup([Constraint("foo", "=", "1"), Constraint("bar", "=", "2")]), + ConstraintGroup([Constraint("bat", "=", "3"), Constraint("man", "=", "4")]), + ], + "|", + ), + ), + ), +) +def test_nested_groups(demand_offer_parser, input_string, output): + result = demand_offer_parser.parse(input_string) + + assert result == output From f22801a4990e68638b10af073e2e8fa912c60ce8 Mon Sep 17 00:00:00 2001 From: approxit Date: Thu, 15 Jun 2023 16:50:32 +0200 Subject: [PATCH 055/123] nearly working oop props and cons --- golem_core/core/market_api/__init__.py | 4 -- .../core/market_api/resources/__init__.py | 4 -- .../market_api/resources/demand/__init__.py | 4 -- .../market_api/resources/demand/demand.py | 8 +-- .../resources/demand/demand_builder.py | 30 +++++---- .../demand/demand_offer_base/__init__.py | 2 - .../core/market_api/resources/proposal.py | 11 ++-- .../core/payment_api/resources/allocation.py | 15 +++-- .../core/payment_api/resources/invoice.py | 1 + golem_core/core/props_cons/base.py | 8 +-- golem_core/core/props_cons/constraints.py | 19 +++--- golem_core/core/props_cons/parsers/base.py | 3 + .../core/props_cons/parsers/textx/parser.py | 9 ++- .../core/props_cons/parsers/textx/syntax.tx | 14 ++-- golem_core/core/props_cons/properties.py | 15 +++-- golem_core/managers/base.py | 3 +- golem_core/managers/negotiation/plugins.py | 5 +- golem_core/managers/negotiation/sequential.py | 66 ++++++++++--------- golem_core/utils/asyncio.py | 17 +++++ golem_core/utils/logging.py | 6 +- tests/unit/parsers/__init__.py | 0 tests/unit/test_demand_builder.py | 27 -------- tests/unit/test_demand_builder_model.py | 1 - tests/unit/test_demand_offer_cons.py | 56 ++++++++++++++++ ...arsers.py => test_demand_offer_parsers.py} | 62 +++++++++-------- tests/unit/test_demand_offer_props.py | 57 ++++++++++++++++ 26 files changed, 282 insertions(+), 165 deletions(-) create mode 100644 golem_core/utils/asyncio.py delete mode 100644 tests/unit/parsers/__init__.py create mode 100644 tests/unit/test_demand_offer_cons.py rename tests/unit/{parsers/test_parsers.py => test_demand_offer_parsers.py} (69%) create mode 100644 tests/unit/test_demand_offer_props.py diff --git a/golem_core/core/market_api/__init__.py b/golem_core/core/market_api/__init__.py index d28ae76a..5026bb1e 100644 --- a/golem_core/core/market_api/__init__.py +++ b/golem_core/core/market_api/__init__.py @@ -17,7 +17,6 @@ ConstraintException, Demand, DemandBuilder, - DemandBuilderDecorator, DemandOfferBaseModel, InvalidPropertiesError, ManifestVmPayload, @@ -28,7 +27,6 @@ TDemandOfferBaseModel, VmPayloadException, constraint, - join_str_constraints, prop, ) @@ -43,7 +41,6 @@ "ManifestVmPayload", "VmPayloadException", "RepositoryVmPayload", - "DemandBuilderDecorator", "DemandBuilder", "Payload", "ManifestVmPayload", @@ -58,7 +55,6 @@ "Activity", "TDemandOfferBaseModel", "DemandOfferBaseModel", - "join_str_constraints", "constraint", "prop", "BaseDemandOfferBaseException", diff --git a/golem_core/core/market_api/resources/__init__.py b/golem_core/core/market_api/resources/__init__.py index c1f0fd9e..a540d9e2 100644 --- a/golem_core/core/market_api/resources/__init__.py +++ b/golem_core/core/market_api/resources/__init__.py @@ -10,7 +10,6 @@ ConstraintException, Demand, DemandBuilder, - DemandBuilderDecorator, DemandOfferBaseModel, InvalidPropertiesError, ManifestVmPayload, @@ -20,7 +19,6 @@ TDemandOfferBaseModel, VmPayloadException, constraint, - join_str_constraints, prop, ) from golem_core.core.market_api.resources.proposal import Proposal @@ -33,7 +31,6 @@ "ManifestVmPayload", "VmPayloadException", "RepositoryVmPayload", - "DemandBuilderDecorator", "DemandBuilder", "Payload", "ManifestVmPayload", @@ -48,7 +45,6 @@ "Activity", "TDemandOfferBaseModel", "DemandOfferBaseModel", - "join_str_constraints", "constraint", "prop", "BaseDemandOfferBaseException", diff --git a/golem_core/core/market_api/resources/demand/__init__.py b/golem_core/core/market_api/resources/demand/__init__.py index 2864f72e..1ab5e0e7 100644 --- a/golem_core/core/market_api/resources/demand/__init__.py +++ b/golem_core/core/market_api/resources/demand/__init__.py @@ -1,7 +1,6 @@ from golem_core.core.market_api.resources.demand.demand import Demand from golem_core.core.market_api.resources.demand.demand_builder import ( DemandBuilder, - DemandBuilderDecorator, ) from golem_core.core.market_api.resources.demand.demand_offer_base import ( INF_CPU_THREADS, @@ -21,7 +20,6 @@ TDemandOfferBaseModel, VmPayloadException, constraint, - join_str_constraints, prop, ) @@ -31,7 +29,6 @@ "ManifestVmPayload", "VmPayloadException", "RepositoryVmPayload", - "DemandBuilderDecorator", "DemandBuilder", "Payload", "ManifestVmPayload", @@ -46,7 +43,6 @@ "Activity", "TDemandOfferBaseModel", "DemandOfferBaseModel", - "join_str_constraints", "constraint", "prop", "BaseDemandOfferBaseException", diff --git a/golem_core/core/market_api/resources/demand/demand.py b/golem_core/core/market_api/resources/demand/demand.py index e15961be..c4a7443e 100644 --- a/golem_core/core/market_api/resources/demand/demand.py +++ b/golem_core/core/market_api/resources/demand/demand.py @@ -111,12 +111,12 @@ async def _get_data(self) -> models.Demand: async def create_from_properties_constraints( cls, node: "GolemNode", - properties: Dict[str, str], - constraints: str, + properties: Properties, + constraints: Constraints, ) -> "Demand": data = models.DemandOfferBase( - properties=properties, - constraints=constraints, + properties=properties.serialize(), + constraints=constraints.serialize(), ) return await cls.create(node, data) diff --git a/golem_core/core/market_api/resources/demand/demand_builder.py b/golem_core/core/market_api/resources/demand/demand_builder.py index 456d4300..8f8f5bda 100644 --- a/golem_core/core/market_api/resources/demand/demand_builder.py +++ b/golem_core/core/market_api/resources/demand/demand_builder.py @@ -1,13 +1,13 @@ -from ctypes import Union +from copy import deepcopy from datetime import datetime, timezone -from typing import TYPE_CHECKING, Iterable, Optional +from typing import TYPE_CHECKING, Iterable, Optional, Union -from golem_core.core.golem_node.golem_node import DEFAULT_EXPIRATION_TIMEOUT, SUBNET from golem_core.core.market_api.resources.demand.demand import Demand from golem_core.core.market_api.resources.demand.demand_offer_base import defaults as dobm_defaults from golem_core.core.market_api.resources.demand.demand_offer_base.model import DemandOfferBaseModel from golem_core.core.payment_api.resources.allocation import Allocation from golem_core.core.props_cons.constraints import Constraint, ConstraintGroup, Constraints +from golem_core.core.props_cons.parsers.base import DemandOfferSyntaxParser from golem_core.core.props_cons.properties import Properties if TYPE_CHECKING: # pragma: no cover @@ -38,8 +38,8 @@ class DemandBuilder: def __init__( self, properties: Optional[Properties] = None, constraints: Optional[Constraints] = None ): - self.properties: Properties = properties if properties is not None else Properties() - self.constraints: Constraints = constraints if constraints is not None else Constraints() + self.properties: Properties = deepcopy(properties) if properties is not None else Properties() + self.constraints: Constraints = deepcopy(constraints) if constraints is not None else Constraints() def __repr__(self): return repr({"properties": self.properties, "constraints": self.constraints}) @@ -69,22 +69,24 @@ def add_constraints(self, *constraints: Union[Constraint, ConstraintGroup]): async def add_default_parameters( self, - subnet: Optional[str] = SUBNET, + parser: DemandOfferSyntaxParser, + subnet: Optional[str] = None, expiration: Optional[datetime] = None, allocations: Iterable[Allocation] = (), ) -> None: - """Subscribe a new demand. - + """ :param payload: Details of the demand :param subnet: Subnet tag :param expiration: Timestamp when all agreements based on this demand will expire TODO: is this correct? :param allocations: Allocations that will be included in the description of this demand. - :param autoclose: Unsubscribe demand on :func:`__aexit__` - :param autostart: Immediately start collecting yagna events for this :any:`Demand`. - Without autostart events for this demand will start being collected after a call to - :func:`Demand.start_collecting_events`. """ + # FIXME: get rid of local import + from golem_core.core.golem_node.golem_node import DEFAULT_EXPIRATION_TIMEOUT, SUBNET + + if subnet is None: + subnet = SUBNET + if expiration is None: expiration = datetime.now(timezone.utc) + DEFAULT_EXPIRATION_TIMEOUT @@ -92,7 +94,9 @@ async def add_default_parameters( await self.add(dobm_defaults.NodeInfo(subnet_tag=subnet)) for allocation in allocations: - properties, constraints = await allocation.get_properties_and_constraints_for_demand() + properties, constraints = await allocation.get_properties_and_constraints_for_demand( + parser + ) self.add_constraints(constraints) self.add_properties(properties) diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/__init__.py b/golem_core/core/market_api/resources/demand/demand_offer_base/__init__.py index 650d57dc..cfadf00e 100644 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/__init__.py +++ b/golem_core/core/market_api/resources/demand/demand_offer_base/__init__.py @@ -16,7 +16,6 @@ DemandOfferBaseModel, TDemandOfferBaseModel, constraint, - join_str_constraints, prop, ) from golem_core.core.market_api.resources.demand.demand_offer_base.payload import ( @@ -40,7 +39,6 @@ "Activity", "TDemandOfferBaseModel", "DemandOfferBaseModel", - "join_str_constraints", "constraint", "prop", "BaseDemandOfferBaseException", diff --git a/golem_core/core/market_api/resources/proposal.py b/golem_core/core/market_api/resources/proposal.py index b027da72..00070385 100644 --- a/golem_core/core/market_api/resources/proposal.py +++ b/golem_core/core/market_api/resources/proposal.py @@ -1,7 +1,7 @@ import asyncio from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from enum import StrEnum +from enum import Enum from typing import TYPE_CHECKING, AsyncIterator, Dict, Optional, Union from ya_market import RequestorApi @@ -19,7 +19,7 @@ from golem_core.core.market_api.resources.demand import Demand -class ProposalState(StrEnum): +class ProposalState(Enum): INITIDAL = "Initial" DRAFT = "Draft" REJECTED = "Rejected" @@ -167,7 +167,7 @@ async def reject(self, reason: str = "") -> None: @api_call_wrapper() async def respond( - self, properties: Optional[Dict] = None, constraints: Optional[str] = None + self, properties: Optional[Properties] = None, constraints: Optional[Constraints] = None ) -> "Proposal": """Respond to a proposal with a counter-proposal. @@ -180,7 +180,10 @@ async def respond( if properties is None and constraints is None: data = await self._response_data() elif properties is not None and constraints is not None: - data = models.DemandOfferBase(properties=properties, constraints=constraints) + data = models.DemandOfferBase( + properties=properties.serialize(), + constraints=constraints.serialize() + ) else: raise ValueError("Both `properties` and `constraints` arguments must be provided!") diff --git a/golem_core/core/payment_api/resources/allocation.py b/golem_core/core/payment_api/resources/allocation.py index ade26ea7..acac48cb 100644 --- a/golem_core/core/payment_api/resources/allocation.py +++ b/golem_core/core/payment_api/resources/allocation.py @@ -8,6 +8,7 @@ from golem_core.core.payment_api.events import NewAllocation from golem_core.core.payment_api.exceptions import NoMatchingAccount from golem_core.core.props_cons.constraints import Constraints +from golem_core.core.props_cons.parsers.base import DemandOfferSyntaxParser from golem_core.core.props_cons.properties import Properties from golem_core.core.resources import _NULL, Resource, ResourceClosed, api_call_wrapper from golem_core.core.resources.base import TModel @@ -88,16 +89,16 @@ async def create(cls, node: "GolemNode", data: models.Allocation) -> "Allocation return cls(node, created.allocation_id, created) @api_call_wrapper() - async def get_properties_and_constraints_for_demand(self) -> Tuple[Properties, Constraints]: + async def get_properties_and_constraints_for_demand(self, parser: DemandOfferSyntaxParser) -> Tuple[Properties, Constraints]: data = await self.api.get_demand_decorations([self.id]) properties = Properties({ - prop.name: prop.value + prop.key: prop.value for prop in data.properties }) - print(data.constraints) - - constraints = self._node._demand_offer_syntax_parser.parse() - - return data.properties, data.constraints + constraints = Constraints( + parser.parse(c) for c in data.constraints + ) + + return properties, constraints diff --git a/golem_core/core/payment_api/resources/invoice.py b/golem_core/core/payment_api/resources/invoice.py index f3584e1a..b0b9bdf4 100644 --- a/golem_core/core/payment_api/resources/invoice.py +++ b/golem_core/core/payment_api/resources/invoice.py @@ -11,6 +11,7 @@ if TYPE_CHECKING: from golem_core.core.market_api.resources.agreement import Agreement # noqa + from golem_core.core.golem_node import GolemNode class Invoice(Resource[RequestorApi, models.Invoice, "Agreement", _NULL, _NULL]): diff --git a/golem_core/core/props_cons/base.py b/golem_core/core/props_cons/base.py index b4dba702..f9947b2b 100644 --- a/golem_core/core/props_cons/base.py +++ b/golem_core/core/props_cons/base.py @@ -10,13 +10,13 @@ PropertyValue = Any -class PropsConstrsSerializer: +class PropsConstrsSerializerMixin: @classmethod - def serialize_value(cls, value: Any) -> Any: + def _serialize_value(cls, value: Any) -> Any: """Return value in primitive format compatible with Golem's property and constraint syntax.""" if isinstance(value, (list, tuple)): - return type(value)(cls.serialize_value(v) for v in value) + return type(value)(cls._serialize_value(v) for v in value) if isinstance(value, datetime.datetime): return int(value.timestamp() * 1000) @@ -27,7 +27,7 @@ def serialize_value(cls, value: Any) -> Any: return value @classmethod - def deserialize_value(cls, value: Any, field: Field) -> Any: + def _deserialize_value(cls, value: Any, field: Field) -> Any: """Return proper value for field from given primitive. Intended to be overriden with additional type serialisation methods. diff --git a/golem_core/core/props_cons/constraints.py b/golem_core/core/props_cons/constraints.py index 7fdff7c5..ef6aba1f 100644 --- a/golem_core/core/props_cons/constraints.py +++ b/golem_core/core/props_cons/constraints.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Any, MutableSequence, Union -from golem_core.core.props_cons.base import PropertyName +from golem_core.core.props_cons.base import PropertyName, PropsConstrsSerializerMixin class ConstraintException(Exception): @@ -23,7 +23,7 @@ class ConstraintGroupOperator(Enum): @dataclass -class MarketDemandOfferSyntaxElement(ABC): +class MarketDemandOfferSyntaxElement(PropsConstrsSerializerMixin, ABC): def __post_init__(self) -> None: self._validate() @@ -47,7 +47,12 @@ class Constraint(MarketDemandOfferSyntaxElement): value: Any def _serialize(self) -> str: - return f"({self.property_name}{self.operator}{self.value})" + serialized_value = self._serialize_value(self.value) + + if isinstance(self.value, (list, tuple)): + serialized_value = '[{}]'.format(', '.join(str(v) for v in serialized_value)) + + return f"({self.property_name}{self.operator}{serialized_value})" @dataclass @@ -60,14 +65,6 @@ def _validate(self) -> None: raise ConstraintException("ConstraintGroup with `!` operator can contain only 1 item!") def _serialize(self) -> str: - items_len = len(self.items) - - if items_len == 0: - return f"({self.operator})" - - if items_len == 1: - return self.items[0].serialize() - items = "\n\t".join(item.serialize() for item in self.items) return f"({self.operator}{items})" diff --git a/golem_core/core/props_cons/parsers/base.py b/golem_core/core/props_cons/parsers/base.py index 7aa6f2fe..d4ef03ce 100644 --- a/golem_core/core/props_cons/parsers/base.py +++ b/golem_core/core/props_cons/parsers/base.py @@ -3,6 +3,9 @@ from golem_core.core.props_cons.constraints import Constraints +class SyntaxException(Exception): + pass + class DemandOfferSyntaxParser(ABC): @abstractmethod def parse(self, syntax: str) -> Constraints: diff --git a/golem_core/core/props_cons/parsers/textx/parser.py b/golem_core/core/props_cons/parsers/textx/parser.py index 4257797a..f9150a60 100644 --- a/golem_core/core/props_cons/parsers/textx/parser.py +++ b/golem_core/core/props_cons/parsers/textx/parser.py @@ -1,9 +1,9 @@ from pathlib import Path -from textx import metamodel_from_file +from textx import metamodel_from_file, TextXSyntaxError from golem_core.core.props_cons.constraints import Constraint, ConstraintGroup, Constraints -from golem_core.core.props_cons.parsers.base import DemandOfferSyntaxParser +from golem_core.core.props_cons.parsers.base import DemandOfferSyntaxParser, SyntaxException class TextXDemandOfferSyntaxParser(DemandOfferSyntaxParser): @@ -18,6 +18,9 @@ def __init__(self): ) def parse(self, syntax: str) -> Constraints: - model = self._metamodel.model_from_str(syntax) + try: + model = self._metamodel.model_from_str(syntax) + except TextXSyntaxError as e: + raise SyntaxException(f"Syntax `{syntax}` parsed with following error: {e}") return model.constraints diff --git a/golem_core/core/props_cons/parsers/textx/syntax.tx b/golem_core/core/props_cons/parsers/textx/syntax.tx index cc9b6ba9..27d75d5b 100644 --- a/golem_core/core/props_cons/parsers/textx/syntax.tx +++ b/golem_core/core/props_cons/parsers/textx/syntax.tx @@ -1,5 +1,9 @@ Model: - constraints=ConstraintGroup + constraints=ConstraintGroupOrSingle +; + +ConstraintGroupOrSingle: + ConstraintGroup | Constraint ; ConstraintGroup: @@ -10,10 +14,6 @@ Constraint: "(" property_path=PropertyName operator=ConstraintOperator value=PropertyValue ")" ; -ConstraintGroupOrSingle: - ConstraintGroup | Constraint -; - ConstraintGroupOperator: "&" | "|" | "!" ; @@ -23,7 +23,7 @@ ConstraintOperator: ; PropertyName: - /\w+(\.\w+)*/ + /[\w-]+(\.[\w-]+)*/ ; ProeprtyValueList: @@ -35,5 +35,5 @@ PropertyValue: ; PropertyValueStr: - /[a-zA-Z0-9_.\/:;]+/ + /[a-zA-Z0-9_.\*\/:;]+/ ; \ No newline at end of file diff --git a/golem_core/core/props_cons/properties.py b/golem_core/core/props_cons/properties.py index 68547c5b..984aefef 100644 --- a/golem_core/core/props_cons/properties.py +++ b/golem_core/core/props_cons/properties.py @@ -1,15 +1,22 @@ from copy import deepcopy from typing import Any, Mapping -from golem_core.core.props_cons.base import PropsConstrsSerializer +from golem_core.core.props_cons.base import PropsConstrsSerializerMixin -class Properties(PropsConstrsSerializer, dict): +_missing = object() + +class Properties(PropsConstrsSerializerMixin, dict): """Low level wrapper class for Golem's Market API properties manipulation.""" - def __init__(self, mapping, /) -> None: + def __init__(self, mapping=_missing, /) -> None: + if mapping is _missing: + super().__init__() + return + mapping_deep_copy = deepcopy(mapping) - super(mapping_deep_copy) + + super().__init__(mapping_deep_copy) def serialize(self) -> Mapping[str, Any]: """Serialize complex objects into format handled by Market API properties specification.""" diff --git a/golem_core/managers/base.py b/golem_core/managers/base.py index ddf6e047..c67f3b71 100644 --- a/golem_core/managers/base.py +++ b/golem_core/managers/base.py @@ -4,6 +4,7 @@ from golem_core.core.activity_api import Activity, Script, commands from golem_core.core.market_api import Agreement, Proposal +from golem_core.core.market_api.resources.demand.demand import DemandData from golem_core.core.market_api.resources.proposal import ProposalData from golem_core.core.payment_api import Allocation from golem_core.core.resources import ResourceEvent @@ -148,5 +149,5 @@ class WorkManager(Manager, ABC): class NegotiationPlugin(ABC): @abstractmethod - async def __call__(self, demand_data: ProposalData, offer_data: ProposalData) -> None: + async def __call__(self, demand_data: DemandData, proposal_data: ProposalData) -> None: ... diff --git a/golem_core/managers/negotiation/plugins.py b/golem_core/managers/negotiation/plugins.py index 6bf22998..3ac2f9c8 100644 --- a/golem_core/managers/negotiation/plugins.py +++ b/golem_core/managers/negotiation/plugins.py @@ -1,5 +1,6 @@ import logging from typing import Sequence +from golem_core.core.market_api.resources.demand.demand import DemandData from golem_core.core.market_api.resources.proposal import ProposalData from golem_core.managers.base import NegotiationPlugin @@ -13,11 +14,11 @@ def __init__(self, blacklist: Sequence[str]) -> None: self._blacklist = blacklist async def __call__( - self, demand_proposal_data: ProposalData, offer_proposal_data: ProposalData + self, demand_data: DemandData, proposal_data: ProposalData ) -> None: logger.debug("Calling blacklist plugin...") - provider_id = offer_proposal_data.issuer_id + provider_id = proposal_data.issuer_id if provider_id in self._blacklist: logger.debug( diff --git a/golem_core/managers/negotiation/sequential.py b/golem_core/managers/negotiation/sequential.py index 40f63576..0a0b25bf 100644 --- a/golem_core/managers/negotiation/sequential.py +++ b/golem_core/managers/negotiation/sequential.py @@ -6,10 +6,15 @@ from golem_core.core.golem_node.golem_node import DEFAULT_EXPIRATION_TIMEOUT, SUBNET, GolemNode from golem_core.core.market_api import Demand, DemandBuilder, Payload, Proposal +from golem_core.core.market_api.resources.demand.demand import DemandData from golem_core.core.market_api.resources.demand.demand_offer_base import defaults as dobm_defaults from golem_core.core.market_api.resources.proposal import ProposalData from golem_core.core.payment_api import Allocation +from golem_core.core.props_cons.constraints import Constraints +from golem_core.core.props_cons.parsers.textx.parser import TextXDemandOfferSyntaxParser +from golem_core.core.props_cons.properties import Properties from golem_core.managers.base import ManagerException, NegotiationManager, NegotiationPlugin +from golem_core.utils.asyncio import create_task_with_logging logger = logging.getLogger(__name__) @@ -33,6 +38,7 @@ def __init__( self._negotiation_loop_task: Optional[asyncio.Task] = None self._plugins: List[NegotiationPlugin] = list(plugins) if plugins is not None else [] self._eligible_proposals: asyncio.Queue[Proposal] = asyncio.Queue() + self._demand_offer_parser = TextXDemandOfferSyntaxParser() def register_plugin(self, plugin: NegotiationPlugin): self._plugins.append(plugin) @@ -57,7 +63,7 @@ async def start(self) -> None: logger.debug(f"Starting failed with `{message}`") raise ManagerException(message) - self._negotiation_loop_task = asyncio.create_task(self._negotiation_loop()) + self._negotiation_loop_task = create_task_with_logging(self._negotiation_loop()) logger.debug("Starting done") @@ -84,6 +90,10 @@ async def _negotiation_loop(self) -> None: demand = await demand_builder.create_demand(self._golem) demand.start_collecting_events() + print(await demand.get_data()) + + logger.debug('Demand published, waiting for proposals...') + try: async for proposal in self._negotiate(demand): await self._eligible_proposals.put(proposal) @@ -94,59 +104,43 @@ async def _prepare_demand_builder(self, allocation: Allocation) -> DemandBuilder logger.debug("Preparing demand...") # FIXME: Code looks duplicated as GolemNode.create_demand does the same - demand_builder = DemandBuilder() - await demand_builder.add( - dobm_defaults.Activity( - expiration=datetime.now(timezone.utc) + DEFAULT_EXPIRATION_TIMEOUT, - multi_activity=True, - ) - ) - await demand_builder.add(dobm_defaults.NodeInfo(subnet_tag=SUBNET)) + await demand_builder.add_default_parameters(self._demand_offer_parser, allocations=[allocation]) await demand_builder.add(self._payload) - ( - allocation_properties, - allocation_constraints, - ) = await allocation.demand_properties_constraints() - demand_builder.add_constraints(*allocation_constraints) - demand_builder.add_properties({p.key: p.value for p in allocation_properties}) - - logger.debug(f"Preparing demand done`") + logger.debug(f"Preparing demand done") return demand_builder async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: - demand_data = self._get_demand_data_from_demand(demand) + demand_data = await self._get_demand_data_from_demand(demand) - async for initial_offer_proposal in demand.initial_proposals(): - offer_proposal = await self._negotiate_proposal(demand_data, initial_offer_proposal) + async for initial_proposal in demand.initial_proposals(): + offer_proposal = await self._negotiate_proposal(demand_data, initial_proposal) if offer_proposal is None: logger.debug( - f"Negotiating proposal `{initial_offer_proposal}` done and proposal was rejected" + f"Negotiating proposal `{initial_proposal}` done and proposal was rejected" ) continue yield offer_proposal async def _negotiate_proposal( - self, demand_builder: DemandBuilder, offer_proposal: Proposal + self, demand_data: DemandData, offer_proposal: Proposal ) -> Optional[Proposal]: logger.debug(f"Negotiating proposal `{offer_proposal}`...") while True: - demand_builder_after_plugins = deepcopy(demand_builder) + demand_data_after_plugins = deepcopy(demand_data) try: logger.debug(f"Applying plugins on `{offer_proposal}`...") for plugin in self._plugins: - demand_builder_after_plugins = await plugin( - demand_builder_after_plugins, offer_proposal - ) + await plugin(demand_data_after_plugins, offer_proposal) except RejectProposal as e: logger.debug( @@ -160,12 +154,12 @@ async def _negotiate_proposal( else: logger.debug(f"Applying plugins on `{offer_proposal}` done") - if offer_proposal.initial or demand_builder_after_plugins != demand_builder: + if offer_proposal.initial or demand_data_after_plugins != demand_data: logger.debug("Sending demand proposal...") demand_proposal = await offer_proposal.respond( - demand_builder_after_plugins.properties, - demand_builder_after_plugins.constraints, + demand_data_after_plugins.properties, + demand_data_after_plugins.constraints, ) logger.debug("Sending demand proposal done") @@ -189,6 +183,16 @@ async def _negotiate_proposal( return offer_proposal - def _get_proposal_data_from_demand(self, demand: Demand) -> ProposalData: + async def _get_demand_data_from_demand(self, demand: Demand) -> ProposalData: # FIXME: Unnecessary serialisation from DemandBuilder to Demand, and from Demand to ProposalData - return ProposalData(demand.data.properties) + data = await demand.get_data() + + constraints = Constraints(self._demand_offer_parser.parse(con) for con in data.constraints) + + return DemandData( + properties=Properties(data.properties), + constraints=constraints, + demand_id=data.demand_id, + requestor_id=data.requestor_id, + timestamp=data.timestamp, + ) diff --git a/golem_core/utils/asyncio.py b/golem_core/utils/asyncio.py new file mode 100644 index 00000000..12638706 --- /dev/null +++ b/golem_core/utils/asyncio.py @@ -0,0 +1,17 @@ +import asyncio +import logging + +logger = logging.getLogger(__name__) + +def create_task_with_logging(coro) -> asyncio.Task: + task = asyncio.create_task(coro) + task.add_done_callback(_handle_task_logging) + return task + +def _handle_task_logging(task: asyncio.Task): + try: + return task.result() + except asyncio.CancelledError: + pass + except Exception: + logger.exception('Background async task encountered unhandled exception!') diff --git a/golem_core/utils/logging.py b/golem_core/utils/logging.py index 332e9ec2..d6de9224 100644 --- a/golem_core/utils/logging.py +++ b/golem_core/utils/logging.py @@ -26,13 +26,13 @@ "level": "INFO", }, "golem_core.managers": { - "level": "INFO", + "level": "DEBUG", }, "golem_core.managers.negotiation": { - "level": "INFO", + "level": "DEBUG", }, "golem_core.managers.proposal": { - "level": "INFO", + "level": "DEBUG", }, }, } diff --git a/tests/unit/parsers/__init__.py b/tests/unit/parsers/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/unit/test_demand_builder.py b/tests/unit/test_demand_builder.py index 97cd97af..8e34c479 100644 --- a/tests/unit/test_demand_builder.py +++ b/tests/unit/test_demand_builder.py @@ -4,7 +4,6 @@ from golem_core.core.market_api import ( DemandBuilder, - DemandBuilderDecorator, DemandOfferBaseModel, constraint, prop, @@ -19,16 +18,6 @@ class ExampleModel(DemandOfferBaseModel): con2: int = constraint("some.con2.path", "<=") -class ExampleBuilderDecorator(DemandBuilderDecorator): - async def decorate_demand_builder(self, demand_builder: DemandBuilder) -> None: - demand_builder.add_properties({"some.fancy.field": "was just added by demand decorator"}) - - -class AnotherExampleBuilderDecorator(DemandBuilderDecorator): - async def decorate_demand_builder(self, demand_builder: DemandBuilder) -> None: - demand_builder.add_constraints("field=added") - - @pytest.mark.asyncio async def test_add(): model = ExampleModel(prop1=1, prop2=2, con1=3, con2=4) @@ -68,19 +57,3 @@ async def test_create_demand(mocker): demand_builder.properties, demand_builder.constraints, ) - - -@pytest.mark.asyncio -async def test_decorate(): - demand_builder = DemandBuilder() - - assert demand_builder.properties == {} - assert demand_builder.constraints == "(&)" - - await demand_builder.decorate(ExampleBuilderDecorator(), AnotherExampleBuilderDecorator()) - - assert demand_builder.properties == { - "some.fancy.field": "was just added by demand decorator", - } - - assert demand_builder.constraints == "field=added" diff --git a/tests/unit/test_demand_builder_model.py b/tests/unit/test_demand_builder_model.py index 48bb4d5a..54d337a8 100644 --- a/tests/unit/test_demand_builder_model.py +++ b/tests/unit/test_demand_builder_model.py @@ -10,7 +10,6 @@ DemandOfferBaseModel, InvalidPropertiesError, constraint, - join_str_constraints, prop, ) diff --git a/tests/unit/test_demand_offer_cons.py b/tests/unit/test_demand_offer_cons.py new file mode 100644 index 00000000..057b6ac6 --- /dev/null +++ b/tests/unit/test_demand_offer_cons.py @@ -0,0 +1,56 @@ +from datetime import datetime +from enum import Enum +import textwrap + +import pytest +from golem_core.core.props_cons.constraints import Constraint, ConstraintException, ConstraintGroup, Constraints +from golem_core.core.props_cons.properties import Properties + + +class ExampleEnum(Enum): + FOO = "BAR" + + +def test_constraints_serialize(): + cons = Constraints([ + Constraint("foo", '=', "bar"), + Constraint("int_field", '=', 123), + Constraint("float_field", '=', 1.5), + Constraint("datetime_field", '=', datetime(2023, 1, 2)), + Constraint("enum_field", '=', ExampleEnum.FOO), + Constraint("list_field", '=', [ + datetime(2023, 1, 2), + ExampleEnum.FOO, + ]), + ConstraintGroup([ + Constraint('some.other.field', '=', 'works!') + ], + "|"), + ]) + + assert cons.serialize() == ( + '(&(foo=bar)\n' + '\t(int_field=123)\n' + '\t(float_field=1.5)\n' + '\t(datetime_field=1672614000000)\n' + '\t(enum_field=BAR)\n' + '\t(list_field=[1672614000000, BAR])\n' + '\t(some.other.field=works!))' + ) + +def test_constraint_group_raises_on_not_operator_with_multiple_items(): + with pytest.raises(ConstraintException): + ConstraintGroup([ + Constraint('foo', '=', 'bar'), + Constraint('bat', '=', 'man'), + ], '!') + + cons_group = ConstraintGroup([ + Constraint('foo', '=', 'bar'), + Constraint('bat', '=', 'man'), + ]) + + cons_group.operator = '!' + + with pytest.raises(ConstraintException): + cons_group.serialize() \ No newline at end of file diff --git a/tests/unit/parsers/test_parsers.py b/tests/unit/test_demand_offer_parsers.py similarity index 69% rename from tests/unit/parsers/test_parsers.py rename to tests/unit/test_demand_offer_parsers.py index 441ffc94..28f44d5e 100644 --- a/tests/unit/parsers/test_parsers.py +++ b/tests/unit/test_demand_offer_parsers.py @@ -8,40 +8,25 @@ def demand_offer_parser(): return TextXDemandOfferSyntaxParser() - -@pytest.mark.parametrize( - "input_string, output", - ( - ("(&)", ConstraintGroup(operator="&")), - ("(|)", ConstraintGroup(operator="|")), - ("(!)", ConstraintGroup(operator="!")), - ), -) -def test_empty_group(demand_offer_parser, input_string, output): - result = demand_offer_parser.parse(input_string) - - assert result == output - - @pytest.mark.parametrize( "input_string, output", ( - ("(& (foo=1))", ConstraintGroup([Constraint("foo", "=", "1")])), - ("(& (float.value=1.5))", ConstraintGroup([Constraint("float.value", "=", "1.5")])), - ("(& (foo=bar))", ConstraintGroup([Constraint("foo", "=", "bar")])), + ("(foo=1)", Constraint("foo", "=", "1")), + ("(float.value=1.5)", Constraint("float.value", "=", "1.5")), + ("(foo=bar)", Constraint("foo", "=", "bar")), ( - "(& (foo=more.complex.value))", - ConstraintGroup([Constraint("foo", "=", "more.complex.value")]), + "(foo=more.complex.value)", + Constraint("foo", "=", "more.complex.value"), ), ( - "(& (foo=http://google.com))", - ConstraintGroup([Constraint("foo", "=", "http://google.com")]), + "(foo=http://google.com)", + Constraint("foo", "=", "http://google.com"), ), - ("(& (foo=[1, 2, 3]))", ConstraintGroup([Constraint("foo", "=", ["1", "2", "3"])])), - ("(& (foo=[a, b, c]))", ConstraintGroup([Constraint("foo", "=", ["a", "b", "c"])])), - ("(& (some.nested.param=1))", ConstraintGroup([Constraint("some.nested.param", "=", "1")])), - ("(& (foo<=1))", ConstraintGroup([Constraint("foo", "<=", "1")])), - ("(& (foo>=1))", ConstraintGroup([Constraint("foo", ">=", "1")])), + ("(foo=[1, 2, 3])", Constraint("foo", "=", ["1", "2", "3"])), + ("(foo=[a, b, c])", Constraint("foo", "=", ["a", "b", "c"])), + ("(some.nested.param=1)", Constraint("some.nested.param", "=", "1")), + ("(foo<=1)", Constraint("foo", "<=", "1")), + ("(foo>=1)", Constraint("foo", ">=", "1")), ), ) def test_single_constraint(demand_offer_parser, input_string, output): @@ -71,9 +56,28 @@ def test_single_constraint(demand_offer_parser, input_string, output): "(| (foo=1) (bar=2))", ConstraintGroup([Constraint("foo", "=", "1"), Constraint("bar", "=", "2")], "|"), ), + + ( + "(| (foo=1) (bar=2))", + ConstraintGroup([Constraint("foo", "=", "1"), Constraint("bar", "=", "2")], "|"), + ), + ), +) +def test_constraint_groups(demand_offer_parser, input_string, output): + result = demand_offer_parser.parse(input_string) + + assert result == output + + +@pytest.mark.parametrize( + "input_string, output", + ( + ("(&)", ConstraintGroup(operator="&")), + ("(|)", ConstraintGroup(operator="|")), + ("(!)", ConstraintGroup(operator="!")), ), ) -def test_multiple_constraints(demand_offer_parser, input_string, output): +def test_constraint_groups_empty(demand_offer_parser, input_string, output): result = demand_offer_parser.parse(input_string) assert result == output @@ -108,7 +112,7 @@ def test_error_not_operator_with_multiple_items(demand_offer_parser): ), ), ) -def test_nested_groups(demand_offer_parser, input_string, output): +def test_constraint_groups_nested(demand_offer_parser, input_string, output): result = demand_offer_parser.parse(input_string) assert result == output diff --git a/tests/unit/test_demand_offer_props.py b/tests/unit/test_demand_offer_props.py new file mode 100644 index 00000000..456ef4c4 --- /dev/null +++ b/tests/unit/test_demand_offer_props.py @@ -0,0 +1,57 @@ +from datetime import datetime +from enum import Enum +from golem_core.core.props_cons.properties import Properties + + +class ExampleEnum(Enum): + FOO = "BAR" + + +def test_property_deepcopies_its_input(): + list_field = [1, 2, 3] + original_dict = { + 'foo': "bar", + "list_field": list_field, + } + props = Properties(original_dict) + + assert props["foo"] == "bar" + assert props["list_field"][0] == 1 + assert props["list_field"][1] == 2 + assert props["list_field"][2] == 3 + + props['foo'] = "123" + props["list_field"].append(4) + + assert props["foo"] == "123" + assert props["list_field"] == [1, 2, 3, 4] + + assert original_dict["foo"] == "bar" + assert original_dict["list_field"] != props["list_field"] + +def test_property_serialize(): + props = Properties({ + "foo": "bar", + "int_field": 123, + "float_field": 1.5, + "datetime_field": datetime(2023, 1, 2), + "enum_field": ExampleEnum.FOO, + "list_field": [ + datetime(2023, 1, 2), + ExampleEnum.FOO, + ] + }) + + serialized_props = props.serialize() + + assert serialized_props == { + "foo": "bar", + "int_field": 123, + "float_field": 1.5, + "datetime_field": 1672614000000, + "enum_field": "BAR", + "list_field": [ + 1672614000000, + "BAR", + ] + } From 1c770cb1e08055d5ff9f8d14927d30c4df941b21 Mon Sep 17 00:00:00 2001 From: approxit Date: Mon, 19 Jun 2023 17:49:54 +0200 Subject: [PATCH 056/123] nearly finished oop props and cons --- .../market_api/resources/demand/__init__.py | 4 +- .../market_api/resources/demand/demand.py | 3 +- .../resources/demand/demand_builder.py | 8 +- .../core/market_api/resources/proposal.py | 13 +-- .../core/payment_api/resources/allocation.py | 17 ++-- .../core/payment_api/resources/invoice.py | 2 +- golem_core/core/props_cons/constraints.py | 23 ++--- golem_core/core/props_cons/parsers/base.py | 1 + .../core/props_cons/parsers/textx/parser.py | 6 +- .../core/props_cons/parsers/textx/syntax.tx | 6 +- golem_core/core/props_cons/properties.py | 6 +- golem_core/managers/negotiation/plugins.py | 6 +- golem_core/managers/negotiation/sequential.py | 37 +++++--- golem_core/utils/asyncio.py | 4 +- golem_core/utils/logging.py | 4 +- tests/unit/test_demand_builder.py | 9 +- tests/unit/test_demand_offer_cons.py | 90 +++++++++++-------- tests/unit/test_demand_offer_parsers.py | 4 +- tests/unit/test_demand_offer_props.py | 33 ++++--- 19 files changed, 150 insertions(+), 126 deletions(-) diff --git a/golem_core/core/market_api/resources/demand/__init__.py b/golem_core/core/market_api/resources/demand/__init__.py index 1ab5e0e7..a3f6431e 100644 --- a/golem_core/core/market_api/resources/demand/__init__.py +++ b/golem_core/core/market_api/resources/demand/__init__.py @@ -1,7 +1,5 @@ from golem_core.core.market_api.resources.demand.demand import Demand -from golem_core.core.market_api.resources.demand.demand_builder import ( - DemandBuilder, -) +from golem_core.core.market_api.resources.demand.demand_builder import DemandBuilder from golem_core.core.market_api.resources.demand.demand_offer_base import ( INF_CPU_THREADS, INF_MEM, diff --git a/golem_core/core/market_api/resources/demand/demand.py b/golem_core/core/market_api/resources/demand/demand.py index c4a7443e..61332985 100644 --- a/golem_core/core/market_api/resources/demand/demand.py +++ b/golem_core/core/market_api/resources/demand/demand.py @@ -1,5 +1,6 @@ import asyncio from dataclasses import dataclass +from datetime import datetime from typing import TYPE_CHECKING, AsyncIterator, Callable, Dict, List, Optional, Union from ya_market import RequestorApi @@ -28,7 +29,7 @@ class DemandData: constraints: Constraints demand_id: str requestor_id: str - timestamp: str + timestamp: datetime class Demand(Resource[RequestorApi, models.Demand, _NULL, Proposal, _NULL], YagnaEventCollector): diff --git a/golem_core/core/market_api/resources/demand/demand_builder.py b/golem_core/core/market_api/resources/demand/demand_builder.py index 8f8f5bda..bb8e0906 100644 --- a/golem_core/core/market_api/resources/demand/demand_builder.py +++ b/golem_core/core/market_api/resources/demand/demand_builder.py @@ -38,8 +38,12 @@ class DemandBuilder: def __init__( self, properties: Optional[Properties] = None, constraints: Optional[Constraints] = None ): - self.properties: Properties = deepcopy(properties) if properties is not None else Properties() - self.constraints: Constraints = deepcopy(constraints) if constraints is not None else Constraints() + self.properties: Properties = ( + deepcopy(properties) if properties is not None else Properties() + ) + self.constraints: Constraints = ( + deepcopy(constraints) if constraints is not None else Constraints() + ) def __repr__(self): return repr({"properties": self.properties, "constraints": self.constraints}) diff --git a/golem_core/core/market_api/resources/proposal.py b/golem_core/core/market_api/resources/proposal.py index 00070385..afde82cf 100644 --- a/golem_core/core/market_api/resources/proposal.py +++ b/golem_core/core/market_api/resources/proposal.py @@ -2,7 +2,7 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone from enum import Enum -from typing import TYPE_CHECKING, AsyncIterator, Dict, Optional, Union +from typing import TYPE_CHECKING, AsyncIterator, Optional, Union, Literal from ya_market import RequestorApi from ya_market import models as models @@ -19,13 +19,7 @@ from golem_core.core.market_api.resources.demand import Demand -class ProposalState(Enum): - INITIDAL = "Initial" - DRAFT = "Draft" - REJECTED = "Rejected" - ACCPETED = "Accepted" - EXPIRED = "Expired" - +ProposalState = Literal["Initial","Draft","Rejected","Accepted","Expired"] @dataclass class ProposalData: @@ -181,8 +175,7 @@ async def respond( data = await self._response_data() elif properties is not None and constraints is not None: data = models.DemandOfferBase( - properties=properties.serialize(), - constraints=constraints.serialize() + properties=properties.serialize(), constraints=constraints.serialize() ) else: raise ValueError("Both `properties` and `constraints` arguments must be provided!") diff --git a/golem_core/core/payment_api/resources/allocation.py b/golem_core/core/payment_api/resources/allocation.py index acac48cb..43f964d0 100644 --- a/golem_core/core/payment_api/resources/allocation.py +++ b/golem_core/core/payment_api/resources/allocation.py @@ -1,6 +1,6 @@ import asyncio from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING, Optional, Tuple from _decimal import Decimal from ya_payment import RequestorApi, models @@ -89,16 +89,13 @@ async def create(cls, node: "GolemNode", data: models.Allocation) -> "Allocation return cls(node, created.allocation_id, created) @api_call_wrapper() - async def get_properties_and_constraints_for_demand(self, parser: DemandOfferSyntaxParser) -> Tuple[Properties, Constraints]: + async def get_properties_and_constraints_for_demand( + self, parser: DemandOfferSyntaxParser + ) -> Tuple[Properties, Constraints]: data = await self.api.get_demand_decorations([self.id]) - properties = Properties({ - prop.key: prop.value - for prop in data.properties - }) + properties = Properties({prop.key: prop.value for prop in data.properties}) + + constraints = Constraints(parser.parse(c) for c in data.constraints) - constraints = Constraints( - parser.parse(c) for c in data.constraints - ) - return properties, constraints diff --git a/golem_core/core/payment_api/resources/invoice.py b/golem_core/core/payment_api/resources/invoice.py index b0b9bdf4..43d17a80 100644 --- a/golem_core/core/payment_api/resources/invoice.py +++ b/golem_core/core/payment_api/resources/invoice.py @@ -10,8 +10,8 @@ from golem_core.core.resources.base import TModel if TYPE_CHECKING: - from golem_core.core.market_api.resources.agreement import Agreement # noqa from golem_core.core.golem_node import GolemNode + from golem_core.core.market_api.resources.agreement import Agreement # noqa class Invoice(Resource[RequestorApi, models.Invoice, "Agreement", _NULL, _NULL]): diff --git a/golem_core/core/props_cons/constraints.py b/golem_core/core/props_cons/constraints.py index ef6aba1f..49294627 100644 --- a/golem_core/core/props_cons/constraints.py +++ b/golem_core/core/props_cons/constraints.py @@ -1,7 +1,6 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field -from enum import Enum -from typing import Any, MutableSequence, Union +from typing import Any, MutableSequence, Union, Literal from golem_core.core.props_cons.base import PropertyName, PropsConstrsSerializerMixin @@ -10,16 +9,8 @@ class ConstraintException(Exception): pass -class ConstraintOperator(Enum): - EQUALS = "=" - GRATER_OR_EQUALS = ">=" - LESS_OR_EQUALS = "<=" - - -class ConstraintGroupOperator(Enum): - AND = "&" - OR = "|" - NOT = "!" +ConstraintOperator = Literal["=", "<=", ">=", "<", ">"] +ConstraintGroupOperator = Literal["&", "|", "!"] @dataclass @@ -50,7 +41,10 @@ def _serialize(self) -> str: serialized_value = self._serialize_value(self.value) if isinstance(self.value, (list, tuple)): - serialized_value = '[{}]'.format(', '.join(str(v) for v in serialized_value)) + if not self.value: + return "" + + serialized_value = "[{}]".format(", ".join(str(v) for v in serialized_value)) return f"({self.property_name}{self.operator}{serialized_value})" @@ -65,7 +59,8 @@ def _validate(self) -> None: raise ConstraintException("ConstraintGroup with `!` operator can contain only 1 item!") def _serialize(self) -> str: - items = "\n\t".join(item.serialize() for item in self.items) + serialized = [item.serialize() for item in self.items] + items = "\n\t".join(s for s in serialized if s) return f"({self.operator}{items})" diff --git a/golem_core/core/props_cons/parsers/base.py b/golem_core/core/props_cons/parsers/base.py index d4ef03ce..9f0667f6 100644 --- a/golem_core/core/props_cons/parsers/base.py +++ b/golem_core/core/props_cons/parsers/base.py @@ -6,6 +6,7 @@ class SyntaxException(Exception): pass + class DemandOfferSyntaxParser(ABC): @abstractmethod def parse(self, syntax: str) -> Constraints: diff --git a/golem_core/core/props_cons/parsers/textx/parser.py b/golem_core/core/props_cons/parsers/textx/parser.py index f9150a60..ab0ebaca 100644 --- a/golem_core/core/props_cons/parsers/textx/parser.py +++ b/golem_core/core/props_cons/parsers/textx/parser.py @@ -1,6 +1,6 @@ from pathlib import Path -from textx import metamodel_from_file, TextXSyntaxError +from textx import TextXSyntaxError, metamodel_from_file from golem_core.core.props_cons.constraints import Constraint, ConstraintGroup, Constraints from golem_core.core.props_cons.parsers.base import DemandOfferSyntaxParser, SyntaxException @@ -8,12 +8,12 @@ class TextXDemandOfferSyntaxParser(DemandOfferSyntaxParser): def __init__(self): - self._metamodel = metamodel_from_file(Path(__file__).with_name("syntax.tx")) + self._metamodel = metamodel_from_file(str(Path(__file__).with_name("syntax.tx"))) self._metamodel.register_obj_processors( { "ConstraintGroup": lambda e: ConstraintGroup(e.items, e.operator), "Constraint": lambda e: Constraint(e.property_path, e.operator, e.value), - "ProeprtyValueList": lambda e: e.items, + "PropertyValueList": lambda e: e.items, } ) diff --git a/golem_core/core/props_cons/parsers/textx/syntax.tx b/golem_core/core/props_cons/parsers/textx/syntax.tx index 27d75d5b..d510e003 100644 --- a/golem_core/core/props_cons/parsers/textx/syntax.tx +++ b/golem_core/core/props_cons/parsers/textx/syntax.tx @@ -19,19 +19,19 @@ ConstraintGroupOperator: ; ConstraintOperator: - "=" | ">=" | "<=" + "=" | "<=" | ">=" | "<" | ">" ; PropertyName: /[\w-]+(\.[\w-]+)*/ ; -ProeprtyValueList: +PropertyValueList: "[" items*=PropertyValue[","] "]" ; PropertyValue: - PropertyValueStr | ProeprtyValueList + PropertyValueStr | PropertyValueList ; PropertyValueStr: diff --git a/golem_core/core/props_cons/properties.py b/golem_core/core/props_cons/properties.py index 984aefef..43ea0605 100644 --- a/golem_core/core/props_cons/properties.py +++ b/golem_core/core/props_cons/properties.py @@ -3,9 +3,9 @@ from golem_core.core.props_cons.base import PropsConstrsSerializerMixin - _missing = object() + class Properties(PropsConstrsSerializerMixin, dict): """Low level wrapper class for Golem's Market API properties manipulation.""" @@ -20,7 +20,9 @@ def __init__(self, mapping=_missing, /) -> None: def serialize(self) -> Mapping[str, Any]: """Serialize complex objects into format handled by Market API properties specification.""" - return {key: self._serialize_property(value) for key, value in self.items()} + return { + key: self._serialize_property(value) for key, value in self.items() if value is not None + } def _serialize_property(self, value: Any) -> Any: return self._serialize_value(value) diff --git a/golem_core/managers/negotiation/plugins.py b/golem_core/managers/negotiation/plugins.py index 3ac2f9c8..70d4610e 100644 --- a/golem_core/managers/negotiation/plugins.py +++ b/golem_core/managers/negotiation/plugins.py @@ -1,7 +1,7 @@ import logging from typing import Sequence -from golem_core.core.market_api.resources.demand.demand import DemandData +from golem_core.core.market_api.resources.demand.demand import DemandData from golem_core.core.market_api.resources.proposal import ProposalData from golem_core.managers.base import NegotiationPlugin from golem_core.managers.negotiation.sequential import RejectProposal @@ -13,9 +13,7 @@ class BlacklistProviderId(NegotiationPlugin): def __init__(self, blacklist: Sequence[str]) -> None: self._blacklist = blacklist - async def __call__( - self, demand_data: DemandData, proposal_data: ProposalData - ) -> None: + async def __call__(self, demand_data: DemandData, proposal_data: ProposalData) -> None: logger.debug("Calling blacklist plugin...") provider_id = proposal_data.issuer_id diff --git a/golem_core/managers/negotiation/sequential.py b/golem_core/managers/negotiation/sequential.py index 0a0b25bf..0ff073a3 100644 --- a/golem_core/managers/negotiation/sequential.py +++ b/golem_core/managers/negotiation/sequential.py @@ -1,13 +1,12 @@ import asyncio import logging from copy import deepcopy -from datetime import datetime, timezone -from typing import AsyncIterator, Awaitable, Callable, List, Optional, Sequence +from datetime import datetime +from typing import AsyncIterator, Awaitable, Callable, List, Optional, Sequence, cast -from golem_core.core.golem_node.golem_node import DEFAULT_EXPIRATION_TIMEOUT, SUBNET, GolemNode +from golem_core.core.golem_node.golem_node import GolemNode from golem_core.core.market_api import Demand, DemandBuilder, Payload, Proposal from golem_core.core.market_api.resources.demand.demand import DemandData -from golem_core.core.market_api.resources.demand.demand_offer_base import defaults as dobm_defaults from golem_core.core.market_api.resources.proposal import ProposalData from golem_core.core.payment_api import Allocation from golem_core.core.props_cons.constraints import Constraints @@ -92,7 +91,7 @@ async def _negotiation_loop(self) -> None: print(await demand.get_data()) - logger.debug('Demand published, waiting for proposals...') + logger.debug("Demand published, waiting for proposals...") try: async for proposal in self._negotiate(demand): @@ -106,7 +105,9 @@ async def _prepare_demand_builder(self, allocation: Allocation) -> DemandBuilder # FIXME: Code looks duplicated as GolemNode.create_demand does the same demand_builder = DemandBuilder() - await demand_builder.add_default_parameters(self._demand_offer_parser, allocations=[allocation]) + await demand_builder.add_default_parameters( + self._demand_offer_parser, allocations=[allocation] + ) await demand_builder.add(self._payload) @@ -135,12 +136,13 @@ async def _negotiate_proposal( while True: demand_data_after_plugins = deepcopy(demand_data) + proposal_data = await self._get_proposal_data_from_proposal(offer_proposal) try: logger.debug(f"Applying plugins on `{offer_proposal}`...") for plugin in self._plugins: - await plugin(demand_data_after_plugins, offer_proposal) + await plugin(demand_data_after_plugins, proposal_data) except RejectProposal as e: logger.debug( @@ -183,16 +185,31 @@ async def _negotiate_proposal( return offer_proposal - async def _get_demand_data_from_demand(self, demand: Demand) -> ProposalData: + async def _get_demand_data_from_demand(self, demand: Demand) -> DemandData: # FIXME: Unnecessary serialisation from DemandBuilder to Demand, and from Demand to ProposalData data = await demand.get_data() - constraints = Constraints(self._demand_offer_parser.parse(con) for con in data.constraints) + constraints = self._demand_offer_parser.parse(data.constraints) return DemandData( properties=Properties(data.properties), constraints=constraints, demand_id=data.demand_id, requestor_id=data.requestor_id, - timestamp=data.timestamp, + timestamp=cast(datetime, data.timestamp), ) + + async def _get_proposal_data_from_proposal(self, proposal: Proposal) -> ProposalData: + data = await proposal.get_data() + + constraints = self._demand_offer_parser.parse(data.constraints) + + return ProposalData( + properties=Properties(data.properties), + constraints=constraints, + proposal_id=data.proposal_id, + issuer_id=data.issuer_id, + state=data.state, + timestamp=cast(datetime, data.timestamp), + prev_proposal_id=data.prev_proposal_id, + ) \ No newline at end of file diff --git a/golem_core/utils/asyncio.py b/golem_core/utils/asyncio.py index 12638706..e8215631 100644 --- a/golem_core/utils/asyncio.py +++ b/golem_core/utils/asyncio.py @@ -3,15 +3,17 @@ logger = logging.getLogger(__name__) + def create_task_with_logging(coro) -> asyncio.Task: task = asyncio.create_task(coro) task.add_done_callback(_handle_task_logging) return task + def _handle_task_logging(task: asyncio.Task): try: return task.result() except asyncio.CancelledError: pass except Exception: - logger.exception('Background async task encountered unhandled exception!') + logger.exception("Background async task encountered unhandled exception!") diff --git a/golem_core/utils/logging.py b/golem_core/utils/logging.py index d6de9224..fe62d81c 100644 --- a/golem_core/utils/logging.py +++ b/golem_core/utils/logging.py @@ -29,10 +29,10 @@ "level": "DEBUG", }, "golem_core.managers.negotiation": { - "level": "DEBUG", + "level": "INFO", }, "golem_core.managers.proposal": { - "level": "DEBUG", + "level": "INFO", }, }, } diff --git a/tests/unit/test_demand_builder.py b/tests/unit/test_demand_builder.py index 8e34c479..5db612d6 100644 --- a/tests/unit/test_demand_builder.py +++ b/tests/unit/test_demand_builder.py @@ -2,12 +2,7 @@ import pytest -from golem_core.core.market_api import ( - DemandBuilder, - DemandOfferBaseModel, - constraint, - prop, -) +from golem_core.core.market_api import DemandBuilder, DemandOfferBaseModel, constraint, prop @dataclass @@ -30,7 +25,7 @@ async def test_add(): "some.prop1.path": 1, "some.prop2.path": 2, } - + assert demand_builder.constraints == "(&(some.con1.path=3)\n\t(some.con2.path<=4))" diff --git a/tests/unit/test_demand_offer_cons.py b/tests/unit/test_demand_offer_cons.py index 057b6ac6..22a59613 100644 --- a/tests/unit/test_demand_offer_cons.py +++ b/tests/unit/test_demand_offer_cons.py @@ -1,10 +1,14 @@ from datetime import datetime from enum import Enum -import textwrap import pytest -from golem_core.core.props_cons.constraints import Constraint, ConstraintException, ConstraintGroup, Constraints -from golem_core.core.props_cons.properties import Properties + +from golem_core.core.props_cons.constraints import ( + Constraint, + ConstraintException, + ConstraintGroup, + Constraints, +) class ExampleEnum(Enum): @@ -12,45 +16,55 @@ class ExampleEnum(Enum): def test_constraints_serialize(): - cons = Constraints([ - Constraint("foo", '=', "bar"), - Constraint("int_field", '=', 123), - Constraint("float_field", '=', 1.5), - Constraint("datetime_field", '=', datetime(2023, 1, 2)), - Constraint("enum_field", '=', ExampleEnum.FOO), - Constraint("list_field", '=', [ - datetime(2023, 1, 2), - ExampleEnum.FOO, - ]), - ConstraintGroup([ - Constraint('some.other.field', '=', 'works!') - ], - "|"), - ]) + cons = Constraints( + [ + Constraint("foo", "=", "bar"), + Constraint("int_field", "=", 123), + Constraint("float_field", "=", 1.5), + Constraint("datetime_field", "=", datetime(2023, 1, 2)), + Constraint("enum_field", "=", ExampleEnum.FOO), + Constraint( + "list_field", + "=", + [ + datetime(2023, 1, 2), + ExampleEnum.FOO, + ], + ), + Constraint("empty_list", "=", []), + ConstraintGroup([Constraint("some.other.field", "=", "works!")], "|"), + ] + ) assert cons.serialize() == ( - '(&(foo=bar)\n' - '\t(int_field=123)\n' - '\t(float_field=1.5)\n' - '\t(datetime_field=1672614000000)\n' - '\t(enum_field=BAR)\n' - '\t(list_field=[1672614000000, BAR])\n' - '\t(some.other.field=works!))' + "(&(foo=bar)\n" + "\t(int_field=123)\n" + "\t(float_field=1.5)\n" + "\t(datetime_field=1672614000000)\n" + "\t(enum_field=BAR)\n" + "\t(list_field=[1672614000000, BAR])\n" + "\t(|(some.other.field=works!)))" ) + def test_constraint_group_raises_on_not_operator_with_multiple_items(): with pytest.raises(ConstraintException): - ConstraintGroup([ - Constraint('foo', '=', 'bar'), - Constraint('bat', '=', 'man'), - ], '!') - - cons_group = ConstraintGroup([ - Constraint('foo', '=', 'bar'), - Constraint('bat', '=', 'man'), - ]) - - cons_group.operator = '!' - + ConstraintGroup( + [ + Constraint("foo", "=", "bar"), + Constraint("bat", "=", "man"), + ], + "!", + ) + + cons_group = ConstraintGroup( + [ + Constraint("foo", "=", "bar"), + Constraint("bat", "=", "man"), + ] + ) + + cons_group.operator = "!" + with pytest.raises(ConstraintException): - cons_group.serialize() \ No newline at end of file + cons_group.serialize() diff --git a/tests/unit/test_demand_offer_parsers.py b/tests/unit/test_demand_offer_parsers.py index 28f44d5e..02d9aedf 100644 --- a/tests/unit/test_demand_offer_parsers.py +++ b/tests/unit/test_demand_offer_parsers.py @@ -8,6 +8,7 @@ def demand_offer_parser(): return TextXDemandOfferSyntaxParser() + @pytest.mark.parametrize( "input_string, output", ( @@ -27,6 +28,8 @@ def demand_offer_parser(): ("(some.nested.param=1)", Constraint("some.nested.param", "=", "1")), ("(foo<=1)", Constraint("foo", "<=", "1")), ("(foo>=1)", Constraint("foo", ">=", "1")), + ("(foo<1)", Constraint("foo", "<", "1")), + ("(foo>1)", Constraint("foo", ">", "1")), ), ) def test_single_constraint(demand_offer_parser, input_string, output): @@ -56,7 +59,6 @@ def test_single_constraint(demand_offer_parser, input_string, output): "(| (foo=1) (bar=2))", ConstraintGroup([Constraint("foo", "=", "1"), Constraint("bar", "=", "2")], "|"), ), - ( "(| (foo=1) (bar=2))", ConstraintGroup([Constraint("foo", "=", "1"), Constraint("bar", "=", "2")], "|"), diff --git a/tests/unit/test_demand_offer_props.py b/tests/unit/test_demand_offer_props.py index 456ef4c4..827e3586 100644 --- a/tests/unit/test_demand_offer_props.py +++ b/tests/unit/test_demand_offer_props.py @@ -1,5 +1,6 @@ from datetime import datetime from enum import Enum + from golem_core.core.props_cons.properties import Properties @@ -10,7 +11,7 @@ class ExampleEnum(Enum): def test_property_deepcopies_its_input(): list_field = [1, 2, 3] original_dict = { - 'foo': "bar", + "foo": "bar", "list_field": list_field, } props = Properties(original_dict) @@ -20,7 +21,7 @@ def test_property_deepcopies_its_input(): assert props["list_field"][1] == 2 assert props["list_field"][2] == 3 - props['foo'] = "123" + props["foo"] = "123" props["list_field"].append(4) assert props["foo"] == "123" @@ -29,18 +30,22 @@ def test_property_deepcopies_its_input(): assert original_dict["foo"] == "bar" assert original_dict["list_field"] != props["list_field"] + def test_property_serialize(): - props = Properties({ - "foo": "bar", - "int_field": 123, - "float_field": 1.5, - "datetime_field": datetime(2023, 1, 2), - "enum_field": ExampleEnum.FOO, - "list_field": [ - datetime(2023, 1, 2), - ExampleEnum.FOO, - ] - }) + props = Properties( + { + "foo": "bar", + "int_field": 123, + "float_field": 1.5, + "datetime_field": datetime(2023, 1, 2), + "enum_field": ExampleEnum.FOO, + "list_field": [ + datetime(2023, 1, 2), + ExampleEnum.FOO, + ], + "nulled_field": None, + } + ) serialized_props = props.serialize() @@ -53,5 +58,5 @@ def test_property_serialize(): "list_field": [ 1672614000000, "BAR", - ] + ], } From 9b79c6cbcc05653073a33cb4baeab767d948a6fb Mon Sep 17 00:00:00 2001 From: approxit Date: Thu, 22 Jun 2023 15:46:15 +0200 Subject: [PATCH 057/123] negotiation plugins done --- examples/managers/basic_composition.py | 10 +- golem_core/core/activity_api/events.py | 5 - golem_core/core/events/__init__.py | 3 - golem_core/core/events/event_bus.py | 137 ++---------------- golem_core/core/events/event_filters.py | 35 ----- golem_core/core/golem_node/golem_node.py | 12 +- golem_core/core/market_api/__init__.py | 5 +- .../core/market_api/resources/__init__.py | 6 +- .../market_api/resources/demand/__init__.py | 6 +- .../resources/demand/demand_builder.py | 14 +- .../demand/demand_offer_base/__init__.py | 6 +- .../demand/demand_offer_base/defaults.py | 8 +- .../demand/demand_offer_base/model.py | 16 +- .../demand/demand_offer_base/payload/base.py | 3 - .../demand/demand_offer_base/payload/vm.py | 14 +- .../core/market_api/resources/proposal.py | 6 +- .../core/payment_api/resources/allocation.py | 2 +- golem_core/core/props_cons/constraints.py | 2 +- golem_core/core/resources/__init__.py | 2 - golem_core/core/resources/event_filters.py | 37 ----- golem_core/managers/activity/single_use.py | 6 +- golem_core/managers/negotiation/plugins.py | 37 ++++- golem_core/managers/negotiation/sequential.py | 35 +++-- golem_core/managers/proposal/stack.py | 26 +++- golem_core/utils/logging.py | 3 + pyproject.toml | 22 ++- tests/unit/test_demand_builder.py | 99 ++++++++++++- tests/unit/test_demand_builder_model.py | 131 ++++------------- tests/unit/test_demand_offer_parsers.py | 6 + tests/unit/test_event_bus.py | 6 +- 30 files changed, 297 insertions(+), 403 deletions(-) delete mode 100644 golem_core/core/events/event_filters.py delete mode 100644 golem_core/core/resources/event_filters.py diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index 2c245001..a51dec9e 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -9,7 +9,7 @@ from golem_core.managers.agreement.single_use import SingleUseAgreementManager from golem_core.managers.base import WorkContext, WorkResult from golem_core.managers.negotiation import SequentialNegotiationManager -from golem_core.managers.negotiation.plugins import BlacklistProviderId +from golem_core.managers.negotiation.plugins import AddChosenPaymentPlatform, BlacklistProviderId from golem_core.managers.payment.pay_all import PayAllPaymentManager from golem_core.managers.proposal import StackProposalManager from golem_core.managers.work.decorators import ( @@ -62,6 +62,7 @@ async def main(): payment_manager.get_allocation, payload, plugins=[ + AddChosenPaymentPlatform(), BlacklistProviderId( [ "0x3b0f605fcb0690458064c10346af0c5f6b7202a5", @@ -76,9 +77,10 @@ async def main(): activity_manager = SingleUseActivityManager(golem, agreement_manager.get_agreement) work_manager = SequentialWorkManager(golem, activity_manager.do_work) - async with golem, payment_manager, negotiation_manager, proposal_manager: - results: List[WorkResult] = await work_manager.do_work_list(work_list) - print(f"\nWORK MANAGER RESULTS:{[result.result for result in results]}\n") + async with golem: + async with payment_manager, negotiation_manager, proposal_manager: + results: List[WorkResult] = await work_manager.do_work_list(work_list) + print(f"\nWORK MANAGER RESULTS:{[result.result for result in results]}\n") if __name__ == "__main__": diff --git a/golem_core/core/activity_api/events.py b/golem_core/core/activity_api/events.py index c6c464e0..1d0920c2 100644 --- a/golem_core/core/activity_api/events.py +++ b/golem_core/core/activity_api/events.py @@ -1,5 +1,3 @@ -from typing import TYPE_CHECKING - from golem_core.core.resources import ( NewResource, ResourceClosed, @@ -7,9 +5,6 @@ ResourceEvent, ) -if TYPE_CHECKING: - pass - class NewActivity(NewResource["Activity"]): pass diff --git a/golem_core/core/events/__init__.py b/golem_core/core/events/__init__.py index 30bce955..0ea35171 100644 --- a/golem_core/core/events/__init__.py +++ b/golem_core/core/events/__init__.py @@ -1,12 +1,9 @@ from golem_core.core.events.base import Event, EventBus, TEvent from golem_core.core.events.event_bus import InMemoryEventBus -from golem_core.core.events.event_filters import AnyEventFilter, EventFilter __all__ = ( "Event", "TEvent", "EventBus", "InMemoryEventBus", - "EventFilter", - "AnyEventFilter", ) diff --git a/golem_core/core/events/event_bus.py b/golem_core/core/events/event_bus.py index 875dfafb..87689cb1 100644 --- a/golem_core/core/events/event_bus.py +++ b/golem_core/core/events/event_bus.py @@ -2,140 +2,16 @@ import logging from collections import defaultdict from dataclasses import dataclass -from typing import ( - TYPE_CHECKING, - Any, - Awaitable, - Callable, - DefaultDict, - Iterable, - List, - Optional, - Type, -) +from typing import Awaitable, Callable, DefaultDict, List, Optional, Type from golem_core.core.events.base import Event from golem_core.core.events.base import EventBus as BaseEventBus from golem_core.core.events.base import EventBusError, TCallbackHandler, TEvent -from golem_core.core.events.event_filters import AnyEventFilter, EventFilter - -if TYPE_CHECKING: - from golem_core.core.resources import Resource, ResourceEvent, TResourceEvent - +from golem_core.utils.asyncio import create_task_with_logging logger = logging.getLogger(__name__) -class EventBus: - """Emit events, listen for events. - - This class has few purposes: - - * Easy monitoring of the execution process (just log all events) - * Convenient communication between separated parts of the code. E.g. we might want to act on - incoming invoices from a component that is not connected to any other part of the code - with - EventBus, we only have to register a callback for NewResource events. - * Using a producer-consumer pattern to implement parts of the app-specific logic. - - Sample usage:: - - async def on_allocation_event(event: ResourceEvent) -> None: - print(f"Something happened to an allocation!", event) - - golem = GolemNode() - event_bus: EventBus = golem.event_bus - event_bus.resource_listen(on_allocation_event, resource_classes=[Allocation]) - - async with golem: - # This will cause execution of on_allocation_event with a NewResource event - allocation = await golem.create_allocation(1) - # Allocation was created with autoclose=True, so now we also got a ResourceClosed event - """ - - def __init__(self) -> None: - self.queue: asyncio.Queue[Event] = asyncio.Queue() - self.consumers: DefaultDict[ - EventFilter, List[Callable[[Any], Awaitable[None]]] - ] = defaultdict(list) - - self._task: Optional[asyncio.Task] = None - - def start(self) -> None: - assert self._task is None - self._task = asyncio.create_task(self._emit_events()) - - async def stop(self) -> None: - if self._task: - await self.queue.join() - self._task.cancel() - self._task = None - - def listen( - self, - callback: Callable[[TEvent], Awaitable[None]], - classes: Iterable[Type[Event]] = (), - ) -> None: - """Execute the callback when :any:`Event` is emitted. - - :param callback: An async function to be executed. - :param classes: A list of :any:`Event` subclasses - if not empty, - `callback` will only be executed on events of matching classes. - """ - template = AnyEventFilter(tuple(classes)) - self.consumers[template].append(callback) - - def resource_listen( - self, - callback: Callable[["TResourceEvent"], Awaitable[None]], - event_classes: Iterable[Type["ResourceEvent"]] = (), - resource_classes: Iterable[Type["Resource"]] = (), - ids: Iterable[str] = (), - ) -> None: - """Execute the callback when :any:`ResourceEvent` is emitted. - - :param callback: An async function to be executed. - :param event_classes: A list of :any:`ResourceEvent` subclasses - if not empty, - `callback` will only be executed only on events of matching classes. - :param resource_classes: A list of :class:`~golem_core.core.resources.Resource` - subclasses - if not empty, `callback` will only be executed on events related to - resources of a matching class. - :param ids: A list of resource IDs - if not empty, - `callback` will only be executed on events related to resources with a matching ID. - """ - # FIXME: Get rid of local import - from golem_core.core.resources.event_filters import ResourceEventFilter - - template = ResourceEventFilter(tuple(event_classes), tuple(resource_classes), tuple(ids)) - self.consumers[template].append(callback) - - def emit(self, event: Event) -> None: - """Emit an event - execute all callbacks listening for matching events. - - If emit(X) was called before emit(Y), then it is guaranteed that callbacks - for event Y will start only after all X callbacks finished (TODO - this should change, - we don't want the EventBus to stop because of a single never-ending callback, - https://github.com/golemfactory/golem-core-python/issues/3). - - :param event: An event that will be emitted. - """ - self.queue.put_nowait(event) - - async def _emit_events(self) -> None: - while True: - event = await self.queue.get() - await self._emit(event) - - async def _emit(self, event: Event) -> None: - # TODO: https://github.com/golemfactory/golem-core-python/issues/3 - tasks = [] - for event_template, callbacks in self.consumers.items(): - if event_template.includes(event): - tasks += [callback(event) for callback in callbacks] - if tasks: - await asyncio.gather(*tasks, return_exceptions=True) - self.queue.task_done() - - @dataclass class _CallbackInfo: callback: Callable[[TEvent], Awaitable[None]] @@ -157,7 +33,9 @@ async def start(self): logger.debug(f"Starting event bus failed with `{message}`") raise EventBusError(message) - self._process_event_queue_loop_task = asyncio.create_task(self._process_event_queue_loop()) + self._process_event_queue_loop_task = create_task_with_logging( + self._process_event_queue_loop() + ) logger.debug("Starting event bus done") @@ -177,7 +55,10 @@ async def stop(self): logger.debug("Stopping event bus done") def is_started(self) -> bool: - return self._process_event_queue_loop_task is not None + return ( + self._process_event_queue_loop_task is not None + and not self._process_event_queue_loop_task.done() + ) async def on( self, diff --git a/golem_core/core/events/event_filters.py b/golem_core/core/events/event_filters.py deleted file mode 100644 index 9f931cc4..00000000 --- a/golem_core/core/events/event_filters.py +++ /dev/null @@ -1,35 +0,0 @@ -from abc import ABC, abstractmethod -from dataclasses import dataclass -from typing import Tuple, Type - -from golem_core.core.events import Event - - -###################################### -# GENERAL NOTE ABOUT EVENT FILTERS -# -# Thanks to the EventFilters, event listeners listening for the same -# sort of events are kept together. Purpose: -# * A little more efficient (less checks to execute on an single event) -# * We might want to implement one day some "this will never happen" logic -# - e.g. after we deleted a resource, it will never change again. This will not be -# possible with unstructured callables. -# -# BUT: this might end up useless - but cleanup will be very easy, as this is internal to the -# EventBus. -class EventFilter(ABC): - """Base class for all EventFilters.""" - - @abstractmethod - def includes(self, event: Event) -> bool: - raise NotImplementedError - - -@dataclass(frozen=True) -class AnyEventFilter(EventFilter): - """Selection based on the Event classes.""" - - event_classes: Tuple[Type[Event], ...] - - def includes(self, event: Event) -> bool: - return not self.event_classes or any(isinstance(event, cls) for cls in self.event_classes) diff --git a/golem_core/core/golem_node/golem_node.py b/golem_core/core/golem_node/golem_node.py index 0992196a..92c8bb8f 100644 --- a/golem_core/core/golem_node/golem_node.py +++ b/golem_core/core/golem_node/golem_node.py @@ -228,7 +228,7 @@ async def create_demand( expiration = datetime.now(timezone.utc) + DEFAULT_EXPIRATION_TIMEOUT builder = DemandBuilder() - await builder.add(dobm_defaults.Activity(expiration=expiration, multi_activity=True)) + await builder.add(dobm_defaults.ActivityInfo(expiration=expiration, multi_activity=True)) await builder.add(dobm_defaults.NodeInfo(subnet_tag=subnet)) await builder.add(payload) @@ -275,12 +275,12 @@ async def create_network( async def _add_builder_allocations( self, builder: DemandBuilder, allocations: Iterable[Allocation] ) -> None: - for allocation in allocations: - properties, constraints = await allocation.demand_properties_constraints() - builder.add_constraints(*constraints) + # TODO (?): https://github.com/golemfactory/golem-core-python/issues/35 - # TODO (?): https://github.com/golemfactory/golem-core-python/issues/35 - builder.add_properties({p.key: p.value for p in properties}) + for allocation in allocations: + properties, constraints = await allocation.get_properties_and_constraints_for_demand() + builder.add_constraints(constraints) + builder.add_properties(properties) ########################### # Single-resource factories for already existing resources diff --git a/golem_core/core/market_api/__init__.py b/golem_core/core/market_api/__init__.py index 5026bb1e..9a225d35 100644 --- a/golem_core/core/market_api/__init__.py +++ b/golem_core/core/market_api/__init__.py @@ -11,7 +11,7 @@ INF_STORAGE, RUNTIME_CAPABILITIES, RUNTIME_NAME, - Activity, + ActivityInfo, Agreement, BaseDemandOfferBaseException, ConstraintException, @@ -52,7 +52,7 @@ "INF_MEM", "INF_STORAGE", "NodeInfo", - "Activity", + "ActivityInfo", "TDemandOfferBaseModel", "DemandOfferBaseModel", "constraint", @@ -64,4 +64,5 @@ "NewAgreement", "AgreementDataChanged", "AgreementClosed", + "PaymentInfo", ) diff --git a/golem_core/core/market_api/resources/__init__.py b/golem_core/core/market_api/resources/__init__.py index a540d9e2..7c536e65 100644 --- a/golem_core/core/market_api/resources/__init__.py +++ b/golem_core/core/market_api/resources/__init__.py @@ -5,7 +5,7 @@ INF_STORAGE, RUNTIME_CAPABILITIES, RUNTIME_NAME, - Activity, + ActivityInfo, BaseDemandOfferBaseException, ConstraintException, Demand, @@ -15,6 +15,7 @@ ManifestVmPayload, NodeInfo, Payload, + PaymentInfo, RepositoryVmPayload, TDemandOfferBaseModel, VmPayloadException, @@ -42,7 +43,7 @@ "INF_MEM", "INF_STORAGE", "NodeInfo", - "Activity", + "ActivityInfo", "TDemandOfferBaseModel", "DemandOfferBaseModel", "constraint", @@ -50,4 +51,5 @@ "BaseDemandOfferBaseException", "ConstraintException", "InvalidPropertiesError", + "PaymentInfo", ) diff --git a/golem_core/core/market_api/resources/demand/__init__.py b/golem_core/core/market_api/resources/demand/__init__.py index a3f6431e..5108f598 100644 --- a/golem_core/core/market_api/resources/demand/__init__.py +++ b/golem_core/core/market_api/resources/demand/__init__.py @@ -6,7 +6,7 @@ INF_STORAGE, RUNTIME_CAPABILITIES, RUNTIME_NAME, - Activity, + ActivityInfo, BaseDemandOfferBaseException, ConstraintException, DemandOfferBaseModel, @@ -14,6 +14,7 @@ ManifestVmPayload, NodeInfo, Payload, + PaymentInfo, RepositoryVmPayload, TDemandOfferBaseModel, VmPayloadException, @@ -38,7 +39,7 @@ "INF_MEM", "INF_STORAGE", "NodeInfo", - "Activity", + "ActivityInfo", "TDemandOfferBaseModel", "DemandOfferBaseModel", "constraint", @@ -46,4 +47,5 @@ "BaseDemandOfferBaseException", "ConstraintException", "InvalidPropertiesError", + "PaymentInfo", ) diff --git a/golem_core/core/market_api/resources/demand/demand_builder.py b/golem_core/core/market_api/resources/demand/demand_builder.py index bb8e0906..5ab8f5ab 100644 --- a/golem_core/core/market_api/resources/demand/demand_builder.py +++ b/golem_core/core/market_api/resources/demand/demand_builder.py @@ -10,7 +10,7 @@ from golem_core.core.props_cons.parsers.base import DemandOfferSyntaxParser from golem_core.core.props_cons.properties import Properties -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: from golem_core.core.golem_node import GolemNode @@ -25,7 +25,7 @@ class DemandBuilder: >>> from datetime import datetime, timezone >>> builder = DemandBuilder() >>> await builder.add(defaults.NodeInfo(name="a node", subnet_tag="testnet")) - >>> await builder.add(defaults.Activity(expiration=datetime.now(timezone.utc))) + >>> await builder.add(defaults.ActivityInfo(expiration=datetime.now(timezone.utc))) >>> print(builder) {'properties': {'golem.node.id.name': 'a node', @@ -67,9 +67,12 @@ def add_properties(self, props: Properties): """Add properties from the given dictionary to this demand definition.""" self.properties.update(props) - def add_constraints(self, *constraints: Union[Constraint, ConstraintGroup]): + def add_constraints(self, constraints: Union[Constraint, ConstraintGroup]): """Add a constraint from given args to the demand definition.""" - self.constraints.items.extend(constraints) + if isinstance(constraints, ConstraintGroup) and constraints.operator == self.constraints.operator: + self.constraints.items.extend(constraints.items) + else: + self.constraints.items.append(constraints) async def add_default_parameters( self, @@ -94,13 +97,14 @@ async def add_default_parameters( if expiration is None: expiration = datetime.now(timezone.utc) + DEFAULT_EXPIRATION_TIMEOUT - await self.add(dobm_defaults.Activity(expiration=expiration, multi_activity=True)) + await self.add(dobm_defaults.ActivityInfo(expiration=expiration, multi_activity=True)) await self.add(dobm_defaults.NodeInfo(subnet_tag=subnet)) for allocation in allocations: properties, constraints = await allocation.get_properties_and_constraints_for_demand( parser ) + self.add_constraints(constraints) self.add_properties(properties) diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/__init__.py b/golem_core/core/market_api/resources/demand/demand_offer_base/__init__.py index cfadf00e..1f321ed7 100644 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/__init__.py +++ b/golem_core/core/market_api/resources/demand/demand_offer_base/__init__.py @@ -4,8 +4,9 @@ INF_STORAGE, RUNTIME_CAPABILITIES, RUNTIME_NAME, - Activity, + ActivityInfo, NodeInfo, + PaymentInfo, ) from golem_core.core.market_api.resources.demand.demand_offer_base.exceptions import ( BaseDemandOfferBaseException, @@ -36,7 +37,7 @@ "INF_MEM", "INF_STORAGE", "NodeInfo", - "Activity", + "ActivityInfo", "TDemandOfferBaseModel", "DemandOfferBaseModel", "constraint", @@ -44,4 +45,5 @@ "BaseDemandOfferBaseException", "ConstraintException", "InvalidPropertiesError", + "PaymentInfo", ) diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/defaults.py b/golem_core/core/market_api/resources/demand/demand_offer_base/defaults.py index 2c8a8d09..d7fb4373 100644 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/defaults.py +++ b/golem_core/core/market_api/resources/demand/demand_offer_base/defaults.py @@ -27,7 +27,7 @@ class NodeInfo(DemandOfferBaseModel): @dataclass -class Activity(DemandOfferBaseModel): +class ActivityInfo(DemandOfferBaseModel): """Activity-related Properties.""" cost_cap: Optional[Decimal] = prop("golem.activity.cost_cap", default=None) @@ -56,3 +56,9 @@ class Activity(DemandOfferBaseModel): multi_activity: Optional[bool] = prop("golem.srv.caps.multi-activity", default=None) """Whether client supports multi_activity (executing more than one activity per agreement). """ + + +@dataclass +class PaymentInfo(DemandOfferBaseModel): + chosen_payment_platform: Optional[str] = prop("golem.com.payment.chosen-platform", default=None) + """Payment platform selected to be used for this demand.""" diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/model.py b/golem_core/core/market_api/resources/demand/demand_offer_base/model.py index 0c1cba92..2730fccc 100644 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/model.py +++ b/golem_core/core/market_api/resources/demand/demand_offer_base/model.py @@ -47,12 +47,14 @@ def _build_properties(self) -> Properties: def _build_constraints(self) -> Constraints: """Return a serialized collection of constraint values.""" return Constraints( - Constraint( - property_name=field.metadata[PROP_KEY], - operator=field.metadata[PROP_OPERATOR], - value=getattr(self, field.name), - ) - for field in self._get_fields(DemandOfferBaseModelFieldType.constraint) + [ + Constraint( + property_name=field.metadata[PROP_KEY], + operator=field.metadata[PROP_OPERATOR], + value=getattr(self, field.name), + ) + for field in self._get_fields(DemandOfferBaseModelFieldType.constraint) + ] ) @classmethod @@ -79,7 +81,7 @@ def from_properties( for field in cls._get_fields(DemandOfferBaseModelFieldType.property) } data = { - field_map[key].name: cls.deserialize_value(val, field_map[key]) + field_map[key].name: Properties._deserialize_value(val, field_map[key]) for (key, val) in props.items() if key in field_map } diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/payload/base.py b/golem_core/core/market_api/resources/demand/demand_offer_base/payload/base.py index 90e0523a..8c3fcced 100644 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/payload/base.py +++ b/golem_core/core/market_api/resources/demand/demand_offer_base/payload/base.py @@ -37,6 +37,3 @@ async def main(): {'properties': {'golem.srv.app.myprop': 'othervalue'}, 'constraints': ['(&(golem.runtime.name=my-runtime)\n\t(golem.inf.mem.gib>=32)\n\t(golem.inf.storage.gib>=1024))']} """ # noqa: E501 - - def __hash__(self) -> int: - return hash((str(self._serialize_properties()), self._serialize_constraints())) diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/payload/vm.py b/golem_core/core/market_api/resources/demand/demand_offer_base/payload/vm.py index fcee1481..508d5688 100644 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/payload/vm.py +++ b/golem_core/core/market_api/resources/demand/demand_offer_base/payload/vm.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from datetime import timedelta from enum import Enum -from typing import Any, Dict, Final, List, Literal, Optional, Tuple +from typing import Final, List, Literal, Optional, Tuple from dns.exception import DNSException from srvresolver.srv_record import SRVRecord @@ -12,6 +12,8 @@ from golem_core.core.market_api.resources.demand.demand_offer_base import defaults from golem_core.core.market_api.resources.demand.demand_offer_base.model import constraint, prop from golem_core.core.market_api.resources.demand.demand_offer_base.payload.base import Payload +from golem_core.core.props_cons.constraints import Constraints +from golem_core.core.props_cons.properties import Properties from golem_core.utils.http import make_http_get_request, make_http_head_request DEFAULT_REPO_URL_SRV: Final[str] = "_girepo._tcp.dev.golem.network" @@ -51,9 +53,6 @@ class BaseVmPayload(Payload, ABC): "golem.srv.comp.vm.package_format", default=VmPackageFormat.GVMKIT_SQUASH ) - def __hash__(self) -> int: - return super().__hash__() - @dataclass class _VmPayload(Payload, ABC): @@ -105,14 +104,11 @@ async def _resolve_package_url(self) -> None: self.package_url = get_package_url(self.image_hash, image_url) - async def serialize(self) -> Tuple[Dict[str, Any], str]: + async def build_properties_and_constraints(self) -> Tuple[Properties, Constraints]: if self.package_url is None: await self._resolve_package_url() - return await super(RepositoryVmPayload, self).serialize() - - def __hash__(self) -> int: - return super().__hash__() + return await super().build_properties_and_constraints() async def resolve_repository_url( diff --git a/golem_core/core/market_api/resources/proposal.py b/golem_core/core/market_api/resources/proposal.py index afde82cf..1eeb5cd5 100644 --- a/golem_core/core/market_api/resources/proposal.py +++ b/golem_core/core/market_api/resources/proposal.py @@ -1,8 +1,7 @@ import asyncio from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from enum import Enum -from typing import TYPE_CHECKING, AsyncIterator, Optional, Union, Literal +from typing import TYPE_CHECKING, AsyncIterator, Literal, Optional, Union from ya_market import RequestorApi from ya_market import models as models @@ -19,7 +18,8 @@ from golem_core.core.market_api.resources.demand import Demand -ProposalState = Literal["Initial","Draft","Rejected","Accepted","Expired"] +ProposalState = Literal["Initial", "Draft", "Rejected", "Accepted", "Expired"] + @dataclass class ProposalData: diff --git a/golem_core/core/payment_api/resources/allocation.py b/golem_core/core/payment_api/resources/allocation.py index 43f964d0..c30b61a2 100644 --- a/golem_core/core/payment_api/resources/allocation.py +++ b/golem_core/core/payment_api/resources/allocation.py @@ -96,6 +96,6 @@ async def get_properties_and_constraints_for_demand( properties = Properties({prop.key: prop.value for prop in data.properties}) - constraints = Constraints(parser.parse(c) for c in data.constraints) + constraints = Constraints([parser.parse(c) for c in data.constraints]) return properties, constraints diff --git a/golem_core/core/props_cons/constraints.py b/golem_core/core/props_cons/constraints.py index 49294627..dca48196 100644 --- a/golem_core/core/props_cons/constraints.py +++ b/golem_core/core/props_cons/constraints.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import Any, MutableSequence, Union, Literal +from typing import Any, Literal, MutableSequence, Union from golem_core.core.props_cons.base import PropertyName, PropsConstrsSerializerMixin diff --git a/golem_core/core/resources/__init__.py b/golem_core/core/resources/__init__.py index 6a806ee2..2ea9cb03 100644 --- a/golem_core/core/resources/__init__.py +++ b/golem_core/core/resources/__init__.py @@ -1,6 +1,5 @@ from golem_core.core.resources.base import _NULL, Resource, TResource, api_call_wrapper from golem_core.core.resources.event_collectors import YagnaEventCollector -from golem_core.core.resources.event_filters import ResourceEventFilter from golem_core.core.resources.events import ( NewResource, ResourceClosed, @@ -32,5 +31,4 @@ "ResourceNotFound", "BaseResourceException", "MissingConfiguration", - "ResourceEventFilter", ) diff --git a/golem_core/core/resources/event_filters.py b/golem_core/core/resources/event_filters.py deleted file mode 100644 index 62ba5bb4..00000000 --- a/golem_core/core/resources/event_filters.py +++ /dev/null @@ -1,37 +0,0 @@ -from dataclasses import dataclass -from typing import TYPE_CHECKING, Tuple, Type - -from golem_core.core.events import Event, EventFilter - -if TYPE_CHECKING: - from golem_core.core.resources import Resource, ResourceEvent - - -@dataclass(frozen=True) -class ResourceEventFilter(EventFilter): - """ResourceEvents with optional filters by event class/resource type/resource id.""" - - event_classes: Tuple[Type["ResourceEvent"], ...] - resource_classes: Tuple[Type["Resource"], ...] - resource_ids: Tuple[str, ...] - - def includes(self, event: Event) -> bool: - # FIXME: Get rid of local import - from golem_core.core.resources import ResourceEvent - - if not isinstance(event, ResourceEvent): - return False - - event_class_match = not self.event_classes or any( - isinstance(event, cls) for cls in self.event_classes - ) - if not event_class_match: - return False - - resource_class_match = not self.resource_classes or any( - isinstance(event.resource, cls) for cls in self.resource_classes - ) - if not resource_class_match: - return False - - return not self.resource_ids or event.resource.id in self.resource_ids diff --git a/golem_core/managers/activity/single_use.py b/golem_core/managers/activity/single_use.py index 8cc719d4..668a3fd7 100644 --- a/golem_core/managers/activity/single_use.py +++ b/golem_core/managers/activity/single_use.py @@ -56,9 +56,9 @@ async def _prepare_activity(self) -> Activity: logger.debug("Yielding activity done") break - except Exception as e: - logger.debug( - f"Creating activity failed with `{e}`, but will be retried with new agreement" + except Exception: + logger.exception( + f"Creating activity failed, but will be retried with new agreement" ) finally: event = AgreementReleased(agreement) diff --git a/golem_core/managers/negotiation/plugins.py b/golem_core/managers/negotiation/plugins.py index 70d4610e..02b6537f 100644 --- a/golem_core/managers/negotiation/plugins.py +++ b/golem_core/managers/negotiation/plugins.py @@ -1,8 +1,9 @@ import logging -from typing import Sequence +from typing import Sequence, Set from golem_core.core.market_api.resources.demand.demand import DemandData from golem_core.core.market_api.resources.proposal import ProposalData +from golem_core.core.props_cons.properties import Properties from golem_core.managers.base import NegotiationPlugin from golem_core.managers.negotiation.sequential import RejectProposal @@ -29,5 +30,37 @@ async def __call__(self, demand_data: DemandData, proposal_data: ProposalData) - ) +class AddChosenPaymentPlatform(NegotiationPlugin): + async def __call__(self, demand_data: DemandData, proposal_data: ProposalData) -> None: + logger.debug("Calling chosen payment platform plugin...") + + if demand_data.properties.get("golem.com.payment.chosen-platform"): + logger.debug( + "Calling chosen payment platform plugin done, ignoring as platform already set" + ) + return + + demand_platforms = self._get_payment_platform_from_properties(demand_data.properties) + proposal_platforms = self._get_payment_platform_from_properties(proposal_data.properties) + common_platforms = list(demand_platforms.intersection(proposal_platforms)) + + if not common_platforms: + raise RejectProposal(f"No common payment platform!") + + chosen_platform = common_platforms[0] + + demand_data.properties["golem.com.payment.chosen-platform"] = chosen_platform + + logger.debug(f"Calling chosen payment platform plugin done with `{chosen_platform}`...") + + def _get_payment_platform_from_properties(self, properties: Properties) -> Set[str]: + return { + property.split(".")[4] + for property in properties + if property.startswith("golem.com.payment.platform.") and property is not None + } + + class MidAgreementPayment(NegotiationPlugin): - pass + async def __call__(self, demand_data: DemandData, proposal_data: ProposalData) -> None: + ... diff --git a/golem_core/managers/negotiation/sequential.py b/golem_core/managers/negotiation/sequential.py index 0ff073a3..ecf9faff 100644 --- a/golem_core/managers/negotiation/sequential.py +++ b/golem_core/managers/negotiation/sequential.py @@ -4,12 +4,13 @@ from datetime import datetime from typing import AsyncIterator, Awaitable, Callable, List, Optional, Sequence, cast +from ya_market import ApiException + from golem_core.core.golem_node.golem_node import GolemNode from golem_core.core.market_api import Demand, DemandBuilder, Payload, Proposal from golem_core.core.market_api.resources.demand.demand import DemandData from golem_core.core.market_api.resources.proposal import ProposalData from golem_core.core.payment_api import Allocation -from golem_core.core.props_cons.constraints import Constraints from golem_core.core.props_cons.parsers.textx.parser import TextXDemandOfferSyntaxParser from golem_core.core.props_cons.properties import Properties from golem_core.managers.base import ManagerException, NegotiationManager, NegotiationPlugin @@ -57,7 +58,7 @@ async def get_proposal(self) -> Proposal: async def start(self) -> None: logger.debug("Starting...") - if self.is_started_started(): + if self.is_started(): message = "Already started!" logger.debug(f"Starting failed with `{message}`") raise ManagerException(message) @@ -69,7 +70,7 @@ async def start(self) -> None: async def stop(self) -> None: logger.debug("Stopping...") - if not self.is_started_started(): + if not self.is_started(): message = "Already stopped!" logger.debug(f"Stopping failed with `{message}`") raise ManagerException(message) @@ -79,8 +80,8 @@ async def stop(self) -> None: logger.debug("Stopping done") - def is_started_started(self) -> bool: - return self._negotiation_loop_task is not None + def is_started(self) -> bool: + return self._negotiation_loop_task is not None and not self._negotiation_loop_task.done() async def _negotiation_loop(self) -> None: allocation = await self._get_allocation() @@ -89,8 +90,6 @@ async def _negotiation_loop(self) -> None: demand = await demand_builder.create_demand(self._golem) demand.start_collecting_events() - print(await demand.get_data()) - logger.debug("Demand published, waiting for proposals...") try: @@ -159,16 +158,26 @@ async def _negotiate_proposal( if offer_proposal.initial or demand_data_after_plugins != demand_data: logger.debug("Sending demand proposal...") - demand_proposal = await offer_proposal.respond( - demand_data_after_plugins.properties, - demand_data_after_plugins.constraints, - ) + demand_data = demand_data_after_plugins + + try: + demand_proposal = await offer_proposal.respond( + demand_data_after_plugins.properties, + demand_data_after_plugins.constraints, + ) + except (ApiException, asyncio.TimeoutError) as e: + logger.debug(f"Sending demand proposal failed with `{e}`") + return None logger.debug("Sending demand proposal done") logger.debug("Waiting for response...") - new_offer_proposal = await demand_proposal.responses().__anext__() + try: + new_offer_proposal = await demand_proposal.responses().__anext__() + except StopAsyncIteration: + logger.debug("Waiting for response failed with rejection") + return None logger.debug(f"Waiting for response done with `{new_offer_proposal}`") @@ -212,4 +221,4 @@ async def _get_proposal_data_from_proposal(self, proposal: Proposal) -> Proposal state=data.state, timestamp=cast(datetime, data.timestamp), prev_proposal_id=data.prev_proposal_id, - ) \ No newline at end of file + ) diff --git a/golem_core/managers/proposal/stack.py b/golem_core/managers/proposal/stack.py index 0ee8fc57..478fa708 100644 --- a/golem_core/managers/proposal/stack.py +++ b/golem_core/managers/proposal/stack.py @@ -1,10 +1,11 @@ import asyncio import logging -from typing import List +from typing import Optional from golem_core.core.golem_node.golem_node import GolemNode from golem_core.core.market_api import Proposal -from golem_core.managers.base import ProposalManager +from golem_core.managers.base import ManagerException, ProposalManager +from golem_core.utils.asyncio import create_task_with_logging logger = logging.getLogger(__name__) @@ -13,23 +14,36 @@ class StackProposalManager(ProposalManager): def __init__(self, golem: GolemNode, get_proposal) -> None: self._get_proposal = get_proposal self._proposals: asyncio.Queue[Proposal] = asyncio.Queue() - self._tasks: List[asyncio.Task] = [] + self._consume_proposals_task: Optional[asyncio.Task] = None async def start(self) -> None: logger.debug("Starting...") - self._tasks.append(asyncio.create_task(self._consume_proposals())) + if self.is_started(): + message = "Already started!" + logger.debug(f"Starting failed with `{message}`") + raise ManagerException(message) + + self._consume_proposals_task = create_task_with_logging(self._consume_proposals()) logger.debug("Starting done") async def stop(self) -> None: logger.debug("Stopping...") - for task in self._tasks: - task.cancel() + if not self.is_started(): + message = "Already stopped!" + logger.debug(f"Stopping failed with `{message}`") + raise ManagerException(message) + + self._consume_proposals_task.cancel() + self._consume_proposals_task = None logger.debug("Stopping done") + def is_started(self) -> bool: + return self._consume_proposals_task is not None and not self._consume_proposals_task.done() + async def _consume_proposals(self) -> None: while True: proposal = await self._get_proposal() diff --git a/golem_core/utils/logging.py b/golem_core/utils/logging.py index fe62d81c..7a838219 100644 --- a/golem_core/utils/logging.py +++ b/golem_core/utils/logging.py @@ -22,6 +22,9 @@ "console", ], }, + "asyncio": { + "level": "DEBUG", + }, "golem_core": { "level": "INFO", }, diff --git a/pyproject.toml b/pyproject.toml index ebd87ea3..7f2bdf2e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,8 @@ [tool.poetry] -name = "golem-core" -version = "0.1.0" -description = "Python Golem (https://www.golem.network/) Core API" -authors = ["Jan Betley ", "GolemFactory "] +name = "golem-api-python" +version = "0.1.1" +description = "Golem Netowrk (https://www.golem.network/) API for Python" +authors = ["Golem Factory "] license = "LGPL-3.0-or-later" classifiers = [ "Development Status :: 3 - Alpha", @@ -11,6 +11,7 @@ classifiers = [ "Topic :: System :: Distributed Computing" ] repository = "https://github.com/golemfactory/golem-core-python" +packages = [{ include = "golem_core" }] [build-system] requires = ["poetry_core>=1.0.0"] @@ -148,7 +149,16 @@ target-version = ['py38'] [tool.pytest.ini_options] asyncio_mode = "auto" -addopts = "--cov --cov-report html --cov-report term-missing -sv" +addopts = "--cov golem_core --cov-report html --cov-report term-missing -sv" testspaths = [ - "tests" + "tests", +] + +[tool.coverage.report] +exclude_also = [ + "if TYPE_CHECKING:", + "if __name__ == .__main__.:", + + # Don't complain about abstract methods, they aren't run: + """@(abc\\.)?abstractmethod""", ] diff --git a/tests/unit/test_demand_builder.py b/tests/unit/test_demand_builder.py index 5db612d6..e85d6cc6 100644 --- a/tests/unit/test_demand_builder.py +++ b/tests/unit/test_demand_builder.py @@ -1,8 +1,8 @@ from dataclasses import dataclass -import pytest - from golem_core.core.market_api import DemandBuilder, DemandOfferBaseModel, constraint, prop +from golem_core.core.props_cons.constraints import Constraint, Constraints, ConstraintGroup +from golem_core.core.props_cons.properties import Properties @dataclass @@ -13,7 +13,6 @@ class ExampleModel(DemandOfferBaseModel): con2: int = constraint("some.con2.path", "<=") -@pytest.mark.asyncio async def test_add(): model = ExampleModel(prop1=1, prop2=2, con1=3, con2=4) @@ -25,15 +24,101 @@ async def test_add(): "some.prop1.path": 1, "some.prop2.path": 2, } - - assert demand_builder.constraints == "(&(some.con1.path=3)\n\t(some.con2.path<=4))" + assert demand_builder.constraints == Constraints( + [ + Constraint("some.con1.path", "=", 3), + Constraint("some.con2.path", "<=", 4), + ] + ) + +def test_add_properties(): + demand_builder = DemandBuilder() + + assert demand_builder.properties.get('foo') != 'bar' + + demand_builder.add_properties(Properties({ + 'foo': 'bar', + })) + + assert demand_builder.properties.get('foo') == 'bar' + + demand_builder.add_properties(Properties({ + 'foo': '123', + 'bat': 'man', + })) + + assert demand_builder.properties.get('foo') == '123' + assert demand_builder.properties.get('bat') == 'man' + +def test_add_constraints(): + demand_builder = DemandBuilder() + + assert demand_builder.constraints == Constraints() + + demand_builder.add_constraints(Constraints([ + Constraint('foo', '=', 'bar'), + ])) + + assert demand_builder.constraints == Constraints([ + Constraint('foo', '=', 'bar'), + ]) + + demand_builder.add_constraints(Constraints([ + Constraint('another.field', '<=', 'value1'), + Constraint('collection.to.add', '>=', 'value2'), + ])) + + assert demand_builder.constraints == Constraints([ + Constraint('foo', '=', 'bar'), + Constraint('another.field', '<=', 'value1'), + Constraint('collection.to.add', '>=', 'value2'), + ]) + + demand_builder.add_constraints(Constraint('single.field', '=', 'works too!')) + + assert demand_builder.constraints == Constraints([ + Constraint('foo', '=', 'bar'), + Constraint('another.field', '<=', 'value1'), + Constraint('collection.to.add', '>=', 'value2'), + Constraint('single.field', '=', 'works too!'), + ]) + + demand_builder.add_constraints(ConstraintGroup([ + Constraint('field.group', '=', 'works too!') + ], "|")) + + assert demand_builder.constraints == Constraints([ + Constraint('foo', '=', 'bar'), + Constraint('another.field', '<=', 'value1'), + Constraint('collection.to.add', '>=', 'value2'), + Constraint('single.field', '=', 'works too!'), + ConstraintGroup([ + Constraint('field.group', '=', 'works too!') + ], "|"), + ]) def test_repr(): - assert str(DemandBuilder()) == "{'properties': {}, 'constraints': []}" + assert ( + str(DemandBuilder()) + == "{'properties': {}, 'constraints': Constraints(items=[], operator='&')}" + ) + +def test_comparison(): + builder_1 = DemandBuilder(Properties({ + 'foo': 'bar', + })) + builder_2 = DemandBuilder(Properties({ + 'foo': 'bar', + })) + builder_3 = DemandBuilder(Properties({ + 'foo': 123, + })) + + assert builder_1 == builder_2 + assert builder_1 != builder_3 -@pytest.mark.asyncio async def test_create_demand(mocker): demand_builder = DemandBuilder() diff --git a/tests/unit/test_demand_builder_model.py b/tests/unit/test_demand_builder_model.py index 54d337a8..4342257d 100644 --- a/tests/unit/test_demand_builder_model.py +++ b/tests/unit/test_demand_builder_model.py @@ -6,12 +6,13 @@ import pytest from golem_core.core.market_api import ( - ConstraintException, DemandOfferBaseModel, InvalidPropertiesError, constraint, prop, ) +from golem_core.core.props_cons.constraints import Constraint, Constraints +from golem_core.core.props_cons.properties import Properties class ExampleEnum(Enum): @@ -54,83 +55,46 @@ class FooZero(DemandOfferBaseModel): ( ( Foo(), - {"bar.dotted.path": "cafebiba"}, - "(&(baz<=100)\n\t(baz>=1))", + Properties({"bar.dotted.path": "cafebiba"}), + Constraints( + [ + Constraint("baz", "<=", 100), + Constraint("baz", ">=", 1), + Constraint("lst", "=", []), + ] + ), ), ( Foo(bar="bar", min_baz=54, max_baz=200), - {"bar.dotted.path": "bar"}, - "(&(baz<=200)\n\t(baz>=54))", + Properties({"bar.dotted.path": "bar"}), + Constraints( + [ + Constraint("baz", "<=", 200), + Constraint("baz", ">=", 54), + Constraint("lst", "=", []), + ] + ), ), ( Foo(lst=["some", 1, "value", 2]), - {"bar.dotted.path": "cafebiba"}, - "(&(baz<=100)\n\t(baz>=1)\n\t(&(lst=some)\n\t(lst=1)\n\t(lst=value)\n\t(lst=2)))", + Properties({"bar.dotted.path": "cafebiba"}), + Constraints( + [ + Constraint("baz", "<=", 100), + Constraint("baz", ">=", 1), + Constraint("lst", "=", ["some", 1, "value", 2]), + ] + ), ), ), ) -@pytest.mark.asyncio async def test_serialize(model, expected_properties, expected_constraints): - properties, constraints = await model.serialize() + properties, constraints = await model.build_properties_and_constraints() assert properties == expected_properties assert constraints == expected_constraints -@pytest.mark.parametrize( - "value, expected", - ( - ( - [1, 2, 3], - [1, 2, 3], - ), - ( - (1, 2, 3), - (1, 2, 3), - ), - ( - datetime.datetime(2023, 4, 6, 12, 54, 50, tzinfo=datetime.timezone.utc), - 1680785690000, - ), - (ExampleEnum.ONE, "one"), - ( - [ - [1, 2], - (ExampleEnum.ONE, ExampleEnum.TWO), - datetime.datetime(2023, 4, 6, 12, 54, 50, tzinfo=datetime.timezone.utc), - ], - [[1, 2], ("one", "two"), 1680785690000], - ), - ), -) -def test_serialize_value(value, expected): - assert DemandOfferBaseModel.serialize_value(value) == expected - - -@pytest.mark.parametrize( - "value, field, expected", - ( - ( - 123, - FooTooFields["baz"], - 123, - ), - ( - "one", - FooTooFields["en"], - ExampleEnum.ONE, - ), - ( - 1680785690000, - FooTooFields["created_at"], - datetime.datetime(2023, 4, 6, 12, 54, 50, tzinfo=datetime.timezone.utc), - ), - ), -) -def test_deserialize_value(value, field, expected): - assert DemandOfferBaseModel.deserialize_value(value, field) == expected - - def test_from_properties(): model = FooToo.from_properties( { @@ -173,44 +137,3 @@ def test_from_properties_custom_validation(): "extra_field": "should_be_ignored", } ) - - -@pytest.mark.parametrize( - "items, operator, expected", - ( - ( - ["A", "B", "C"], - "&", - "(&A\n\tB\n\tC)", - ), - ( - ["A", "B", "C"], - "|", - "(|A\n\tB\n\tC)", - ), - ( - [ - "A", - ], - "!", - "(!A)", - ), - ( - [], - "&", - "(&)", - ), - ( - ["A"], - "&", - "A", - ), - ), -) -def test_join_str_constraints(items, operator, expected): - assert join_str_constraints(items, operator) == expected - - -def test_join_str_constraints_negation_with_multiple_constraints(): - with pytest.raises(ConstraintException): - join_str_constraints(["A", "B", "C"], "!") diff --git a/tests/unit/test_demand_offer_parsers.py b/tests/unit/test_demand_offer_parsers.py index 02d9aedf..1df96be3 100644 --- a/tests/unit/test_demand_offer_parsers.py +++ b/tests/unit/test_demand_offer_parsers.py @@ -1,6 +1,7 @@ import pytest from golem_core.core.props_cons.constraints import Constraint, ConstraintException, ConstraintGroup +from golem_core.core.props_cons.parsers.base import SyntaxException from golem_core.core.props_cons.parsers.textx.parser import TextXDemandOfferSyntaxParser @@ -9,6 +10,11 @@ def demand_offer_parser(): return TextXDemandOfferSyntaxParser() +def test_parse_raises_exception_on_bad_syntax(demand_offer_parser): + with pytest.raises(SyntaxException): + demand_offer_parser.parse("NOT VALID SYNTAX") + + @pytest.mark.parametrize( "input_string, output", ( diff --git a/tests/unit/test_event_bus.py b/tests/unit/test_event_bus.py index 77a4c747..72de8c06 100644 --- a/tests/unit/test_event_bus.py +++ b/tests/unit/test_event_bus.py @@ -97,7 +97,7 @@ async def test_emit_multiple(mocker, event_bus): async def test_emit_once(mocker, event_bus): - callback_mock = mocker.Mock() + callback_mock = mocker.AsyncMock() callback_handler = await event_bus.on_once(ExampleEvent, callback_mock) @@ -111,9 +111,7 @@ async def test_emit_once(mocker, event_bus): await event_bus.stop() # Waits for all callbacks to be called - assert callback_mock.mock_calls == [ - mocker.call(event1), - ] + callback_mock.assert_called_once_with(event1) with pytest.raises(EventBusError, match="callback handler is not found"): await event_bus.off(callback_handler) From da8fb8dc79cf3ed854c361af3a6aba433d39d729 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 26 Jun 2023 15:23:48 +0200 Subject: [PATCH 058/123] Fix core example 3 --- golem_core/core/golem_node/golem_node.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/golem_core/core/golem_node/golem_node.py b/golem_core/core/golem_node/golem_node.py index 92c8bb8f..72d5ce2a 100644 --- a/golem_core/core/golem_node/golem_node.py +++ b/golem_core/core/golem_node/golem_node.py @@ -278,7 +278,9 @@ async def _add_builder_allocations( # TODO (?): https://github.com/golemfactory/golem-core-python/issues/35 for allocation in allocations: - properties, constraints = await allocation.get_properties_and_constraints_for_demand() + properties, constraints = await allocation.get_properties_and_constraints_for_demand( + self._demand_offer_syntax_parser + ) builder.add_constraints(constraints) builder.add_properties(properties) From 111db3a15749af37adc5c17158d39785c33b8139 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 27 Jun 2023 11:37:58 +0200 Subject: [PATCH 059/123] Fix con list serialization --- .../resources/demand/demand_builder.py | 5 +- golem_core/core/props_cons/constraints.py | 4 +- tests/unit/test_demand_builder.py | 162 +++++++++++------- tests/unit/test_demand_offer_cons.py | 3 +- 4 files changed, 109 insertions(+), 65 deletions(-) diff --git a/golem_core/core/market_api/resources/demand/demand_builder.py b/golem_core/core/market_api/resources/demand/demand_builder.py index 5ab8f5ab..97e8be88 100644 --- a/golem_core/core/market_api/resources/demand/demand_builder.py +++ b/golem_core/core/market_api/resources/demand/demand_builder.py @@ -69,7 +69,10 @@ def add_properties(self, props: Properties): def add_constraints(self, constraints: Union[Constraint, ConstraintGroup]): """Add a constraint from given args to the demand definition.""" - if isinstance(constraints, ConstraintGroup) and constraints.operator == self.constraints.operator: + if ( + isinstance(constraints, ConstraintGroup) + and constraints.operator == self.constraints.operator + ): self.constraints.items.extend(constraints.items) else: self.constraints.items.append(constraints) diff --git a/golem_core/core/props_cons/constraints.py b/golem_core/core/props_cons/constraints.py index dca48196..0ebdaa87 100644 --- a/golem_core/core/props_cons/constraints.py +++ b/golem_core/core/props_cons/constraints.py @@ -44,7 +44,9 @@ def _serialize(self) -> str: if not self.value: return "" - serialized_value = "[{}]".format(", ".join(str(v) for v in serialized_value)) + return ConstraintGroup( + [Constraint(self.property_name, self.operator, v) for v in serialized_value] + ).serialize() return f"({self.property_name}{self.operator}{serialized_value})" diff --git a/tests/unit/test_demand_builder.py b/tests/unit/test_demand_builder.py index e85d6cc6..7775f8b7 100644 --- a/tests/unit/test_demand_builder.py +++ b/tests/unit/test_demand_builder.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from golem_core.core.market_api import DemandBuilder, DemandOfferBaseModel, constraint, prop -from golem_core.core.props_cons.constraints import Constraint, Constraints, ConstraintGroup +from golem_core.core.props_cons.constraints import Constraint, ConstraintGroup, Constraints from golem_core.core.props_cons.properties import Properties @@ -32,71 +32,96 @@ async def test_add(): ] ) + def test_add_properties(): demand_builder = DemandBuilder() - assert demand_builder.properties.get('foo') != 'bar' + assert demand_builder.properties.get("foo") != "bar" - demand_builder.add_properties(Properties({ - 'foo': 'bar', - })) + demand_builder.add_properties( + Properties( + { + "foo": "bar", + } + ) + ) - assert demand_builder.properties.get('foo') == 'bar' + assert demand_builder.properties.get("foo") == "bar" - demand_builder.add_properties(Properties({ - 'foo': '123', - 'bat': 'man', - })) + demand_builder.add_properties( + Properties( + { + "foo": "123", + "bat": "man", + } + ) + ) + + assert demand_builder.properties.get("foo") == "123" + assert demand_builder.properties.get("bat") == "man" - assert demand_builder.properties.get('foo') == '123' - assert demand_builder.properties.get('bat') == 'man' def test_add_constraints(): demand_builder = DemandBuilder() assert demand_builder.constraints == Constraints() - demand_builder.add_constraints(Constraints([ - Constraint('foo', '=', 'bar'), - ])) - - assert demand_builder.constraints == Constraints([ - Constraint('foo', '=', 'bar'), - ]) - - demand_builder.add_constraints(Constraints([ - Constraint('another.field', '<=', 'value1'), - Constraint('collection.to.add', '>=', 'value2'), - ])) - - assert demand_builder.constraints == Constraints([ - Constraint('foo', '=', 'bar'), - Constraint('another.field', '<=', 'value1'), - Constraint('collection.to.add', '>=', 'value2'), - ]) - - demand_builder.add_constraints(Constraint('single.field', '=', 'works too!')) - - assert demand_builder.constraints == Constraints([ - Constraint('foo', '=', 'bar'), - Constraint('another.field', '<=', 'value1'), - Constraint('collection.to.add', '>=', 'value2'), - Constraint('single.field', '=', 'works too!'), - ]) - - demand_builder.add_constraints(ConstraintGroup([ - Constraint('field.group', '=', 'works too!') - ], "|")) - - assert demand_builder.constraints == Constraints([ - Constraint('foo', '=', 'bar'), - Constraint('another.field', '<=', 'value1'), - Constraint('collection.to.add', '>=', 'value2'), - Constraint('single.field', '=', 'works too!'), - ConstraintGroup([ - Constraint('field.group', '=', 'works too!') - ], "|"), - ]) + demand_builder.add_constraints( + Constraints( + [ + Constraint("foo", "=", "bar"), + ] + ) + ) + + assert demand_builder.constraints == Constraints( + [ + Constraint("foo", "=", "bar"), + ] + ) + + demand_builder.add_constraints( + Constraints( + [ + Constraint("another.field", "<=", "value1"), + Constraint("collection.to.add", ">=", "value2"), + ] + ) + ) + + assert demand_builder.constraints == Constraints( + [ + Constraint("foo", "=", "bar"), + Constraint("another.field", "<=", "value1"), + Constraint("collection.to.add", ">=", "value2"), + ] + ) + + demand_builder.add_constraints(Constraint("single.field", "=", "works too!")) + + assert demand_builder.constraints == Constraints( + [ + Constraint("foo", "=", "bar"), + Constraint("another.field", "<=", "value1"), + Constraint("collection.to.add", ">=", "value2"), + Constraint("single.field", "=", "works too!"), + ] + ) + + demand_builder.add_constraints( + ConstraintGroup([Constraint("field.group", "=", "works too!")], "|") + ) + + assert demand_builder.constraints == Constraints( + [ + Constraint("foo", "=", "bar"), + Constraint("another.field", "<=", "value1"), + Constraint("collection.to.add", ">=", "value2"), + Constraint("single.field", "=", "works too!"), + ConstraintGroup([Constraint("field.group", "=", "works too!")], "|"), + ] + ) + def test_repr(): assert ( @@ -104,16 +129,29 @@ def test_repr(): == "{'properties': {}, 'constraints': Constraints(items=[], operator='&')}" ) + def test_comparison(): - builder_1 = DemandBuilder(Properties({ - 'foo': 'bar', - })) - builder_2 = DemandBuilder(Properties({ - 'foo': 'bar', - })) - builder_3 = DemandBuilder(Properties({ - 'foo': 123, - })) + builder_1 = DemandBuilder( + Properties( + { + "foo": "bar", + } + ) + ) + builder_2 = DemandBuilder( + Properties( + { + "foo": "bar", + } + ) + ) + builder_3 = DemandBuilder( + Properties( + { + "foo": 123, + } + ) + ) assert builder_1 == builder_2 assert builder_1 != builder_3 diff --git a/tests/unit/test_demand_offer_cons.py b/tests/unit/test_demand_offer_cons.py index 22a59613..a831a141 100644 --- a/tests/unit/test_demand_offer_cons.py +++ b/tests/unit/test_demand_offer_cons.py @@ -42,7 +42,8 @@ def test_constraints_serialize(): "\t(float_field=1.5)\n" "\t(datetime_field=1672614000000)\n" "\t(enum_field=BAR)\n" - "\t(list_field=[1672614000000, BAR])\n" + "\t(&(list_field=1672614000000)\n" + "\t(list_field=BAR))\n" "\t(|(some.other.field=works!)))" ) From 9c460c39630b8d11f9da72c1938bdffdf51429e0 Mon Sep 17 00:00:00 2001 From: approxit Date: Tue, 27 Jun 2023 10:42:42 +0200 Subject: [PATCH 060/123] exports reorganized --- README.md | 18 ++-- examples/attach.py | 9 +- examples/core_example.py | 8 +- examples/detached_activity.py | 8 +- .../exception_handling/exception_handling.py | 14 +-- examples/managers/basic_composition.py | 24 ++--- examples/managers/ssh.py | 22 ++--- examples/rate_providers/rate_providers.py | 14 +-- .../score_based_providers.py | 14 +-- examples/service.py | 14 +-- .../examples/blender/blender.py | 4 +- .../examples/execute_tasks_hello_world.py | 4 +- .../examples/pipeline_example.py | 14 +-- .../task_api_draft/examples/redundance.py | 4 +- examples/task_api_draft/examples/yacat.py | 20 ++-- .../examples/yacat_no_business_logic.py | 4 +- .../task_api_draft/task_api/activity_pool.py | 10 +- .../task_api_draft/task_api/execute_tasks.py | 14 +-- .../task_api/redundance_manager.py | 4 +- {golem_core => golem}/__init__.py | 0 {golem_core => golem}/__main__.py | 2 +- golem/cli/__init__.py | 3 + {golem_core => golem}/cli/cli.py | 6 +- {golem_core => golem}/cli/utils.py | 6 +- golem/event_bus/__init__.py | 7 ++ .../core/events => golem/event_bus}/base.py | 12 +-- golem/event_bus/in_memory/__init__.py | 5 + .../event_bus/in_memory}/event_bus.py | 8 +- golem/exceptions.py | 2 + golem/managers/__init__.py | 3 + .../managers/activity/__init__.py | 4 +- .../managers/activity/defaults.py | 2 +- .../managers/activity/single_use.py | 12 +-- golem/managers/agreement/__init__.py | 7 ++ .../managers/agreement/events.py | 2 +- .../managers/agreement/single_use.py | 8 +- {golem_core => golem}/managers/base.py | 16 ++-- golem/managers/negotiation/__init__.py | 3 + .../managers/negotiation/plugins.py | 10 +- .../managers/negotiation/sequential.py | 22 ++--- .../managers/network}/__init__.py | 0 .../managers/network/single.py | 10 +- golem/managers/payment/__init__.py | 3 + .../managers/payment/default.py | 12 +-- .../managers/payment/pay_all.py | 15 +-- golem/managers/proposal/__init__.py | 3 + .../managers/proposal/stack.py | 8 +- .../managers/work/__init__.py | 4 +- .../managers/work/decorators.py | 2 +- .../managers/work/sequential.py | 4 +- .../golem_node => golem/node}/__init__.py | 5 +- .../resources/low.py => golem/node/api.py | 6 +- golem/node/event_collectors/__init__.py | 8 ++ .../node}/event_collectors/base.py | 2 +- .../node}/event_collectors/utils.py | 2 +- .../core/golem_node => golem/node}/events.py | 4 +- .../golem_node.py => golem/node/node.py | 18 ++-- golem/payload/__init__.py | 18 ++++ .../model.py => golem/payload/base.py | 94 +++++++++++++------ .../payload}/constraints.py | 9 +- .../payload}/defaults.py | 2 +- golem/payload/exceptions.py | 13 +++ .../base.py => golem/payload/mixins.py | 7 +- golem/payload/parsers/__init__.py | 7 ++ golem/payload/parsers/base.py | 13 +++ golem/payload/parsers/textx/__init__.py | 5 + .../payload}/parsers/textx/parser.py | 8 +- .../payload}/parsers/textx/syntax.tx | 0 .../payload}/properties.py | 4 +- .../demand_offer_base => golem}/payload/vm.py | 16 ++-- golem/pipeline/__init__.py | 15 +++ {golem_core => golem}/pipeline/buffer.py | 2 +- {golem_core => golem}/pipeline/chain.py | 4 +- {golem_core => golem}/pipeline/exceptions.py | 4 +- {golem_core => golem}/pipeline/limit.py | 0 {golem_core => golem}/pipeline/map.py | 2 +- {golem_core => golem}/pipeline/sort.py | 0 {golem_core => golem}/pipeline/zip.py | 0 .../resources}/__init__.py | 0 golem/resources/activity/__init__.py | 27 ++++++ .../resources/activity}/activity.py | 20 ++-- .../resources/activity}/commands.py | 2 +- golem/resources/activity/events.py | 10 ++ .../resources/activity}/pipeline.py | 4 +- golem/resources/agreement/__init__.py | 11 +++ .../resources/agreement}/agreement.py | 22 ++--- golem/resources/agreement/events.py | 10 ++ golem/resources/agreement/pipeline.py | 3 + golem/resources/allocation/__init__.py | 13 +++ .../resources/allocation}/allocation.py | 22 ++--- golem/resources/allocation/events.py | 10 ++ .../resources/allocation}/exceptions.py | 10 +- {golem_core/core => golem}/resources/base.py | 10 +- golem/resources/debit_note/__init__.py | 10 ++ .../resources/debit_note}/debit_note.py | 12 +-- .../resources/debit_note/event_collectors.py | 20 ++++ golem/resources/debit_note/events.py | 10 ++ golem/resources/demand/__init__.py | 11 +++ .../resources/demand/demand.py | 14 +-- .../resources/demand/demand_builder.py | 22 ++--- golem/resources/demand/events.py | 10 ++ golem/resources/event_collectors.py | 48 ++++++++++ .../core => golem}/resources/events.py | 4 +- .../core => golem}/resources/exceptions.py | 10 +- golem/resources/invoice/__init__.py | 10 ++ golem/resources/invoice/event_collectors.py | 20 ++++ golem/resources/invoice/events.py | 13 +++ .../resources/invoice}/invoice.py | 12 +-- golem/resources/network/__init__.py | 13 +++ .../resources/network}/events.py | 2 +- .../resources/network}/exceptions.py | 8 +- .../resources/network}/network.py | 10 +- golem/resources/pooling_batch/__init__.py | 20 ++++ golem/resources/pooling_batch/events.py | 9 ++ .../resources/pooling_batch}/exceptions.py | 10 +- .../resources/pooling_batch}/pooling_batch.py | 12 +-- golem/resources/proposal/__init__.py | 13 +++ golem/resources/proposal/events.py | 13 +++ .../resources/proposal}/pipeline.py | 6 +- .../resources/proposal}/proposal.py | 18 ++-- .../parsers => golem/utils}/__init__.py | 0 {golem_core => golem}/utils/asyncio.py | 0 {golem_core => golem}/utils/http.py | 0 {golem_core => golem}/utils/logging.py | 13 ++- golem/utils/storage/__init__.py | 7 ++ .../common.py => golem/utils/storage/base.py | 0 .../utils/storage/gftp}/__init__.py | 0 .../utils/storage/gftp/provider.py | 4 +- {golem_core => golem}/utils/storage/utils.py | 0 {golem_core => golem}/utils/typing.py | 0 golem_core/cli/__init__.py | 3 - golem_core/core/activity_api/__init__.py | 38 -------- golem_core/core/activity_api/events.py | 29 ------ .../core/activity_api/resources/__init__.py | 7 -- golem_core/core/events/__init__.py | 9 -- golem_core/core/exceptions.py | 5 - golem_core/core/market_api/__init__.py | 68 -------------- golem_core/core/market_api/events.py | 37 -------- golem_core/core/market_api/exceptions.py | 5 - .../core/market_api/resources/__init__.py | 55 ----------- .../market_api/resources/demand/__init__.py | 51 ---------- .../demand/demand_offer_base/__init__.py | 49 ---------- .../demand/demand_offer_base/exceptions.py | 13 --- .../demand_offer_base/payload/__init__.py | 13 --- .../demand/demand_offer_base/payload/base.py | 39 -------- golem_core/core/network_api/__init__.py | 8 -- .../core/network_api/resources/__init__.py | 3 - golem_core/core/payment_api/__init__.py | 22 ----- golem_core/core/payment_api/events.py | 37 -------- .../core/payment_api/resources/__init__.py | 17 ---- .../payment_api/resources/event_collectors.py | 78 --------------- golem_core/core/props_cons/parsers/base.py | 13 --- golem_core/core/resources/__init__.py | 34 ------- .../resources/event_collectors/__init__.py | 11 --- golem_core/exceptions.py | 2 - golem_core/managers/__init__.py | 3 - golem_core/managers/agreement/__init__.py | 7 -- golem_core/managers/negotiation/__init__.py | 3 - golem_core/managers/network/__init__.py | 0 golem_core/managers/payment/__init__.py | 3 - golem_core/managers/proposal/__init__.py | 3 - golem_core/pipeline/__init__.py | 15 --- golem_core/utils/__init__.py | 0 golem_core/utils/storage/__init__.py | 8 -- tests/integration/helpers.py | 10 +- tests/integration/test_1.py | 8 +- tests/integration/test_app_session_id.py | 6 +- tests/unit/test_app_session_id.py | 2 +- tests/unit/test_command.py | 2 +- tests/unit/test_demand_builder.py | 8 +- tests/unit/test_demand_builder_model.py | 6 +- tests/unit/test_demand_offer_cons.py | 2 +- tests/unit/test_demand_offer_parsers.py | 18 ++-- tests/unit/test_demand_offer_props.py | 2 +- tests/unit/test_event_bus.py | 6 +- tests/unit/test_mid.py | 2 +- tests/unit/utils/test_storage.py | 2 +- 177 files changed, 888 insertions(+), 1124 deletions(-) rename {golem_core => golem}/__init__.py (100%) rename {golem_core => golem}/__main__.py (55%) create mode 100644 golem/cli/__init__.py rename {golem_core => golem}/cli/cli.py (92%) rename {golem_core => golem}/cli/utils.py (95%) create mode 100644 golem/event_bus/__init__.py rename {golem_core/core/events => golem/event_bus}/base.py (87%) create mode 100644 golem/event_bus/in_memory/__init__.py rename {golem_core/core/events => golem/event_bus/in_memory}/event_bus.py (96%) create mode 100644 golem/exceptions.py create mode 100644 golem/managers/__init__.py rename {golem_core => golem}/managers/activity/__init__.py (57%) rename {golem_core => golem}/managers/activity/defaults.py (89%) rename {golem_core => golem}/managers/activity/single_use.py (91%) create mode 100644 golem/managers/agreement/__init__.py rename {golem_core => golem}/managers/agreement/events.py (50%) rename {golem_core => golem}/managers/agreement/single_use.py (89%) rename {golem_core => golem}/managers/base.py (87%) create mode 100644 golem/managers/negotiation/__init__.py rename {golem_core => golem}/managers/negotiation/plugins.py (87%) rename {golem_core => golem}/managers/negotiation/sequential.py (90%) rename {golem_core/core => golem/managers/network}/__init__.py (100%) rename {golem_core => golem}/managers/network/single.py (85%) create mode 100644 golem/managers/payment/__init__.py rename {golem_core => golem}/managers/payment/default.py (91%) rename {golem_core => golem}/managers/payment/pay_all.py (89%) create mode 100644 golem/managers/proposal/__init__.py rename {golem_core => golem}/managers/proposal/stack.py (88%) rename {golem_core => golem}/managers/work/__init__.py (62%) rename {golem_core => golem}/managers/work/decorators.py (95%) rename {golem_core => golem}/managers/work/sequential.py (91%) rename {golem_core/core/golem_node => golem/node}/__init__.py (64%) rename golem_core/core/resources/low.py => golem/node/api.py (97%) create mode 100644 golem/node/event_collectors/__init__.py rename {golem_core/core/resources => golem/node}/event_collectors/base.py (97%) rename {golem_core/core/resources => golem/node}/event_collectors/utils.py (96%) rename {golem_core/core/golem_node => golem/node}/events.py (89%) rename golem_core/core/golem_node/golem_node.py => golem/node/node.py (96%) create mode 100644 golem/payload/__init__.py rename golem_core/core/market_api/resources/demand/demand_offer_base/model.py => golem/payload/base.py (61%) rename {golem_core/core/props_cons => golem/payload}/constraints.py (86%) rename {golem_core/core/market_api/resources/demand/demand_offer_base => golem/payload}/defaults.py (96%) create mode 100644 golem/payload/exceptions.py rename golem_core/core/props_cons/base.py => golem/payload/mixins.py (90%) create mode 100644 golem/payload/parsers/__init__.py create mode 100644 golem/payload/parsers/base.py create mode 100644 golem/payload/parsers/textx/__init__.py rename {golem_core/core/props_cons => golem/payload}/parsers/textx/parser.py (71%) rename {golem_core/core/props_cons => golem/payload}/parsers/textx/syntax.tx (100%) rename {golem_core/core/props_cons => golem/payload}/properties.py (85%) rename {golem_core/core/market_api/resources/demand/demand_offer_base => golem}/payload/vm.py (89%) create mode 100644 golem/pipeline/__init__.py rename {golem_core => golem}/pipeline/buffer.py (98%) rename {golem_core => golem}/pipeline/chain.py (94%) rename {golem_core => golem}/pipeline/exceptions.py (92%) rename {golem_core => golem}/pipeline/limit.py (100%) rename {golem_core => golem}/pipeline/map.py (98%) rename {golem_core => golem}/pipeline/sort.py (100%) rename {golem_core => golem}/pipeline/zip.py (100%) rename {golem_core/core/props_cons => golem/resources}/__init__.py (100%) create mode 100644 golem/resources/activity/__init__.py rename {golem_core/core/activity_api/resources => golem/resources/activity}/activity.py (89%) rename {golem_core/core/activity_api => golem/resources/activity}/commands.py (98%) create mode 100644 golem/resources/activity/events.py rename {golem_core/core/activity_api => golem/resources/activity}/pipeline.py (83%) create mode 100644 golem/resources/agreement/__init__.py rename {golem_core/core/market_api/resources => golem/resources/agreement}/agreement.py (88%) create mode 100644 golem/resources/agreement/events.py create mode 100644 golem/resources/agreement/pipeline.py create mode 100644 golem/resources/allocation/__init__.py rename {golem_core/core/payment_api/resources => golem/resources/allocation}/allocation.py (81%) create mode 100644 golem/resources/allocation/events.py rename {golem_core/core/payment_api => golem/resources/allocation}/exceptions.py (73%) rename {golem_core/core => golem}/resources/base.py (96%) create mode 100644 golem/resources/debit_note/__init__.py rename {golem_core/core/payment_api/resources => golem/resources/debit_note}/debit_note.py (76%) create mode 100644 golem/resources/debit_note/event_collectors.py create mode 100644 golem/resources/debit_note/events.py create mode 100644 golem/resources/demand/__init__.py rename {golem_core/core/market_api => golem}/resources/demand/demand.py (91%) rename {golem_core/core/market_api => golem}/resources/demand/demand_builder.py (82%) create mode 100644 golem/resources/demand/events.py create mode 100644 golem/resources/event_collectors.py rename {golem_core/core => golem}/resources/events.py (97%) rename {golem_core/core => golem}/resources/exceptions.py (79%) create mode 100644 golem/resources/invoice/__init__.py create mode 100644 golem/resources/invoice/event_collectors.py create mode 100644 golem/resources/invoice/events.py rename {golem_core/core/payment_api/resources => golem/resources/invoice}/invoice.py (75%) create mode 100644 golem/resources/network/__init__.py rename {golem_core/core/network_api => golem/resources/network}/events.py (80%) rename {golem_core/core/network_api => golem/resources/network}/exceptions.py (65%) rename {golem_core/core/network_api/resources => golem/resources/network}/network.py (94%) create mode 100644 golem/resources/pooling_batch/__init__.py create mode 100644 golem/resources/pooling_batch/events.py rename {golem_core/core/activity_api => golem/resources/pooling_batch}/exceptions.py (87%) rename {golem_core/core/activity_api/resources => golem/resources/pooling_batch}/pooling_batch.py (92%) create mode 100644 golem/resources/proposal/__init__.py create mode 100644 golem/resources/proposal/events.py rename {golem_core/core/market_api => golem/resources/proposal}/pipeline.py (70%) rename {golem_core/core/market_api/resources => golem/resources/proposal}/proposal.py (92%) rename {golem_core/core/props_cons/parsers => golem/utils}/__init__.py (100%) rename {golem_core => golem}/utils/asyncio.py (100%) rename {golem_core => golem}/utils/http.py (100%) rename {golem_core => golem}/utils/logging.py (91%) create mode 100644 golem/utils/storage/__init__.py rename golem_core/utils/storage/common.py => golem/utils/storage/base.py (100%) rename {golem_core/core/props_cons/parsers/textx => golem/utils/storage/gftp}/__init__.py (100%) rename golem_core/utils/storage/gftp.py => golem/utils/storage/gftp/provider.py (99%) rename {golem_core => golem}/utils/storage/utils.py (100%) rename {golem_core => golem}/utils/typing.py (100%) delete mode 100644 golem_core/cli/__init__.py delete mode 100644 golem_core/core/activity_api/__init__.py delete mode 100644 golem_core/core/activity_api/events.py delete mode 100644 golem_core/core/activity_api/resources/__init__.py delete mode 100644 golem_core/core/events/__init__.py delete mode 100644 golem_core/core/exceptions.py delete mode 100644 golem_core/core/market_api/__init__.py delete mode 100644 golem_core/core/market_api/events.py delete mode 100644 golem_core/core/market_api/exceptions.py delete mode 100644 golem_core/core/market_api/resources/__init__.py delete mode 100644 golem_core/core/market_api/resources/demand/__init__.py delete mode 100644 golem_core/core/market_api/resources/demand/demand_offer_base/__init__.py delete mode 100644 golem_core/core/market_api/resources/demand/demand_offer_base/exceptions.py delete mode 100644 golem_core/core/market_api/resources/demand/demand_offer_base/payload/__init__.py delete mode 100644 golem_core/core/market_api/resources/demand/demand_offer_base/payload/base.py delete mode 100644 golem_core/core/network_api/__init__.py delete mode 100644 golem_core/core/network_api/resources/__init__.py delete mode 100644 golem_core/core/payment_api/__init__.py delete mode 100644 golem_core/core/payment_api/events.py delete mode 100644 golem_core/core/payment_api/resources/__init__.py delete mode 100644 golem_core/core/payment_api/resources/event_collectors.py delete mode 100644 golem_core/core/props_cons/parsers/base.py delete mode 100644 golem_core/core/resources/__init__.py delete mode 100644 golem_core/core/resources/event_collectors/__init__.py delete mode 100644 golem_core/exceptions.py delete mode 100644 golem_core/managers/__init__.py delete mode 100644 golem_core/managers/agreement/__init__.py delete mode 100644 golem_core/managers/negotiation/__init__.py delete mode 100644 golem_core/managers/network/__init__.py delete mode 100644 golem_core/managers/payment/__init__.py delete mode 100644 golem_core/managers/proposal/__init__.py delete mode 100644 golem_core/pipeline/__init__.py delete mode 100644 golem_core/utils/__init__.py delete mode 100644 golem_core/utils/storage/__init__.py diff --git a/README.md b/README.md index 3b5e807e..d7930c08 100644 --- a/README.md +++ b/README.md @@ -20,19 +20,19 @@ $ poetry run python examples/rate_providers/rate_providers.py ## CLI ```bash -$ python -m golem_core status +$ python -m golem status -$ python -m golem_core find-node --runtime vm -$ python -m golem_core find-node --runtime vm --subnet public-beta -$ python -m golem_core find-node --runtime vm --timeout 7 # stops after 7 seconds -$ python -m golem_core find-node --runtime vm --timeout 1m # stops after 60 seconds +$ python -m golem find-node --runtime vm +$ python -m golem find-node --runtime vm --subnet public-beta +$ python -m golem find-node --runtime vm --timeout 7 # stops after 7 seconds +$ python -m golem find-node --runtime vm --timeout 1m # stops after 60 seconds -$ python -m golem_core allocation list +$ python -m golem allocation list -$ python -m golem_core allocation new 1 -$ python -m golem_core allocation new 2 --driver erc20 --network goerli +$ python -m golem allocation new 1 +$ python -m golem allocation new 2 --driver erc20 --network goerli -$ python -m golem_core allocation clean +$ python -m golem allocation clean ``` ## Docs diff --git a/examples/attach.py b/examples/attach.py index aeb34a17..a8ef627b 100644 --- a/examples/attach.py +++ b/examples/attach.py @@ -1,10 +1,11 @@ import asyncio import sys -from golem_core.core.activity_api import commands -from golem_core.core.golem_node import GolemNode -from golem_core.core.payment_api import DebitNote, NewDebitNote -from golem_core.core.resources import NewResource +from golem.resources.activity import commands +from golem.resources.golem_node import GolemNode +from golem.resources.payment import DebitNote +from golem.resources.debit_note.events import NewDebitNote +from golem.resources.resources import NewResource ACTIVITY_ID = sys.argv[1].strip() diff --git a/examples/core_example.py b/examples/core_example.py index cba778ca..6677d413 100644 --- a/examples/core_example.py +++ b/examples/core_example.py @@ -4,7 +4,7 @@ from tempfile import TemporaryDirectory from typing import AsyncGenerator, Optional -from golem_core.core.activity_api import ( +from golem.resources.activity import ( Activity, BatchError, CommandCancelled, @@ -12,9 +12,9 @@ Script, commands, ) -from golem_core.core.golem_node import GolemNode -from golem_core.core.market_api import RepositoryVmPayload -from golem_core.core.resources import NewResource, ResourceClosed, ResourceEvent +from golem.resources.golem_node import GolemNode +from golem.resources.market import RepositoryVmPayload +from golem.resources.resources import NewResource, ResourceClosed, ResourceEvent PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/examples/detached_activity.py b/examples/detached_activity.py index 2db5ff75..7906269a 100644 --- a/examples/detached_activity.py +++ b/examples/detached_activity.py @@ -1,9 +1,9 @@ import asyncio -from golem_core.core.activity_api import Activity, commands -from golem_core.core.golem_node import GolemNode -from golem_core.core.market_api import Agreement, Proposal, RepositoryVmPayload, default_negotiate -from golem_core.pipeline import Chain, Map +from golem.resources.activity import Activity, commands +from golem.resources.golem_node import GolemNode +from golem.resources.market import Agreement, Proposal, RepositoryVmPayload, default_negotiate +from golem.pipeline import Chain, Map PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/examples/exception_handling/exception_handling.py b/examples/exception_handling/exception_handling.py index 6faee3a5..0cdad688 100644 --- a/examples/exception_handling/exception_handling.py +++ b/examples/exception_handling/exception_handling.py @@ -1,18 +1,18 @@ import asyncio from typing import Callable, Tuple -from golem_core.core.activity_api import Activity, BatchError, BatchTimeoutError, commands -from golem_core.core.events.base import Event -from golem_core.core.golem_node import GolemNode -from golem_core.core.market_api import ( +from golem.resources.activity import Activity, BatchError, BatchTimeoutError, commands +from golem.resources.events.base import Event +from golem.resources.golem_node import GolemNode +from golem.resources.market import ( RepositoryVmPayload, default_create_activity, default_create_agreement, default_negotiate, ) -from golem_core.managers import DefaultPaymentManager -from golem_core.pipeline import Buffer, Chain, Limit, Map -from golem_core.utils.logging import DefaultLogger +from golem.managers import DefaultPaymentManager +from golem.pipeline import Buffer, Chain, Limit, Map +from golem.utils.logging import DefaultLogger PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index a51dec9e..d76ffbc5 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -3,22 +3,22 @@ from random import randint from typing import List -from golem_core.core.golem_node.golem_node import GolemNode -from golem_core.core.market_api import RepositoryVmPayload -from golem_core.managers.activity.single_use import SingleUseActivityManager -from golem_core.managers.agreement.single_use import SingleUseAgreementManager -from golem_core.managers.base import WorkContext, WorkResult -from golem_core.managers.negotiation import SequentialNegotiationManager -from golem_core.managers.negotiation.plugins import AddChosenPaymentPlatform, BlacklistProviderId -from golem_core.managers.payment.pay_all import PayAllPaymentManager -from golem_core.managers.proposal import StackProposalManager -from golem_core.managers.work.decorators import ( +from golem.resources.golem_node.golem_node import GolemNode +from golem.resources.market import RepositoryVmPayload +from golem.managers.activity.single_use import SingleUseActivityManager +from golem.managers.agreement.single_use import SingleUseAgreementManager +from golem.managers.base import WorkContext, WorkResult +from golem.managers.negotiation import SequentialNegotiationManager +from golem.managers.negotiation.plugins import AddChosenPaymentPlatform, BlacklistProviderId +from golem.managers.payment.pay_all import PayAllPaymentManager +from golem.managers.proposal import StackProposalManager +from golem.managers.work.decorators import ( redundancy_cancel_others_on_first_done, retry, work_decorator, ) -from golem_core.managers.work.sequential import SequentialWorkManager -from golem_core.utils.logging import DEFAULT_LOGGING +from golem.managers.work.sequential import SequentialWorkManager +from golem.utils.logging import DEFAULT_LOGGING async def commands_work_example(context: WorkContext) -> str: diff --git a/examples/managers/ssh.py b/examples/managers/ssh.py index a3f3c1eb..006bcf97 100644 --- a/examples/managers/ssh.py +++ b/examples/managers/ssh.py @@ -4,17 +4,17 @@ import string from uuid import uuid4 -from golem_core.core.golem_node.golem_node import GolemNode -from golem_core.core.market_api import RepositoryVmPayload -from golem_core.managers.activity.single_use import SingleUseActivityManager -from golem_core.managers.agreement.single_use import SingleUseAgreementManager -from golem_core.managers.base import WorkContext, WorkResult -from golem_core.managers.negotiation import SequentialNegotiationManager -from golem_core.managers.network.single import SingleNetworkManager -from golem_core.managers.payment.pay_all import PayAllPaymentManager -from golem_core.managers.proposal import StackProposalManager -from golem_core.managers.work.sequential import SequentialWorkManager -from golem_core.utils.logging import DEFAULT_LOGGING +from golem.resources.golem_node.golem_node import GolemNode +from golem.resources.market import RepositoryVmPayload +from golem.managers.activity.single_use import SingleUseActivityManager +from golem.managers.agreement.single_use import SingleUseAgreementManager +from golem.managers.base import WorkContext, WorkResult +from golem.managers.negotiation import SequentialNegotiationManager +from golem.managers.network.single import SingleNetworkManager +from golem.managers.payment.pay_all import PayAllPaymentManager +from golem.managers.proposal import StackProposalManager +from golem.managers.work.sequential import SequentialWorkManager +from golem.utils.logging import DEFAULT_LOGGING def on_activity_start(get_network_deploy_args): diff --git a/examples/rate_providers/rate_providers.py b/examples/rate_providers/rate_providers.py index d12d8672..c193a87c 100644 --- a/examples/rate_providers/rate_providers.py +++ b/examples/rate_providers/rate_providers.py @@ -4,19 +4,19 @@ from pathlib import Path from typing import Any, AsyncIterator, Callable, Dict, Optional, Tuple -from golem_core.core.activity_api import Activity, commands -from golem_core.core.events.base import Event -from golem_core.core.golem_node import GolemNode -from golem_core.core.market_api import ( +from golem.resources.activity import Activity, commands +from golem.resources.events.base import Event +from golem.resources.golem_node import GolemNode +from golem.resources.market import ( Proposal, RepositoryVmPayload, default_create_activity, default_create_agreement, default_negotiate, ) -from golem_core.managers import DefaultPaymentManager -from golem_core.pipeline import Buffer, Chain, Limit, Map -from golem_core.utils.logging import DefaultLogger +from golem.managers import DefaultPaymentManager +from golem.pipeline import Buffer, Chain, Limit, Map +from golem.utils.logging import DefaultLogger FRAME_CONFIG_TEMPLATE = json.loads(Path(__file__).with_name("frame_params.json").read_text()) PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/examples/score_based_providers/score_based_providers.py b/examples/score_based_providers/score_based_providers.py index fa747323..f1a5cc63 100644 --- a/examples/score_based_providers/score_based_providers.py +++ b/examples/score_based_providers/score_based_providers.py @@ -2,19 +2,19 @@ from datetime import timedelta from typing import Callable, Dict, Optional, Tuple -from golem_core.core.activity_api import Activity, commands -from golem_core.core.events.base import Event -from golem_core.core.golem_node import GolemNode -from golem_core.core.market_api import ( +from golem.resources.activity import Activity, commands +from golem.resources.events.base import Event +from golem.resources.golem_node import GolemNode +from golem.resources.market import ( Proposal, RepositoryVmPayload, default_create_activity, default_create_agreement, default_negotiate, ) -from golem_core.managers import DefaultPaymentManager -from golem_core.pipeline import Buffer, Chain, Limit, Map, Sort -from golem_core.utils.logging import DefaultLogger +from golem.managers import DefaultPaymentManager +from golem.pipeline import Buffer, Chain, Limit, Map, Sort +from golem.utils.logging import DefaultLogger PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/examples/service.py b/examples/service.py index 4591e39f..96a6ce98 100644 --- a/examples/service.py +++ b/examples/service.py @@ -5,18 +5,18 @@ from urllib.parse import urlparse from uuid import uuid4 -from golem_core.core.activity_api import Activity, commands -from golem_core.core.events.base import Event -from golem_core.core.golem_node import GolemNode -from golem_core.core.market_api import ( +from golem.resources.activity import Activity, commands +from golem.resources.events.base import Event +from golem.resources.golem_node import GolemNode +from golem.resources.market import ( RepositoryVmPayload, default_create_activity, default_create_agreement, default_negotiate, ) -from golem_core.core.network_api import Network -from golem_core.pipeline import Buffer, Chain, Limit, Map -from golem_core.utils.logging import DefaultLogger +from golem.resources.network import Network +from golem.pipeline import Buffer, Chain, Limit, Map +from golem.utils.logging import DefaultLogger PAYLOAD = RepositoryVmPayload( "1e06505997e8bd1b9e1a00bd10d255fc6a390905e4d6840a22a79902", diff --git a/examples/task_api_draft/examples/blender/blender.py b/examples/task_api_draft/examples/blender/blender.py index 1c141150..eef1e5b3 100644 --- a/examples/task_api_draft/examples/blender/blender.py +++ b/examples/task_api_draft/examples/blender/blender.py @@ -3,8 +3,8 @@ from pathlib import Path from examples.task_api_draft.task_api.execute_tasks import execute_tasks -from golem_core.core.activity_api import Activity, commands -from golem_core.core.market_api import RepositoryVmPayload +from golem.resources.activity import Activity, commands +from golem.resources.market import RepositoryVmPayload FRAME_CONFIG_TEMPLATE = json.loads(Path(__file__).with_name("frame_params.json").read_text()) FRAMES = list(range(0, 60, 10)) diff --git a/examples/task_api_draft/examples/execute_tasks_hello_world.py b/examples/task_api_draft/examples/execute_tasks_hello_world.py index 19a36261..27659367 100644 --- a/examples/task_api_draft/examples/execute_tasks_hello_world.py +++ b/examples/task_api_draft/examples/execute_tasks_hello_world.py @@ -1,8 +1,8 @@ import asyncio from examples.task_api_draft.task_api.execute_tasks import execute_tasks -from golem_core.core.activity_api import Activity, commands -from golem_core.core.market_api import RepositoryVmPayload +from golem.resources.activity import Activity, commands +from golem.resources.market import RepositoryVmPayload TASK_DATA = list(range(7)) BUDGET = 1 diff --git a/examples/task_api_draft/examples/pipeline_example.py b/examples/task_api_draft/examples/pipeline_example.py index 95c83790..3450c54d 100644 --- a/examples/task_api_draft/examples/pipeline_example.py +++ b/examples/task_api_draft/examples/pipeline_example.py @@ -4,19 +4,19 @@ from typing import AsyncIterator, Callable, Tuple from examples.task_api_draft.task_api.activity_pool import ActivityPool -from golem_core.core.activity_api import Activity, commands -from golem_core.core.events.base import Event -from golem_core.core.golem_node import GolemNode -from golem_core.core.market_api import ( +from golem.resources.activity import Activity, commands +from golem.resources.events.base import Event +from golem.resources.golem_node import GolemNode +from golem.resources.market import ( Proposal, RepositoryVmPayload, default_create_activity, default_create_agreement, default_negotiate, ) -from golem_core.managers import DefaultPaymentManager -from golem_core.pipeline import Buffer, Chain, Map, Sort, Zip -from golem_core.utils.logging import DefaultLogger +from golem.managers import DefaultPaymentManager +from golem.pipeline import Buffer, Chain, Map, Sort, Zip +from golem.utils.logging import DefaultLogger PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/examples/task_api_draft/examples/redundance.py b/examples/task_api_draft/examples/redundance.py index 4e342f03..4feae5b4 100644 --- a/examples/task_api_draft/examples/redundance.py +++ b/examples/task_api_draft/examples/redundance.py @@ -2,8 +2,8 @@ import random from examples.task_api_draft.task_api.execute_tasks import execute_tasks -from golem_core.core.activity_api import Activity, commands -from golem_core.core.market_api import RepositoryVmPayload +from golem.resources.activity import Activity, commands +from golem.resources.market import RepositoryVmPayload PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/examples/task_api_draft/examples/yacat.py b/examples/task_api_draft/examples/yacat.py index 5b305c42..cf53dc78 100644 --- a/examples/task_api_draft/examples/yacat.py +++ b/examples/task_api_draft/examples/yacat.py @@ -23,21 +23,21 @@ tasks_queue, ) from examples.task_api_draft.task_api.activity_pool import ActivityPool -from golem_core.core.activity_api import Activity, PoolingBatch, default_prepare_activity -from golem_core.core.events import Event -from golem_core.core.golem_node import GolemNode -from golem_core.core.market_api import ( +from golem.resources.activity import Activity, PoolingBatch, default_prepare_activity +from golem.event_bus import Event +from golem.resources.golem_node import GolemNode +from golem.resources.market import ( Proposal, default_create_activity, default_create_agreement, default_negotiate, ) -from golem_core.core.payment_api import DebitNote -from golem_core.core.payment_api.events import NewDebitNote -from golem_core.core.resources import NewResource, ResourceClosed -from golem_core.managers import DefaultPaymentManager -from golem_core.pipeline import Buffer, Chain, Map, Sort, Zip -from golem_core.utils.logging import DefaultLogger +from golem.resources.payment import DebitNote +from golem.resources.debit_note.events import NewDebitNote +from golem.resources.resources import NewResource, ResourceClosed +from golem.managers import DefaultPaymentManager +from golem.pipeline import Buffer, Chain, Map, Sort, Zip +from golem.utils.logging import DefaultLogger ########################### # APP LOGIC CONFIG diff --git a/examples/task_api_draft/examples/yacat_no_business_logic.py b/examples/task_api_draft/examples/yacat_no_business_logic.py index 423baf42..139d2060 100644 --- a/examples/task_api_draft/examples/yacat_no_business_logic.py +++ b/examples/task_api_draft/examples/yacat_no_business_logic.py @@ -5,8 +5,8 @@ from typing import List, Union from examples.task_api_draft.task_api.execute_tasks import execute_tasks -from golem_core.core.activity_api import Activity, Run -from golem_core.core.market_api import RepositoryVmPayload +from golem.resources.activity import Activity, Run +from golem.resources.market import RepositoryVmPayload PAYLOAD = RepositoryVmPayload("055911c811e56da4d75ffc928361a78ed13077933ffa8320fb1ec2db") PASSWORD_LENGTH = 3 diff --git a/examples/task_api_draft/task_api/activity_pool.py b/examples/task_api_draft/task_api/activity_pool.py index 8530eb29..4ed11e9c 100644 --- a/examples/task_api_draft/task_api/activity_pool.py +++ b/examples/task_api_draft/task_api/activity_pool.py @@ -2,8 +2,8 @@ import inspect from typing import AsyncIterator, Awaitable, List, Union -from golem_core.core.activity_api import Activity -from golem_core.pipeline.exceptions import InputStreamExhausted +from golem.resources.activity import Activity +from golem.pipeline.exceptions import InputStreamExhausted class ActivityPool: @@ -11,10 +11,10 @@ class ActivityPool: Sample usage:: - from golem_core import GolemNode + from golem import GolemNode from examples - from golem_core.pipeline import Chain, Buffer, Map - from golem_core.commands import Run + from golem.pipeline import Chain, Buffer, Map + from golem.commands import Run async def say_hi(activity): batch = await activity.execute_commands(Run(f"echo -n 'Hi, this is {activity}'")) diff --git a/examples/task_api_draft/task_api/execute_tasks.py b/examples/task_api_draft/task_api/execute_tasks.py index 070a8987..e588a74b 100644 --- a/examples/task_api_draft/task_api/execute_tasks.py +++ b/examples/task_api_draft/task_api/execute_tasks.py @@ -2,10 +2,10 @@ from random import random from typing import AsyncIterator, Awaitable, Callable, Iterable, Optional, Tuple, TypeVar -from golem_core.core.activity_api import Activity, default_prepare_activity -from golem_core.core.events.base import Event -from golem_core.core.golem_node import GolemNode -from golem_core.core.market_api import ( +from golem.resources.activity import Activity, default_prepare_activity +from golem.resources.events.base import Event +from golem.resources.golem_node import GolemNode +from golem.resources.market import ( Demand, Payload, Proposal, @@ -13,9 +13,9 @@ default_create_agreement, default_negotiate, ) -from golem_core.managers import DefaultPaymentManager -from golem_core.pipeline import Buffer, Chain, Map, Sort, Zip -from golem_core.utils.logging import DefaultLogger +from golem.managers import DefaultPaymentManager +from golem.pipeline import Buffer, Chain, Map, Sort, Zip +from golem.utils.logging import DefaultLogger from .activity_pool import ActivityPool from .redundance_manager import RedundanceManager diff --git a/examples/task_api_draft/task_api/redundance_manager.py b/examples/task_api_draft/task_api/redundance_manager.py index 68cc666a..13c11492 100644 --- a/examples/task_api_draft/task_api/redundance_manager.py +++ b/examples/task_api_draft/task_api/redundance_manager.py @@ -12,8 +12,8 @@ TypeVar, ) -from golem_core.core.activity_api import Activity -from golem_core.core.market_api import Proposal +from golem.resources.activity import Activity +from golem.resources.market import Proposal from .task_data_stream import TaskDataStream diff --git a/golem_core/__init__.py b/golem/__init__.py similarity index 100% rename from golem_core/__init__.py rename to golem/__init__.py diff --git a/golem_core/__main__.py b/golem/__main__.py similarity index 55% rename from golem_core/__main__.py rename to golem/__main__.py index 3644605e..cd54e5f3 100644 --- a/golem_core/__main__.py +++ b/golem/__main__.py @@ -1,4 +1,4 @@ -from golem_core.cli import cli +from golem.cli import cli if __name__ == "__main__": cli() diff --git a/golem/cli/__init__.py b/golem/cli/__init__.py new file mode 100644 index 00000000..870c2f74 --- /dev/null +++ b/golem/cli/__init__.py @@ -0,0 +1,3 @@ +from golem.cli.cli import cli + +__all__ = ("cli",) diff --git a/golem_core/cli/cli.py b/golem/cli/cli.py similarity index 92% rename from golem_core/cli/cli.py rename to golem/cli/cli.py index 4556c156..a93766dc 100644 --- a/golem_core/cli/cli.py +++ b/golem/cli/cli.py @@ -3,7 +3,7 @@ import click -from golem_core.cli.utils import ( +from golem.cli.utils import ( CliPayload, async_golem_wrapper, format_allocations, @@ -11,7 +11,7 @@ format_proposals, parse_timedelta_str, ) -from golem_core.core.golem_node import PAYMENT_DRIVER, PAYMENT_NETWORK, SUBNET, GolemNode +from golem.node import PAYMENT_DRIVER, PAYMENT_NETWORK, SUBNET, GolemNode @click.group() @@ -33,7 +33,7 @@ async def allocation_list(golem: GolemNode) -> None: @allocation.command("new") @click.argument("amount", type=float) -@click.option("--network_api", type=str, default=PAYMENT_NETWORK) +@click.option("--network", type=str, default=PAYMENT_NETWORK) @click.option("--driver", type=str, default=PAYMENT_DRIVER) @async_golem_wrapper async def allocation_new( diff --git a/golem_core/cli/utils.py b/golem/cli/utils.py similarity index 95% rename from golem_core/cli/utils.py rename to golem/cli/utils.py index e7fa2142..99ae7efe 100644 --- a/golem_core/cli/utils.py +++ b/golem/cli/utils.py @@ -8,9 +8,9 @@ from prettytable import PrettyTable from typing_extensions import Concatenate, ParamSpec -from golem_core.core.golem_node import GolemNode -from golem_core.core.market_api import RUNTIME_NAME, Demand, Payload, Proposal, constraint -from golem_core.core.payment_api import Allocation +from golem.resources.golem_node import GolemNode +from golem.resources.market import RUNTIME_NAME, Demand, Payload, Proposal, constraint +from golem.resources.payment import Allocation def format_allocations(allocations: List[Allocation]) -> str: diff --git a/golem/event_bus/__init__.py b/golem/event_bus/__init__.py new file mode 100644 index 00000000..c281f026 --- /dev/null +++ b/golem/event_bus/__init__.py @@ -0,0 +1,7 @@ +from golem.event_bus.base import EventBus, TEvent, Event + +__all__ = ( + "EventBus", + "TEvent", + "Event", +) diff --git a/golem_core/core/events/base.py b/golem/event_bus/base.py similarity index 87% rename from golem_core/core/events/base.py rename to golem/event_bus/base.py index d1db7dd1..576b0dc7 100644 --- a/golem_core/core/events/base.py +++ b/golem/event_bus/base.py @@ -1,19 +1,17 @@ from abc import ABC, abstractmethod -from typing import Awaitable, Callable, Optional, Type, TypeVar +from typing import TypeVar, Type, Callable, Awaitable, Optional -from golem_core.core.exceptions import BaseCoreException +from golem.exceptions import GolemException -TEvent = TypeVar("TEvent", bound="Event") +TCallbackHandler = TypeVar("TCallbackHandler") +TEvent = TypeVar("TEvent", bound="Event") class Event(ABC): """Base class for all events.""" -TCallbackHandler = TypeVar("TCallbackHandler") - - -class EventBusError(BaseCoreException): +class EventBusError(GolemException): pass diff --git a/golem/event_bus/in_memory/__init__.py b/golem/event_bus/in_memory/__init__.py new file mode 100644 index 00000000..bb27dbe1 --- /dev/null +++ b/golem/event_bus/in_memory/__init__.py @@ -0,0 +1,5 @@ +from golem.event_bus.in_memory.event_bus import InMemoryEventBus + +__all__ = ( + 'InMemoryEventBus', +) \ No newline at end of file diff --git a/golem_core/core/events/event_bus.py b/golem/event_bus/in_memory/event_bus.py similarity index 96% rename from golem_core/core/events/event_bus.py rename to golem/event_bus/in_memory/event_bus.py index 87689cb1..adda58f0 100644 --- a/golem_core/core/events/event_bus.py +++ b/golem/event_bus/in_memory/event_bus.py @@ -4,10 +4,10 @@ from dataclasses import dataclass from typing import Awaitable, Callable, DefaultDict, List, Optional, Type -from golem_core.core.events.base import Event -from golem_core.core.events.base import EventBus as BaseEventBus -from golem_core.core.events.base import EventBusError, TCallbackHandler, TEvent -from golem_core.utils.asyncio import create_task_with_logging +from golem.resources.events.base import Event +from golem.resources.events.base import EventBus as BaseEventBus +from golem.resources.events.base import EventBusError, TCallbackHandler, TEvent +from golem.utils.asyncio import create_task_with_logging logger = logging.getLogger(__name__) diff --git a/golem/exceptions.py b/golem/exceptions.py new file mode 100644 index 00000000..6c67d3c3 --- /dev/null +++ b/golem/exceptions.py @@ -0,0 +1,2 @@ +class GolemException(Exception): + pass diff --git a/golem/managers/__init__.py b/golem/managers/__init__.py new file mode 100644 index 00000000..aadc1973 --- /dev/null +++ b/golem/managers/__init__.py @@ -0,0 +1,3 @@ +from golem.managers.payment import DefaultPaymentManager + +__all__ = ("DefaultPaymentManager",) diff --git a/golem_core/managers/activity/__init__.py b/golem/managers/activity/__init__.py similarity index 57% rename from golem_core/managers/activity/__init__.py rename to golem/managers/activity/__init__.py index 0cfc216a..b203330f 100644 --- a/golem_core/managers/activity/__init__.py +++ b/golem/managers/activity/__init__.py @@ -1,8 +1,8 @@ -from golem_core.managers.activity.defaults import ( +from golem.managers.activity.defaults import ( default_on_activity_start, default_on_activity_stop, ) -from golem_core.managers.activity.single_use import SingleUseActivityManager +from golem.managers.activity.single_use import SingleUseActivityManager __all__ = ( "SingleUseActivityManager", diff --git a/golem_core/managers/activity/defaults.py b/golem/managers/activity/defaults.py similarity index 89% rename from golem_core/managers/activity/defaults.py rename to golem/managers/activity/defaults.py index 949543ad..41b7111c 100644 --- a/golem_core/managers/activity/defaults.py +++ b/golem/managers/activity/defaults.py @@ -1,4 +1,4 @@ -from golem_core.managers.base import WorkContext +from golem.managers.base import WorkContext async def default_on_activity_start(context: WorkContext): diff --git a/golem_core/managers/activity/single_use.py b/golem/managers/activity/single_use.py similarity index 91% rename from golem_core/managers/activity/single_use.py rename to golem/managers/activity/single_use.py index 668a3fd7..0f5e62ee 100644 --- a/golem_core/managers/activity/single_use.py +++ b/golem/managers/activity/single_use.py @@ -2,15 +2,15 @@ from contextlib import asynccontextmanager from typing import Awaitable, Callable, Optional -from golem_core.core.activity_api import Activity -from golem_core.core.golem_node.golem_node import GolemNode -from golem_core.core.market_api import Agreement -from golem_core.managers.activity.defaults import ( +from golem.resources.activity import Activity +from golem.resources.golem_node.golem_node import GolemNode +from golem.resources.market import Agreement +from golem.managers.activity.defaults import ( default_on_activity_start, default_on_activity_stop, ) -from golem_core.managers.agreement import AgreementReleased -from golem_core.managers.base import ActivityManager, Work, WorkContext, WorkResult +from golem.managers.agreement import AgreementReleased +from golem.managers.base import ActivityManager, Work, WorkContext, WorkResult logger = logging.getLogger(__name__) diff --git a/golem/managers/agreement/__init__.py b/golem/managers/agreement/__init__.py new file mode 100644 index 00000000..2ba5ea00 --- /dev/null +++ b/golem/managers/agreement/__init__.py @@ -0,0 +1,7 @@ +from golem.managers.agreement.events import AgreementReleased +from golem.managers.agreement.single_use import SingleUseAgreementManager + +__all__ = ( + "AgreementReleased", + "SingleUseAgreementManager", +) diff --git a/golem_core/managers/agreement/events.py b/golem/managers/agreement/events.py similarity index 50% rename from golem_core/managers/agreement/events.py rename to golem/managers/agreement/events.py index 5a14728d..5ad374c4 100644 --- a/golem_core/managers/agreement/events.py +++ b/golem/managers/agreement/events.py @@ -1,4 +1,4 @@ -from golem_core.managers.base import ManagerEvent +from golem.managers.base import ManagerEvent class AgreementReleased(ManagerEvent): diff --git a/golem_core/managers/agreement/single_use.py b/golem/managers/agreement/single_use.py similarity index 89% rename from golem_core/managers/agreement/single_use.py rename to golem/managers/agreement/single_use.py index f92007bb..3a2f6dda 100644 --- a/golem_core/managers/agreement/single_use.py +++ b/golem/managers/agreement/single_use.py @@ -1,10 +1,10 @@ import logging from typing import Awaitable, Callable -from golem_core.core.golem_node.golem_node import GolemNode -from golem_core.core.market_api import Agreement, Proposal -from golem_core.managers.agreement.events import AgreementReleased -from golem_core.managers.base import AgreementManager +from golem.resources.golem_node.golem_node import GolemNode +from golem.resources.market import Agreement, Proposal +from golem.managers.agreement.events import AgreementReleased +from golem.managers.base import AgreementManager logger = logging.getLogger(__name__) diff --git a/golem_core/managers/base.py b/golem/managers/base.py similarity index 87% rename from golem_core/managers/base.py rename to golem/managers/base.py index c67f3b71..c1a285bf 100644 --- a/golem_core/managers/base.py +++ b/golem/managers/base.py @@ -2,13 +2,13 @@ from dataclasses import dataclass, field from typing import Any, Awaitable, Callable, Dict, List, Optional, Union -from golem_core.core.activity_api import Activity, Script, commands -from golem_core.core.market_api import Agreement, Proposal -from golem_core.core.market_api.resources.demand.demand import DemandData -from golem_core.core.market_api.resources.proposal import ProposalData -from golem_core.core.payment_api import Allocation -from golem_core.core.resources import ResourceEvent -from golem_core.exceptions import BaseGolemException +from golem.resources.activity import Activity, Script, commands +from golem.resources.market import Agreement, Proposal +from golem.resources.demand.demand import DemandData +from golem.resources.market.proposal import ProposalData +from golem.resources.payment import Allocation +from golem.resources.resources import ResourceEvent +from golem.exceptions import GolemException class Batch: @@ -90,7 +90,7 @@ class ManagerEvent(ResourceEvent, ABC): pass -class ManagerException(BaseGolemException): +class ManagerException(GolemException): pass diff --git a/golem/managers/negotiation/__init__.py b/golem/managers/negotiation/__init__.py new file mode 100644 index 00000000..d45d6578 --- /dev/null +++ b/golem/managers/negotiation/__init__.py @@ -0,0 +1,3 @@ +from golem.managers.negotiation.sequential import SequentialNegotiationManager + +__all__ = ("SequentialNegotiationManager",) diff --git a/golem_core/managers/negotiation/plugins.py b/golem/managers/negotiation/plugins.py similarity index 87% rename from golem_core/managers/negotiation/plugins.py rename to golem/managers/negotiation/plugins.py index 02b6537f..d2d8e94a 100644 --- a/golem_core/managers/negotiation/plugins.py +++ b/golem/managers/negotiation/plugins.py @@ -1,11 +1,11 @@ import logging from typing import Sequence, Set -from golem_core.core.market_api.resources.demand.demand import DemandData -from golem_core.core.market_api.resources.proposal import ProposalData -from golem_core.core.props_cons.properties import Properties -from golem_core.managers.base import NegotiationPlugin -from golem_core.managers.negotiation.sequential import RejectProposal +from golem.resources.demand.demand import DemandData +from golem.resources.market.proposal import ProposalData +from golem.payload.properties import Properties +from golem.managers.base import NegotiationPlugin +from golem.managers.negotiation.sequential import RejectProposal logger = logging.getLogger(__name__) diff --git a/golem_core/managers/negotiation/sequential.py b/golem/managers/negotiation/sequential.py similarity index 90% rename from golem_core/managers/negotiation/sequential.py rename to golem/managers/negotiation/sequential.py index ecf9faff..2e4b8a17 100644 --- a/golem_core/managers/negotiation/sequential.py +++ b/golem/managers/negotiation/sequential.py @@ -6,15 +6,15 @@ from ya_market import ApiException -from golem_core.core.golem_node.golem_node import GolemNode -from golem_core.core.market_api import Demand, DemandBuilder, Payload, Proposal -from golem_core.core.market_api.resources.demand.demand import DemandData -from golem_core.core.market_api.resources.proposal import ProposalData -from golem_core.core.payment_api import Allocation -from golem_core.core.props_cons.parsers.textx.parser import TextXDemandOfferSyntaxParser -from golem_core.core.props_cons.properties import Properties -from golem_core.managers.base import ManagerException, NegotiationManager, NegotiationPlugin -from golem_core.utils.asyncio import create_task_with_logging +from golem.resources.golem_node.golem_node import GolemNode +from golem.resources.market import Demand, DemandBuilder, Payload, Proposal +from golem.resources.demand.demand import DemandData +from golem.resources.market.proposal import ProposalData +from golem.resources.payment import Allocation +from golem.payload.parsers import TextXDemandOfferSyntaxParser +from golem.payload.properties import Properties +from golem.managers.base import ManagerException, NegotiationManager, NegotiationPlugin +from golem.utils.asyncio import create_task_with_logging logger = logging.getLogger(__name__) @@ -198,7 +198,7 @@ async def _get_demand_data_from_demand(self, demand: Demand) -> DemandData: # FIXME: Unnecessary serialisation from DemandBuilder to Demand, and from Demand to ProposalData data = await demand.get_data() - constraints = self._demand_offer_parser.parse(data.constraints) + constraints = self._demand_offer_parser.parse_constraints(data.constraints) return DemandData( properties=Properties(data.properties), @@ -211,7 +211,7 @@ async def _get_demand_data_from_demand(self, demand: Demand) -> DemandData: async def _get_proposal_data_from_proposal(self, proposal: Proposal) -> ProposalData: data = await proposal.get_data() - constraints = self._demand_offer_parser.parse(data.constraints) + constraints = self._demand_offer_parser.parse_constraints(data.constraints) return ProposalData( properties=Properties(data.properties), diff --git a/golem_core/core/__init__.py b/golem/managers/network/__init__.py similarity index 100% rename from golem_core/core/__init__.py rename to golem/managers/network/__init__.py diff --git a/golem_core/managers/network/single.py b/golem/managers/network/single.py similarity index 85% rename from golem_core/managers/network/single.py rename to golem/managers/network/single.py index a7d44dd0..ae78794b 100644 --- a/golem_core/managers/network/single.py +++ b/golem/managers/network/single.py @@ -3,11 +3,11 @@ from typing import Dict from urllib.parse import urlparse -from golem_core.core.golem_node.golem_node import GolemNode -from golem_core.core.market_api.events import NewAgreement -from golem_core.core.network_api import Network -from golem_core.core.network_api.resources.network import DeployArgsType -from golem_core.managers.base import NetworkManager +from golem.resources.golem_node.golem_node import GolemNode +from golem.resources.agreement.events import NewAgreement +from golem.resources.network import Network +from golem.resources.network.network import DeployArgsType +from golem.managers.base import NetworkManager logger = logging.getLogger(__name__) diff --git a/golem/managers/payment/__init__.py b/golem/managers/payment/__init__.py new file mode 100644 index 00000000..3a592759 --- /dev/null +++ b/golem/managers/payment/__init__.py @@ -0,0 +1,3 @@ +from golem.managers.payment.default import DefaultPaymentManager + +__all__ = ("DefaultPaymentManager",) diff --git a/golem_core/managers/payment/default.py b/golem/managers/payment/default.py similarity index 91% rename from golem_core/managers/payment/default.py rename to golem/managers/payment/default.py index 5d11e636..c3717146 100644 --- a/golem_core/managers/payment/default.py +++ b/golem/managers/payment/default.py @@ -2,13 +2,13 @@ from datetime import datetime, timedelta from typing import TYPE_CHECKING, Set -from golem_core.core.payment_api import NewDebitNote -from golem_core.core.payment_api.events import NewInvoice +from golem.resources.debit_note.events import NewDebitNote +from golem.resources.payment.events import NewInvoice if TYPE_CHECKING: - from golem_core.core.golem_node import GolemNode - from golem_core.core.market_api import NewAgreement - from golem_core.core.payment_api import Allocation + from golem.resources.golem_node import GolemNode + from golem.resources.agreement.events import NewAgreement + from golem.resources.payment import Allocation class DefaultPaymentManager: @@ -41,7 +41,7 @@ def __init__(self, node: "GolemNode", allocation: "Allocation"): """ # FIXME: Resolve local import due to cyclic imports - from golem_core.core.market_api import Agreement + from golem.resources.market import Agreement self._node = node self.allocation = allocation diff --git a/golem_core/managers/payment/pay_all.py b/golem/managers/payment/pay_all.py similarity index 89% rename from golem_core/managers/payment/pay_all.py rename to golem/managers/payment/pay_all.py index 0effb6d0..d37aeb14 100644 --- a/golem_core/managers/payment/pay_all.py +++ b/golem/managers/payment/pay_all.py @@ -3,13 +3,14 @@ from decimal import Decimal from typing import Optional -from golem_core.core.golem_node.golem_node import PAYMENT_DRIVER, PAYMENT_NETWORK, GolemNode -from golem_core.core.market_api import AgreementClosed, NewAgreement -from golem_core.core.payment_api.events import NewDebitNote, NewInvoice -from golem_core.core.payment_api.resources.allocation import Allocation -from golem_core.core.payment_api.resources.debit_note import DebitNote -from golem_core.core.payment_api.resources.invoice import Invoice -from golem_core.managers.base import PaymentManager +from golem.resources.golem_node.golem_node import PAYMENT_DRIVER, PAYMENT_NETWORK, GolemNode +from golem.resources.agreement.events import NewAgreement, AgreementClosed +from golem.resources.payment.events import NewInvoice +from golem.resources.debit_note.events import NewDebitNote +from golem.resources.allocation.allocation import Allocation +from golem.resources.debit_note.debit_note import DebitNote +from golem.resources.payment.resources.invoice import Invoice +from golem.managers.base import PaymentManager logger = logging.getLogger(__name__) diff --git a/golem/managers/proposal/__init__.py b/golem/managers/proposal/__init__.py new file mode 100644 index 00000000..b42aaf11 --- /dev/null +++ b/golem/managers/proposal/__init__.py @@ -0,0 +1,3 @@ +from golem.managers.proposal.stack import StackProposalManager + +__all__ = ("StackProposalManager",) diff --git a/golem_core/managers/proposal/stack.py b/golem/managers/proposal/stack.py similarity index 88% rename from golem_core/managers/proposal/stack.py rename to golem/managers/proposal/stack.py index 478fa708..9659e863 100644 --- a/golem_core/managers/proposal/stack.py +++ b/golem/managers/proposal/stack.py @@ -2,10 +2,10 @@ import logging from typing import Optional -from golem_core.core.golem_node.golem_node import GolemNode -from golem_core.core.market_api import Proposal -from golem_core.managers.base import ManagerException, ProposalManager -from golem_core.utils.asyncio import create_task_with_logging +from golem.resources.golem_node.golem_node import GolemNode +from golem.resources.market import Proposal +from golem.managers.base import ManagerException, ProposalManager +from golem.utils.asyncio import create_task_with_logging logger = logging.getLogger(__name__) diff --git a/golem_core/managers/work/__init__.py b/golem/managers/work/__init__.py similarity index 62% rename from golem_core/managers/work/__init__.py rename to golem/managers/work/__init__.py index 8796bacb..cbbd09a2 100644 --- a/golem_core/managers/work/__init__.py +++ b/golem/managers/work/__init__.py @@ -1,9 +1,9 @@ -from golem_core.managers.work.decorators import ( +from golem.managers.work.decorators import ( redundancy_cancel_others_on_first_done, retry, work_decorator, ) -from golem_core.managers.work.sequential import SequentialWorkManager +from golem.managers.work.sequential import SequentialWorkManager __all__ = ( "SequentialWorkManager", diff --git a/golem_core/managers/work/decorators.py b/golem/managers/work/decorators.py similarity index 95% rename from golem_core/managers/work/decorators.py rename to golem/managers/work/decorators.py index fe8db5fe..402d7a1b 100644 --- a/golem_core/managers/work/decorators.py +++ b/golem/managers/work/decorators.py @@ -1,7 +1,7 @@ import asyncio from functools import wraps -from golem_core.managers.base import DoWorkCallable, Work, WorkDecorator, WorkResult +from golem.managers.base import DoWorkCallable, Work, WorkDecorator, WorkResult def work_decorator(decorator: WorkDecorator): diff --git a/golem_core/managers/work/sequential.py b/golem/managers/work/sequential.py similarity index 91% rename from golem_core/managers/work/sequential.py rename to golem/managers/work/sequential.py index f28dd635..411df8a1 100644 --- a/golem_core/managers/work/sequential.py +++ b/golem/managers/work/sequential.py @@ -1,8 +1,8 @@ import logging from typing import List -from golem_core.core.golem_node.golem_node import GolemNode -from golem_core.managers.base import DoWorkCallable, Work, WorkManager, WorkResult +from golem.resources.golem_node.golem_node import GolemNode +from golem.managers.base import DoWorkCallable, Work, WorkManager, WorkResult logger = logging.getLogger(__name__) diff --git a/golem_core/core/golem_node/__init__.py b/golem/node/__init__.py similarity index 64% rename from golem_core/core/golem_node/__init__.py rename to golem/node/__init__.py index f36312c7..a5cf7d46 100644 --- a/golem_core/core/golem_node/__init__.py +++ b/golem/node/__init__.py @@ -1,10 +1,11 @@ -from golem_core.core.golem_node.events import ( +from golem.node.events import ( GolemNodeEvent, SessionStarted, ShutdownFinished, ShutdownStarted, ) -from golem_core.core.golem_node.golem_node import PAYMENT_DRIVER, PAYMENT_NETWORK, SUBNET, GolemNode + +from golem.node.node import GolemNode, PAYMENT_NETWORK, PAYMENT_DRIVER, SUBNET __all__ = ( "GolemNode", diff --git a/golem_core/core/resources/low.py b/golem/node/api.py similarity index 97% rename from golem_core/core/resources/low.py rename to golem/node/api.py index 0467d185..f77e4bdd 100644 --- a/golem_core/core/resources/low.py +++ b/golem/node/api.py @@ -8,11 +8,11 @@ import ya_net import ya_payment -from golem_core.core.resources.exceptions import MissingConfiguration +from golem.resources.resources.exceptions import MissingConfiguration if TYPE_CHECKING: - from golem_core.core.golem_node import GolemNode - from golem_core.core.resources import Resource + from golem.resources.golem_node import GolemNode + from golem.resources.resources import Resource TRequestorApi = TypeVar("TRequestorApi") diff --git a/golem/node/event_collectors/__init__.py b/golem/node/event_collectors/__init__.py new file mode 100644 index 00000000..581d3159 --- /dev/null +++ b/golem/node/event_collectors/__init__.py @@ -0,0 +1,8 @@ +from golem.node.event_collectors.base import YagnaEventCollector +from golem.node.event_collectors.utils import is_intermittent_error, is_gsb_endpoint_not_found_error + +__all__ = ( + "YagnaEventCollector", + "is_gsb_endpoint_not_found_error", + "is_intermittent_error", +) diff --git a/golem_core/core/resources/event_collectors/base.py b/golem/node/event_collectors/base.py similarity index 97% rename from golem_core/core/resources/event_collectors/base.py rename to golem/node/event_collectors/base.py index 0ef8e609..bca4bd7c 100644 --- a/golem_core/core/resources/event_collectors/base.py +++ b/golem/node/event_collectors/base.py @@ -4,7 +4,7 @@ # TODO: replace Any here from typing import Any, Callable, Dict, List, Optional -from golem_core.core.resources.event_collectors.utils import ( +from golem.node.event_collectors.utils import ( is_gsb_endpoint_not_found_error, is_intermittent_error, ) diff --git a/golem_core/core/resources/event_collectors/utils.py b/golem/node/event_collectors/utils.py similarity index 96% rename from golem_core/core/resources/event_collectors/utils.py rename to golem/node/event_collectors/utils.py index ffd6ccba..1e836af4 100644 --- a/golem_core/core/resources/event_collectors/utils.py +++ b/golem/node/event_collectors/utils.py @@ -9,7 +9,7 @@ def is_intermittent_error(e: Exception) -> bool: - """Check if `e` indicates an intermittent communication failure such as network_api timeout.""" + """Check if `e` indicates an intermittent communication failure such as network timeout.""" is_timeout_exception = isinstance(e, asyncio.TimeoutError) or ( isinstance( diff --git a/golem_core/core/golem_node/events.py b/golem/node/events.py similarity index 89% rename from golem_core/core/golem_node/events.py rename to golem/node/events.py index fac8ea94..4121f118 100644 --- a/golem_core/core/golem_node/events.py +++ b/golem/node/events.py @@ -1,10 +1,10 @@ from abc import ABC from typing import TYPE_CHECKING -from golem_core.core.events import Event +from golem.event_bus import Event if TYPE_CHECKING: - from golem_core.core.golem_node import GolemNode + from golem.resources.golem_node import GolemNode class GolemNodeEvent(Event, ABC): diff --git a/golem_core/core/golem_node/golem_node.py b/golem/node/node.py similarity index 96% rename from golem_core/core/golem_node/golem_node.py rename to golem/node/node.py index 72d5ce2a..8e2c6960 100644 --- a/golem_core/core/golem_node/golem_node.py +++ b/golem/node/node.py @@ -6,21 +6,21 @@ from typing import Any, DefaultDict, Dict, Iterable, List, Optional, Set, Type, Union from uuid import uuid4 -from golem_core.core.activity_api import Activity, PoolingBatch -from golem_core.core.events import EventBus, InMemoryEventBus -from golem_core.core.golem_node.events import SessionStarted, ShutdownFinished, ShutdownStarted -from golem_core.core.market_api import Agreement, Demand, DemandBuilder, Payload, Proposal -from golem_core.core.market_api.resources.demand.demand_offer_base import defaults as dobm_defaults -from golem_core.core.network_api import Network -from golem_core.core.payment_api import ( +from golem.resources.activity import Activity, PoolingBatch +from golem.event_bus import EventBus, InMemoryEventBus +from golem.node.events import SessionStarted, ShutdownFinished, ShutdownStarted +from golem.resources.market import Agreement, Demand, DemandBuilder, Payload, Proposal +from golem.resources.market.resources.demand.demand_offer_base import defaults as dobm_defaults +from golem.resources.network import Network +from golem.resources.payment import ( Allocation, DebitNote, DebitNoteEventCollector, Invoice, InvoiceEventCollector, ) -from golem_core.core.props_cons.parsers.textx.parser import TextXDemandOfferSyntaxParser -from golem_core.core.resources import ApiConfig, ApiFactory, Resource, TResource +from golem.payload.parsers import TextXDemandOfferSyntaxParser +from golem.resources.resources import ApiConfig, ApiFactory, Resource, TResource PAYMENT_DRIVER: str = os.getenv("YAGNA_PAYMENT_DRIVER", "erc20").lower() PAYMENT_NETWORK: str = os.getenv("YAGNA_PAYMENT_NETWORK", "goerli").lower() diff --git a/golem/payload/__init__.py b/golem/payload/__init__.py new file mode 100644 index 00000000..02063e8a --- /dev/null +++ b/golem/payload/__init__.py @@ -0,0 +1,18 @@ +from golem.payload.base import Payload +from golem.payload.constraints import Constraints, ConstraintException +from golem.payload.exceptions import PayloadException, InvalidProperties +from golem.payload.properties import Properties +from golem.payload.vm import VmPayload, RepositoryVmPayload, ManifestVmPayload, VmPayloadException + +__all__ = ( + 'Payload', + 'VmPayload', + 'RepositoryVmPayload', + 'ManifestVmPayload', + 'VmPayloadException', + 'Constraints', + 'Properties', + 'PayloadException', + 'ConstraintException', + 'InvalidProperties', +) \ No newline at end of file diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/model.py b/golem/payload/base.py similarity index 61% rename from golem_core/core/market_api/resources/demand/demand_offer_base/model.py rename to golem/payload/base.py index 2730fccc..43436a75 100644 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/model.py +++ b/golem/payload/base.py @@ -1,32 +1,71 @@ import abc import dataclasses import enum -from typing import Any, Dict, Final, List, Tuple, Type, TypeVar +from typing import Any, Dict, Final, List, Tuple, Type, TypeVar, TypeAlias -from golem_core.core.market_api.resources.demand.demand_offer_base.exceptions import ( - InvalidPropertiesError, +from golem.payload.exceptions import ( + InvalidProperties, ) -from golem_core.core.props_cons.constraints import Constraint, ConstraintOperator, Constraints -from golem_core.core.props_cons.properties import Properties +from golem.payload.constraints import Constraint, ConstraintOperator, Constraints +from golem.payload.properties import Properties -TDemandOfferBaseModel = TypeVar("TDemandOfferBaseModel", bound="DemandOfferBaseModel") +TPayload = TypeVar("TPayload", bound="Payload") + +PropertyName: TypeAlias = str +PropertyValue: TypeAlias = Any PROP_KEY: Final[str] = "key" PROP_OPERATOR: Final[str] = "operator" PROP_MODEL_FIELD_TYPE: Final[str] = "model_field_type" -class DemandOfferBaseModelFieldType(enum.Enum): +class PayloadFieldType(enum.Enum): constraint = "constraint" property = "property" @dataclasses.dataclass -class DemandOfferBaseModel(abc.ABC): +class Payload(abc.ABC): """Base class for convenient declaration of Golem's property and constraints syntax. Provides helper methods to translate fields between python class and Golem's property and constraints syntax. + + Base class for descriptions of the payload required by the requestor. + + example usage:: + + import asyncio + + from dataclasses import dataclass + from golem.resources.market import DemandBuilder, prop, constraint, Payload, RUNTIME_NAME, INF_MEM, INF_STORAGE + + CUSTOM_RUNTIME_NAME = "my-runtime" + CUSTOM_PROPERTY = "golem.srv.app.myprop" + + + @dataclass + class MyPayload(Payload): + myprop: str = prop(CUSTOM_PROPERTY, default="myvalue") + runtime: str = constraint(RUNTIME_NAME, default=CUSTOM_RUNTIME_NAME) + min_mem_gib: float = constraint(INF_MEM, ">=", default=16) + min_storage_gib: float = constraint(INF_STORAGE, ">=", default=1024) + + + async def main(): + builder = DemandBuilder() + payload = MyPayload(myprop="othervalue", min_mem_gib=32) + await builder.add(payload) + print(builder) + + asyncio.run(main()) + + output:: + + {'properties': {'golem.srv.app.myprop': 'othervalue'}, 'constraints': ['(&(golem.runtime.name=my-runtime)\n\t(golem.inf.mem.gib>=32)\n\t(golem.inf.storage.gib>=1024))']} + + + """ def __init__(self, **kwargs): # pragma: no cover @@ -40,7 +79,7 @@ def _build_properties(self) -> Properties: return Properties( { field.metadata[PROP_KEY]: getattr(self, field.name) - for field in self._get_fields(DemandOfferBaseModelFieldType.property) + for field in self._get_fields(PayloadFieldType.property) } ) @@ -53,12 +92,12 @@ def _build_constraints(self) -> Constraints: operator=field.metadata[PROP_OPERATOR], value=getattr(self, field.name), ) - for field in self._get_fields(DemandOfferBaseModelFieldType.constraint) + for field in self._get_fields(PayloadFieldType.constraint) ] ) @classmethod - def _get_fields(cls, field_type: DemandOfferBaseModelFieldType) -> List[dataclasses.Field]: + def _get_fields(cls, field_type: PayloadFieldType) -> List[dataclasses.Field]: """Return a list of fields based on given type.""" return [ f @@ -68,8 +107,8 @@ def _get_fields(cls, field_type: DemandOfferBaseModelFieldType) -> List[dataclas @classmethod def from_properties( - cls: Type[TDemandOfferBaseModel], props: Dict[str, Any] - ) -> TDemandOfferBaseModel: + cls: Type[TPayload], props: Dict[str, Any] + ) -> TPayload: """Initialize the model with properties from given dictionary. Only properties defined in model will be picked up from given dictionary, ignoring other @@ -78,7 +117,7 @@ def from_properties( """ field_map = { field.metadata[PROP_KEY]: field - for field in cls._get_fields(DemandOfferBaseModelFieldType.property) + for field in cls._get_fields(PayloadFieldType.property) } data = { field_map[key].name: Properties._deserialize_value(val, field_map[key]) @@ -88,16 +127,16 @@ def from_properties( try: return cls(**data) except TypeError as e: - raise InvalidPropertiesError(f"Missing key: {e}") from e + raise InvalidProperties(f"Missing key: {e}") from e except Exception as e: - raise InvalidPropertiesError(str(e)) from e + raise InvalidProperties(str(e)) from e def prop( key: str, *, default: Any = dataclasses.MISSING, default_factory: Any = dataclasses.MISSING ): """ - Return a property-type dataclass field for a DemandOfferBaseModel. + Return a property-type dataclass field for a Payload. :param key: the key of the property, e.g. "golem.runtime.name" :param default: the default value of the property @@ -106,10 +145,10 @@ def prop( example: ```python >>> from dataclasses import dataclass - >>> from golem_core.core.market_api import DemandOfferBaseModel, prop, DemandBuilder + >>> from golem.resources.market import Payload, prop, DemandBuilder >>> >>> @dataclass - ... class Foo(DemandOfferBaseModel): + ... class Foo(Payload): ... bar: int = prop("bar", default=100) ... >>> builder = DemandBuilder() @@ -121,7 +160,7 @@ def prop( return dataclasses.field( # type: ignore[call-overload] default=default, default_factory=default_factory, - metadata={PROP_KEY: key, PROP_MODEL_FIELD_TYPE: DemandOfferBaseModelFieldType.property}, + metadata={PROP_KEY: key, PROP_MODEL_FIELD_TYPE: PayloadFieldType.property}, ) @@ -132,7 +171,7 @@ def constraint( default: Any = dataclasses.MISSING, default_factory: Any = dataclasses.MISSING, ): - """Return a constraint-type dataclass field for a DemandOfferBaseModel. + """Return a constraint-type dataclass field for a Payload. :param key: the key of the property on which the constraint is made - e.g. "golem.srv.comp.task_package" @@ -143,10 +182,10 @@ def constraint( example: ```python >>> from dataclasses import dataclass - >>> from golem_core.core.market_api import DemandOfferBaseModel, constraint, DemandBuilder + >>> from golem.resources.market import Payload, constraint, DemandBuilder >>> >>> @dataclass - ... class Foo(DemandOfferBaseModel): + ... class Foo(Payload): ... max_baz: int = constraint("baz", "<=", default=100) ... >>> builder = DemandBuilder() @@ -161,13 +200,6 @@ def constraint( metadata={ PROP_KEY: key, PROP_OPERATOR: operator, - PROP_MODEL_FIELD_TYPE: DemandOfferBaseModelFieldType.constraint, + PROP_MODEL_FIELD_TYPE: PayloadFieldType.constraint, }, ) - - -__all__ = ( - "DemandOfferBaseModel", - "prop", - "constraint", -) diff --git a/golem_core/core/props_cons/constraints.py b/golem/payload/constraints.py similarity index 86% rename from golem_core/core/props_cons/constraints.py rename to golem/payload/constraints.py index 0ebdaa87..da1218d1 100644 --- a/golem_core/core/props_cons/constraints.py +++ b/golem/payload/constraints.py @@ -2,7 +2,8 @@ from dataclasses import dataclass, field from typing import Any, Literal, MutableSequence, Union -from golem_core.core.props_cons.base import PropertyName, PropsConstrsSerializerMixin +from golem.payload.base import PropertyName +from golem.payload.mixins import PropsConsSerializerMixin class ConstraintException(Exception): @@ -14,7 +15,7 @@ class ConstraintException(Exception): @dataclass -class MarketDemandOfferSyntaxElement(PropsConstrsSerializerMixin, ABC): +class PayloadSyntaxElement(PropsConsSerializerMixin, ABC): def __post_init__(self) -> None: self._validate() @@ -32,7 +33,7 @@ def _validate(self) -> None: @dataclass -class Constraint(MarketDemandOfferSyntaxElement): +class Constraint(PayloadSyntaxElement): property_name: PropertyName operator: ConstraintOperator value: Any @@ -52,7 +53,7 @@ def _serialize(self) -> str: @dataclass -class ConstraintGroup(MarketDemandOfferSyntaxElement): +class ConstraintGroup(PayloadSyntaxElement): items: MutableSequence[Union["ConstraintGroup", Constraint]] = field(default_factory=list) operator: ConstraintGroupOperator = "&" diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/defaults.py b/golem/payload/defaults.py similarity index 96% rename from golem_core/core/market_api/resources/demand/demand_offer_base/defaults.py rename to golem/payload/defaults.py index d7fb4373..c1672300 100644 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/defaults.py +++ b/golem/payload/defaults.py @@ -3,7 +3,7 @@ from decimal import Decimal from typing import Optional -from golem_core.core.market_api.resources.demand.demand_offer_base.model import ( +from golem.resources.market.resources.demand.demand_offer_base.model import ( DemandOfferBaseModel, prop, ) diff --git a/golem/payload/exceptions.py b/golem/payload/exceptions.py new file mode 100644 index 00000000..c85c5765 --- /dev/null +++ b/golem/payload/exceptions.py @@ -0,0 +1,13 @@ +from golem.exceptions import GolemException + + +class PayloadException(GolemException): + pass + + +class ConstraintException(PayloadException): + pass + + +class InvalidProperties(PayloadException): + """`properties` given to `Payload.from_properties(cls, properties)` are invalid.""" diff --git a/golem_core/core/props_cons/base.py b/golem/payload/mixins.py similarity index 90% rename from golem_core/core/props_cons/base.py rename to golem/payload/mixins.py index f9947b2b..3379cb3a 100644 --- a/golem_core/core/props_cons/base.py +++ b/golem/payload/mixins.py @@ -4,13 +4,10 @@ from dataclasses import Field from typing import Any -from golem_core.utils.typing import match_type_union_aware +from golem.utils.typing import match_type_union_aware -PropertyName = str -PropertyValue = Any - -class PropsConstrsSerializerMixin: +class PropsConsSerializerMixin: @classmethod def _serialize_value(cls, value: Any) -> Any: """Return value in primitive format compatible with Golem's property and constraint syntax.""" diff --git a/golem/payload/parsers/__init__.py b/golem/payload/parsers/__init__.py new file mode 100644 index 00000000..8ebf7c5c --- /dev/null +++ b/golem/payload/parsers/__init__.py @@ -0,0 +1,7 @@ +from golem.payload.parsers.base import PayloadSyntaxParser, SyntaxException + + +__all__ = ( + 'PayloadSyntaxParser', + 'SyntaxException', +) \ No newline at end of file diff --git a/golem/payload/parsers/base.py b/golem/payload/parsers/base.py new file mode 100644 index 00000000..4fdd5f8d --- /dev/null +++ b/golem/payload/parsers/base.py @@ -0,0 +1,13 @@ +from abc import ABC, abstractmethod + +from golem.payload.constraints import Constraints + + +class SyntaxException(Exception): + pass + + +class PayloadSyntaxParser(ABC): + @abstractmethod + def parse_constraints(self, syntax: str) -> Constraints: + ... diff --git a/golem/payload/parsers/textx/__init__.py b/golem/payload/parsers/textx/__init__.py new file mode 100644 index 00000000..d2df8115 --- /dev/null +++ b/golem/payload/parsers/textx/__init__.py @@ -0,0 +1,5 @@ +from golem.payload.parsers.textx.parser import TextXPayloadSyntaxParser + +__all__ = ( + 'TextXPayloadSyntaxParser', +) \ No newline at end of file diff --git a/golem_core/core/props_cons/parsers/textx/parser.py b/golem/payload/parsers/textx/parser.py similarity index 71% rename from golem_core/core/props_cons/parsers/textx/parser.py rename to golem/payload/parsers/textx/parser.py index ab0ebaca..5be7fc9b 100644 --- a/golem_core/core/props_cons/parsers/textx/parser.py +++ b/golem/payload/parsers/textx/parser.py @@ -2,11 +2,11 @@ from textx import TextXSyntaxError, metamodel_from_file -from golem_core.core.props_cons.constraints import Constraint, ConstraintGroup, Constraints -from golem_core.core.props_cons.parsers.base import DemandOfferSyntaxParser, SyntaxException +from golem.payload.constraints import Constraint, ConstraintGroup, Constraints +from golem.payload.parsers.base import PayloadSyntaxParser, SyntaxException -class TextXDemandOfferSyntaxParser(DemandOfferSyntaxParser): +class TextXPayloadSyntaxParser(PayloadSyntaxParser): def __init__(self): self._metamodel = metamodel_from_file(str(Path(__file__).with_name("syntax.tx"))) self._metamodel.register_obj_processors( @@ -17,7 +17,7 @@ def __init__(self): } ) - def parse(self, syntax: str) -> Constraints: + def parse_constraints(self, syntax: str) -> Constraints: try: model = self._metamodel.model_from_str(syntax) except TextXSyntaxError as e: diff --git a/golem_core/core/props_cons/parsers/textx/syntax.tx b/golem/payload/parsers/textx/syntax.tx similarity index 100% rename from golem_core/core/props_cons/parsers/textx/syntax.tx rename to golem/payload/parsers/textx/syntax.tx diff --git a/golem_core/core/props_cons/properties.py b/golem/payload/properties.py similarity index 85% rename from golem_core/core/props_cons/properties.py rename to golem/payload/properties.py index 43ea0605..cedf56ae 100644 --- a/golem_core/core/props_cons/properties.py +++ b/golem/payload/properties.py @@ -1,12 +1,12 @@ from copy import deepcopy from typing import Any, Mapping -from golem_core.core.props_cons.base import PropsConstrsSerializerMixin +from golem.payload.mixins import PropsConsSerializerMixin _missing = object() -class Properties(PropsConstrsSerializerMixin, dict): +class Properties(PropsConsSerializerMixin, dict): """Low level wrapper class for Golem's Market API properties manipulation.""" def __init__(self, mapping=_missing, /) -> None: diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/payload/vm.py b/golem/payload/vm.py similarity index 89% rename from golem_core/core/market_api/resources/demand/demand_offer_base/payload/vm.py rename to golem/payload/vm.py index 508d5688..fb38a53e 100644 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/payload/vm.py +++ b/golem/payload/vm.py @@ -3,24 +3,24 @@ from dataclasses import dataclass from datetime import timedelta from enum import Enum -from typing import Final, List, Literal, Optional, Tuple +from typing import Final, List, Literal, Optional, Tuple, TypeAlias from dns.exception import DNSException from srvresolver.srv_record import SRVRecord from srvresolver.srv_resolver import SRVResolver -from golem_core.core.market_api.resources.demand.demand_offer_base import defaults -from golem_core.core.market_api.resources.demand.demand_offer_base.model import constraint, prop -from golem_core.core.market_api.resources.demand.demand_offer_base.payload.base import Payload -from golem_core.core.props_cons.constraints import Constraints -from golem_core.core.props_cons.properties import Properties -from golem_core.utils.http import make_http_get_request, make_http_head_request +from golem.resources.market.resources.demand.demand_offer_base import defaults +from golem.resources.market.resources.demand.demand_offer_base.model import constraint, prop +from golem.resources.market.resources.demand.demand_offer_base.payload.base import Payload +from golem.payload.constraints import Constraints +from golem.payload.properties import Properties +from golem.utils.http import make_http_get_request, make_http_head_request DEFAULT_REPO_URL_SRV: Final[str] = "_girepo._tcp.dev.golem.network" DEFAULT_REPO_URL_FALLBACK: Final[str] = "http://girepo.dev.golem.network:8000" DEFAULT_REPO_URL_TIMEOUT: Final[timedelta] = timedelta(seconds=10) -VmCaps = Literal["vpn", "inet", "manifest-support"] +VmCaps: TypeAlias = Literal["vpn", "inet", "manifest-support"] logger = logging.getLogger(__name__) diff --git a/golem/pipeline/__init__.py b/golem/pipeline/__init__.py new file mode 100644 index 00000000..0b0ee339 --- /dev/null +++ b/golem/pipeline/__init__.py @@ -0,0 +1,15 @@ +from golem.pipeline.buffer import Buffer +from golem.pipeline.chain import Chain +from golem.pipeline.limit import Limit +from golem.pipeline.map import Map +from golem.pipeline.sort import Sort +from golem.pipeline.zip import Zip + +__all__ = ( + "Chain", + "Limit", + "Map", + "Zip", + "Buffer", + "Sort", +) diff --git a/golem_core/pipeline/buffer.py b/golem/pipeline/buffer.py similarity index 98% rename from golem_core/pipeline/buffer.py rename to golem/pipeline/buffer.py index 2b51defd..547ee6b3 100644 --- a/golem_core/pipeline/buffer.py +++ b/golem/pipeline/buffer.py @@ -2,7 +2,7 @@ import inspect from typing import AsyncIterator, Awaitable, Generic, List, TypeVar, Union -from golem_core.pipeline.exceptions import InputStreamExhausted +from golem.pipeline.exceptions import InputStreamExhausted DataType = TypeVar("DataType") diff --git a/golem_core/pipeline/chain.py b/golem/pipeline/chain.py similarity index 94% rename from golem_core/pipeline/chain.py rename to golem/pipeline/chain.py index 3e8191c5..d99e93a2 100644 --- a/golem_core/pipeline/chain.py +++ b/golem/pipeline/chain.py @@ -20,8 +20,8 @@ async def int_2_str(numbers: AsyncIterator[int]) -> AsyncIterator[str]: A more Golem-specific usage:: - from golem_core import GolemNode, RepositoryVmPayload - from golem_core.mid import ( + from golem import GolemNode, RepositoryVmPayload + from golem.mid import ( Buffer, Chain, Map, default_negotiate, default_create_agreement, default_create_activity ) diff --git a/golem_core/pipeline/exceptions.py b/golem/pipeline/exceptions.py similarity index 92% rename from golem_core/pipeline/exceptions.py rename to golem/pipeline/exceptions.py index 17df6afd..5b3b2428 100644 --- a/golem_core/pipeline/exceptions.py +++ b/golem/pipeline/exceptions.py @@ -1,7 +1,7 @@ -from golem_core.exceptions import BaseGolemException +from golem.exceptions import GolemException -class InputStreamExhausted(BaseGolemException): +class InputStreamExhausted(GolemException): """Excepion used internally by pipeline-level components. This is something like StopAsyncIteration, but in context when StopAsyncIteration can't be used. diff --git a/golem_core/pipeline/limit.py b/golem/pipeline/limit.py similarity index 100% rename from golem_core/pipeline/limit.py rename to golem/pipeline/limit.py diff --git a/golem_core/pipeline/map.py b/golem/pipeline/map.py similarity index 98% rename from golem_core/pipeline/map.py rename to golem/pipeline/map.py index e6274e56..157b04e9 100644 --- a/golem_core/pipeline/map.py +++ b/golem/pipeline/map.py @@ -2,7 +2,7 @@ import inspect from typing import AsyncIterator, Awaitable, Callable, Generic, Tuple, TypeVar, Union -from golem_core.pipeline.exceptions import InputStreamExhausted +from golem.pipeline.exceptions import InputStreamExhausted InType = TypeVar("InType") OutType = TypeVar("OutType") diff --git a/golem_core/pipeline/sort.py b/golem/pipeline/sort.py similarity index 100% rename from golem_core/pipeline/sort.py rename to golem/pipeline/sort.py diff --git a/golem_core/pipeline/zip.py b/golem/pipeline/zip.py similarity index 100% rename from golem_core/pipeline/zip.py rename to golem/pipeline/zip.py diff --git a/golem_core/core/props_cons/__init__.py b/golem/resources/__init__.py similarity index 100% rename from golem_core/core/props_cons/__init__.py rename to golem/resources/__init__.py diff --git a/golem/resources/activity/__init__.py b/golem/resources/activity/__init__.py new file mode 100644 index 00000000..c9375bea --- /dev/null +++ b/golem/resources/activity/__init__.py @@ -0,0 +1,27 @@ +from golem.resources.activity.activity import Activity +from golem.resources.activity.commands import ( + Command, + Deploy, + DownloadFile, + Run, + Script, + SendFile, + Start, +) +from golem.resources.activity.events import NewActivity, ActivityDataChanged, ActivityClosed +from golem.resources.activity.pipeline import default_prepare_activity + +__all__ = ( + "Activity", + 'NewActivity', + 'ActivityDataChanged', + 'ActivityClosed', + "Command", + "Script", + "Deploy", + "Start", + "Run", + "SendFile", + "DownloadFile", + "default_prepare_activity", +) \ No newline at end of file diff --git a/golem_core/core/activity_api/resources/activity.py b/golem/resources/activity/activity.py similarity index 89% rename from golem_core/core/activity_api/resources/activity.py rename to golem/resources/activity/activity.py index 7bb714b1..afb74b11 100644 --- a/golem_core/core/activity_api/resources/activity.py +++ b/golem/resources/activity/activity.py @@ -5,19 +5,19 @@ from ya_activity import models -from golem_core.core.activity_api.commands import Command, Script -from golem_core.core.activity_api.events import ActivityClosed, NewActivity -from golem_core.core.activity_api.resources.pooling_batch import PoolingBatch -from golem_core.core.payment_api import DebitNote -from golem_core.core.resources import _NULL, ActivityApi, Resource, api_call_wrapper +from golem.resources.activity.commands import Command, Script +from golem.resources.activity.events import ActivityClosed, NewActivity +from golem.resources.pooling_batch.pooling_batch import PoolingBatch +from golem.resources.payment import DebitNote +from golem.resources.resources import _NULL, ActivityApi, Resource, api_call_wrapper if TYPE_CHECKING: - from golem_core.core.golem_node.golem_node import GolemNode - from golem_core.core.market_api import Agreement # noqa + from golem.resources.golem_node.golem_node import GolemNode + from golem.resources.market import Agreement # noqa class Activity(Resource[ActivityApi, _NULL, "Agreement", PoolingBatch, _NULL]): - """A single activity_api on the Golem Network. + """A single activity on the Golem Network. Either created by :any:`Agreement.create_activity()` or via :any:`GolemNode.activity()`. """ @@ -104,7 +104,7 @@ async def execute_commands(self, *commands: Command) -> PoolingBatch: Sample usage:: - batch = await activity_api.execute_commands( + batch = await activity.execute_commands( Deploy(), Start(), Run("echo -n 'hello world'"), @@ -137,7 +137,7 @@ async def execute_script(self, script: "Script") -> PoolingBatch: result = script.add_command(Run("echo -n 'hello world'")) script.add_command(Run("sleep 1000")) - batch = await activity_api.execute_script(script) + batch = await activity.execute_script(script) # This line doesn't wait for the whole batch to finish print((await result).stdout) # "hello world" diff --git a/golem_core/core/activity_api/commands.py b/golem/resources/activity/commands.py similarity index 98% rename from golem_core/core/activity_api/commands.py rename to golem/resources/activity/commands.py index 1a7acdfb..7633dabb 100644 --- a/golem_core/core/activity_api/commands.py +++ b/golem/resources/activity/commands.py @@ -7,7 +7,7 @@ from ya_activity import models -from golem_core.utils.storage import Destination, GftpProvider, Source +from golem.storage import Destination, GftpProvider, Source ArgsDict = Mapping[str, Union[str, List, Dict[str, Any]]] diff --git a/golem/resources/activity/events.py b/golem/resources/activity/events.py new file mode 100644 index 00000000..a740de61 --- /dev/null +++ b/golem/resources/activity/events.py @@ -0,0 +1,10 @@ +class NewActivity(NewResource["Activity"]): + pass + + +class ActivityDataChanged(ResourceDataChanged["Activity"]): + pass + + +class ActivityClosed(ResourceClosed["Activity"]): + pass diff --git a/golem_core/core/activity_api/pipeline.py b/golem/resources/activity/pipeline.py similarity index 83% rename from golem_core/core/activity_api/pipeline.py rename to golem/resources/activity/pipeline.py index e56db263..d7270c51 100644 --- a/golem_core/core/activity_api/pipeline.py +++ b/golem/resources/activity/pipeline.py @@ -1,5 +1,5 @@ -from golem_core.core.activity_api.commands import Deploy, Start -from golem_core.core.activity_api.resources import Activity +from golem.resources.activity.commands import Deploy, Start +from golem.resources.activity.resources import Activity # TODO: Move default functions to Activity class diff --git a/golem/resources/agreement/__init__.py b/golem/resources/agreement/__init__.py new file mode 100644 index 00000000..74100277 --- /dev/null +++ b/golem/resources/agreement/__init__.py @@ -0,0 +1,11 @@ +from golem.resources.agreement.events import NewAgreement, AgreementDataChanged, AgreementClosed +from golem.resources.agreement.pipeline import default_create_activity +from golem.resources.agreement.agreement import Agreement + +__all__ = ( + Agreement, + NewAgreement, + AgreementDataChanged, + AgreementClosed, + default_create_activity, +) diff --git a/golem_core/core/market_api/resources/agreement.py b/golem/resources/agreement/agreement.py similarity index 88% rename from golem_core/core/market_api/resources/agreement.py rename to golem/resources/agreement/agreement.py index af5501ec..0e71f1b5 100644 --- a/golem_core/core/market_api/resources/agreement.py +++ b/golem/resources/agreement/agreement.py @@ -6,14 +6,14 @@ from ya_market import models as models from ya_market.exceptions import ApiException -from golem_core.core.activity_api import Activity -from golem_core.core.market_api import AgreementClosed, NewAgreement -from golem_core.core.payment_api import Invoice -from golem_core.core.resources import _NULL, Resource, api_call_wrapper -from golem_core.core.resources.base import TModel +from golem.resources.activity import Activity +from golem.resources.agreement.events import NewAgreement, AgreementClosed +from golem.resources.payment import Invoice +from golem.resources.resources import _NULL, Resource, api_call_wrapper +from golem.resources.resources.base import TModel if TYPE_CHECKING: - from golem_core.core.market_api.resources.proposal import Proposal # noqa + from golem.resources.market.proposal import Proposal # noqa class Agreement(Resource[RequestorApi, models.Agreement, "Proposal", Activity, _NULL]): @@ -24,8 +24,8 @@ class Agreement(Resource[RequestorApi, models.Agreement, "Proposal", Activity, _ agreement = await proposal.create_agreement() await agreement.confirm() await agreement.wait_for_approval() - activity_api = await agreement.create_activity() - # Use the activity_api + activity = await agreement.create_activity() + # Use the activity await agreement.terminate() """ @@ -67,10 +67,10 @@ async def create_activity( ) -> "Activity": """Create a new :any:`Activity` for this :any:`Agreement`. - :param autoclose: Destroy the activity_api when the :any:`GolemNode` closes. + :param autoclose: Destroy the activity when the :any:`GolemNode` closes. :param timeout: Request timeout. """ - from golem_core.core.activity_api.resources import Activity + from golem.resources.activity.resources import Activity activity = await Activity.create(self.node, self.id, timeout) if autoclose: @@ -106,7 +106,7 @@ def invoice(self) -> Optional[Invoice]: @property def activities(self) -> List["Activity"]: """A list of :any:`Activity` created for this :any:`Agreement`.""" - from golem_core.core.activity_api.resources import Activity # circular imports prevention + from golem.resources.activity.resources import Activity # circular imports prevention return [child for child in self.children if isinstance(child, Activity)] diff --git a/golem/resources/agreement/events.py b/golem/resources/agreement/events.py new file mode 100644 index 00000000..b6164e92 --- /dev/null +++ b/golem/resources/agreement/events.py @@ -0,0 +1,10 @@ +class NewAgreement(NewResource["Agreement"]): + pass + + +class AgreementDataChanged(ResourceDataChanged["Agreement"]): + pass + + +class AgreementClosed(ResourceClosed["Agreement"]): + pass diff --git a/golem/resources/agreement/pipeline.py b/golem/resources/agreement/pipeline.py new file mode 100644 index 00000000..5a442fc3 --- /dev/null +++ b/golem/resources/agreement/pipeline.py @@ -0,0 +1,3 @@ +async def default_create_activity(agreement: Agreement) -> Activity: + """Create a new :any:`Activity` for a given :any:`Agreement`.""" + return await agreement.create_activity() diff --git a/golem/resources/allocation/__init__.py b/golem/resources/allocation/__init__.py new file mode 100644 index 00000000..dafe3ace --- /dev/null +++ b/golem/resources/allocation/__init__.py @@ -0,0 +1,13 @@ +from golem.resources.allocation.allocation import Allocation +from golem.resources.allocation.events import NewAllocation, AllocationDataChanged, AllocationClosed +from golem.resources.allocation.exceptions import AllocationException, NoMatchingAccount + + +__all__ = ( + 'Allocation', + 'AllocationException', + 'NoMatchingAccount', + 'NewAllocation', + 'AllocationDataChanged', + 'AllocationClosed', +) diff --git a/golem_core/core/payment_api/resources/allocation.py b/golem/resources/allocation/allocation.py similarity index 81% rename from golem_core/core/payment_api/resources/allocation.py rename to golem/resources/allocation/allocation.py index c30b61a2..50e55ee7 100644 --- a/golem_core/core/payment_api/resources/allocation.py +++ b/golem/resources/allocation/allocation.py @@ -2,19 +2,19 @@ from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Optional, Tuple -from _decimal import Decimal +from decimal import Decimal from ya_payment import RequestorApi, models -from golem_core.core.payment_api.events import NewAllocation -from golem_core.core.payment_api.exceptions import NoMatchingAccount -from golem_core.core.props_cons.constraints import Constraints -from golem_core.core.props_cons.parsers.base import DemandOfferSyntaxParser -from golem_core.core.props_cons.properties import Properties -from golem_core.core.resources import _NULL, Resource, ResourceClosed, api_call_wrapper -from golem_core.core.resources.base import TModel +from golem.resources.allocation.events import NewAllocation +from golem.resources.allocation.exceptions import NoMatchingAccount +from golem.payload.constraints import Constraints +from golem.payload.parsers.base import PayloadSyntaxParser +from golem.payload.properties import Properties +from golem.resources.resources import _NULL, Resource, ResourceClosed, api_call_wrapper +from golem.resources.resources.base import TModel if TYPE_CHECKING: - from golem_core.core.golem_node import GolemNode + from golem.resources.golem_node import GolemNode class Allocation(Resource[RequestorApi, models.Allocation, _NULL, _NULL, _NULL]): @@ -90,12 +90,12 @@ async def create(cls, node: "GolemNode", data: models.Allocation) -> "Allocation @api_call_wrapper() async def get_properties_and_constraints_for_demand( - self, parser: DemandOfferSyntaxParser + self, parser: PayloadSyntaxParser ) -> Tuple[Properties, Constraints]: data = await self.api.get_demand_decorations([self.id]) properties = Properties({prop.key: prop.value for prop in data.properties}) - constraints = Constraints([parser.parse(c) for c in data.constraints]) + constraints = Constraints([parser.parse_constraints(c) for c in data.constraints]) return properties, constraints diff --git a/golem/resources/allocation/events.py b/golem/resources/allocation/events.py new file mode 100644 index 00000000..b00d0dbd --- /dev/null +++ b/golem/resources/allocation/events.py @@ -0,0 +1,10 @@ +class NewAllocation(NewResource["Allocation"]): + pass + + +class AllocationDataChanged(ResourceDataChanged["Allocation"]): + pass + + +class AllocationClosed(ResourceClosed["Allocation"]): + pass diff --git a/golem_core/core/payment_api/exceptions.py b/golem/resources/allocation/exceptions.py similarity index 73% rename from golem_core/core/payment_api/exceptions.py rename to golem/resources/allocation/exceptions.py index 25d97433..9071c356 100644 --- a/golem_core/core/payment_api/exceptions.py +++ b/golem/resources/allocation/exceptions.py @@ -1,12 +1,12 @@ -from golem_core.core.exceptions import BaseCoreException +from golem.resources.exceptions import ResourceException -class BasePaymentApiException(BaseCoreException): +class AllocationException(ResourceException): pass -class NoMatchingAccount(BasePaymentApiException): - """Raised when a new :any:`Allocation` is created for a (network_api, driver) pair without \ +class NoMatchingAccount(AllocationException): + """Raised when a new :any:`Allocation` is created for a (network, driver) pair without \ matching `yagna` account.""" def __init__(self, network: str, driver: str): @@ -15,7 +15,7 @@ def __init__(self, network: str, driver: str): # NOTE: we don't really care about this sort of compatibility, but this is # a message developers are used to so maybe it's worth reusing - msg = f"No payment account available for driver `{driver}` and network_api `{network}`" + msg = f"No payment account available for driver `{driver}` and network `{network}`" super().__init__(msg) @property diff --git a/golem_core/core/resources/base.py b/golem/resources/base.py similarity index 96% rename from golem_core/core/resources/base.py rename to golem/resources/base.py index 52f33d49..bd6d7539 100644 --- a/golem_core/core/resources/base.py +++ b/golem/resources/base.py @@ -20,12 +20,12 @@ from ya_net import ApiException as NetApiException from ya_payment import ApiException as PaymentApiException -from golem_core.core.resources.events import ResourceDataChanged -from golem_core.core.resources.exceptions import ResourceNotFound -from golem_core.core.resources.low import TRequestorApi, get_requestor_api +from golem.resources.events import ResourceDataChanged +from golem.resources.exceptions import ResourceNotFound +from golem.node import TRequestorApi, get_requestor_api if TYPE_CHECKING: - from golem_core.core.golem_node.golem_node import GolemNode + from golem.node import GolemNode all_api_exceptions = ( PaymentApiException, @@ -181,7 +181,7 @@ def add_event(self, event: TEvent) -> None: def events(self) -> List[TEvent]: """Returns a list of all `yagna` events related to this :class:`Resource`. - Note: these are **yagna** events and should not be confused with `golem_core.core.events`. + Note: these are **yagna** events and should not be confused with `golem.resources.events`. """ return self._events.copy() diff --git a/golem/resources/debit_note/__init__.py b/golem/resources/debit_note/__init__.py new file mode 100644 index 00000000..45687890 --- /dev/null +++ b/golem/resources/debit_note/__init__.py @@ -0,0 +1,10 @@ +from golem.resources.debit_note.debit_note import DebitNote +from golem.resources.debit_note.events import NewDebitNote, DebitNoteDataChanged, DebitNoteClosed + + +__all__ = ( + 'DebitNote', + 'NewDebitNote', + 'DebitNoteDataChanged', + 'DebitNoteClosed', +) diff --git a/golem_core/core/payment_api/resources/debit_note.py b/golem/resources/debit_note/debit_note.py similarity index 76% rename from golem_core/core/payment_api/resources/debit_note.py rename to golem/resources/debit_note/debit_note.py index 6159328e..f527149b 100644 --- a/golem_core/core/payment_api/resources/debit_note.py +++ b/golem/resources/debit_note/debit_note.py @@ -4,14 +4,14 @@ from _decimal import Decimal from ya_payment import RequestorApi, models -from golem_core.core.payment_api import NewDebitNote -from golem_core.core.payment_api.resources.allocation import Allocation -from golem_core.core.resources import _NULL, Resource, api_call_wrapper -from golem_core.core.resources.base import TModel +from golem.resources.debit_note.events import NewDebitNote +from golem.resources.allocation.allocation import Allocation +from golem.resources.resources import _NULL, Resource, api_call_wrapper +from golem.resources.resources.base import TModel if TYPE_CHECKING: - from golem_core.core.activity_api import Activity # noqa - from golem_core.core.golem_node import GolemNode + from golem.resources.activity import Activity # noqa + from golem.resources.golem_node import GolemNode class DebitNote(Resource[RequestorApi, models.DebitNote, "Activity", _NULL, _NULL]): diff --git a/golem/resources/debit_note/event_collectors.py b/golem/resources/debit_note/event_collectors.py new file mode 100644 index 00000000..0c2805d2 --- /dev/null +++ b/golem/resources/debit_note/event_collectors.py @@ -0,0 +1,20 @@ +from typing import Callable, Tuple, TYPE_CHECKING + +from golem.resources.debit_note import DebitNote +from golem.resources.event_collectors import PaymentEventCollector, DebitNoteEvent + +if TYPE_CHECKING: + from golem.resources.activity import Activity + + +class DebitNoteEventCollector(PaymentEventCollector): + @property + def _collect_events_func(self) -> Callable: + return DebitNote._get_api(self.node).get_debit_note_events + + async def _get_event_resources(self, event: DebitNoteEvent) -> Tuple[DebitNote, "Activity"]: + assert event.debit_note_id is not None + debit_note = self.node.debit_note(event.debit_note_id) + await debit_note.get_data() + activity = self.node.activity(debit_note.data.activity_id) + return debit_note, activity diff --git a/golem/resources/debit_note/events.py b/golem/resources/debit_note/events.py new file mode 100644 index 00000000..19e16ce8 --- /dev/null +++ b/golem/resources/debit_note/events.py @@ -0,0 +1,10 @@ +class NewDebitNote(NewResource["DebitNote"]): + pass + + +class DebitNoteDataChanged(ResourceDataChanged["DebitNote"]): + pass + + +class DebitNoteClosed(ResourceClosed["DebitNote"]): + pass diff --git a/golem/resources/demand/__init__.py b/golem/resources/demand/__init__.py new file mode 100644 index 00000000..d9fe88f4 --- /dev/null +++ b/golem/resources/demand/__init__.py @@ -0,0 +1,11 @@ +from golem.resources.demand.demand import Demand, DemandData +from golem.resources.demand.events import NewDemand, DemandDataChanged, DemandClosed + + +__all__ = ( + 'Demand', + 'DemandData', + 'NewDemand', + 'DemandDataChanged', + 'DemandClosed', +) diff --git a/golem_core/core/market_api/resources/demand/demand.py b/golem/resources/demand/demand.py similarity index 91% rename from golem_core/core/market_api/resources/demand/demand.py rename to golem/resources/demand/demand.py index 61332985..35d183ef 100644 --- a/golem_core/core/market_api/resources/demand/demand.py +++ b/golem/resources/demand/demand.py @@ -6,21 +6,21 @@ from ya_market import RequestorApi from ya_market import models as models -from golem_core.core.market_api.events import DemandClosed, NewDemand -from golem_core.core.market_api.resources.proposal import Proposal -from golem_core.core.props_cons.constraints import Constraints -from golem_core.core.props_cons.properties import Properties -from golem_core.core.resources import ( +from golem.resources.demand.events import NewDemand, DemandClosed +from golem.resources.market.proposal import Proposal +from golem.payload.constraints import Constraints +from golem.payload.properties import Properties +from golem.resources.resources import ( _NULL, Resource, ResourceNotFound, YagnaEventCollector, api_call_wrapper, ) -from golem_core.core.resources.base import TModel +from golem.resources.resources.base import TModel if TYPE_CHECKING: - from golem_core.core.golem_node import GolemNode + from golem.resources.golem_node import GolemNode @dataclass diff --git a/golem_core/core/market_api/resources/demand/demand_builder.py b/golem/resources/demand/demand_builder.py similarity index 82% rename from golem_core/core/market_api/resources/demand/demand_builder.py rename to golem/resources/demand/demand_builder.py index 97e8be88..835cb224 100644 --- a/golem_core/core/market_api/resources/demand/demand_builder.py +++ b/golem/resources/demand/demand_builder.py @@ -2,16 +2,16 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Iterable, Optional, Union -from golem_core.core.market_api.resources.demand.demand import Demand -from golem_core.core.market_api.resources.demand.demand_offer_base import defaults as dobm_defaults -from golem_core.core.market_api.resources.demand.demand_offer_base.model import DemandOfferBaseModel -from golem_core.core.payment_api.resources.allocation import Allocation -from golem_core.core.props_cons.constraints import Constraint, ConstraintGroup, Constraints -from golem_core.core.props_cons.parsers.base import DemandOfferSyntaxParser -from golem_core.core.props_cons.properties import Properties +from golem.resources.demand.demand import Demand +from golem.resources.market.resources.demand.demand_offer_base import defaults as dobm_defaults +from golem.resources.market.resources.demand.demand_offer_base.model import DemandOfferBaseModel +from golem.resources.allocation.allocation import Allocation +from golem.payload.constraints import Constraint, ConstraintGroup, Constraints +from golem.payload.parsers.base import PayloadSyntaxParser +from golem.payload.properties import Properties if TYPE_CHECKING: - from golem_core.core.golem_node import GolemNode + from golem.resources.golem_node import GolemNode class DemandBuilder: @@ -21,7 +21,7 @@ class DemandBuilder: example usage: ```python - >>> from golem_core.core.market_api import DemandBuilder, pipeline + >>> from golem.resources.market import DemandBuilder, pipeline >>> from datetime import datetime, timezone >>> builder = DemandBuilder() >>> await builder.add(defaults.NodeInfo(name="a node", subnet_tag="testnet")) @@ -79,7 +79,7 @@ def add_constraints(self, constraints: Union[Constraint, ConstraintGroup]): async def add_default_parameters( self, - parser: DemandOfferSyntaxParser, + parser: PayloadSyntaxParser, subnet: Optional[str] = None, expiration: Optional[datetime] = None, allocations: Iterable[Allocation] = (), @@ -92,7 +92,7 @@ async def add_default_parameters( :param allocations: Allocations that will be included in the description of this demand. """ # FIXME: get rid of local import - from golem_core.core.golem_node.golem_node import DEFAULT_EXPIRATION_TIMEOUT, SUBNET + from golem.resources.golem_node.golem_node import DEFAULT_EXPIRATION_TIMEOUT, SUBNET if subnet is None: subnet = SUBNET diff --git a/golem/resources/demand/events.py b/golem/resources/demand/events.py new file mode 100644 index 00000000..7cd3171d --- /dev/null +++ b/golem/resources/demand/events.py @@ -0,0 +1,10 @@ +class NewDemand(NewResource["Demand"]): + pass + + +class DemandDataChanged(ResourceDataChanged["Demand"]): + pass + + +class DemandClosed(ResourceClosed["Demand"]): + pass diff --git a/golem/resources/event_collectors.py b/golem/resources/event_collectors.py new file mode 100644 index 00000000..41ce0775 --- /dev/null +++ b/golem/resources/event_collectors.py @@ -0,0 +1,48 @@ +from abc import ABC, abstractmethod +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, Dict, Tuple, Union, TypeAlias + +from ya_payment import models + +from golem.resources.resources import Resource, YagnaEventCollector + +if TYPE_CHECKING: + from golem_core.core.golem_node.golem_node import GolemNode + +InvoiceEvent: TypeAlias = Union[ + models.InvoiceReceivedEvent, + models.InvoiceAcceptedEvent, + models.InvoiceReceivedEvent, + models.InvoiceFailedEvent, + models.InvoiceSettledEvent, + models.InvoiceCancelledEvent, +] + +DebitNoteEvent: TypeAlias = Union[ + models.DebitNoteReceivedEvent, + models.DebitNoteAcceptedEvent, + models.DebitNoteReceivedEvent, + models.DebitNoteFailedEvent, + models.DebitNoteSettledEvent, + models.DebitNoteCancelledEvent, +] + + +class PaymentEventCollector(YagnaEventCollector, ABC): + def __init__(self, node: "GolemNode"): + self.node = node + self.min_ts = datetime.now(timezone.utc) + + def _collect_events_kwargs(self) -> Dict: + return {"after_timestamp": self.min_ts, "app_session_id": self.node.app_session_id} + + async def _process_event(self, event: Union[InvoiceEvent, DebitNoteEvent]) -> None: + self.min_ts = max(event.event_date, self.min_ts) + resource, parent_resource = await self._get_event_resources(event) + resource.add_event(event) + if resource._parent is None: + parent_resource.add_child(resource) + + @abstractmethod + async def _get_event_resources(self, event: Any) -> Tuple[Resource, Resource]: + raise NotImplementedError diff --git a/golem_core/core/resources/events.py b/golem/resources/events.py similarity index 97% rename from golem_core/core/resources/events.py rename to golem/resources/events.py index af76b0ba..5064adb6 100644 --- a/golem_core/core/resources/events.py +++ b/golem/resources/events.py @@ -1,10 +1,10 @@ from abc import ABC from typing import TYPE_CHECKING, Any, Dict, Generic, Tuple, TypeVar -from golem_core.core.events.base import Event +from golem.event_bus import Event if TYPE_CHECKING: - from golem_core.core.resources.base import Resource + from golem.resources.base import Resource TResourceEvent = TypeVar("TResourceEvent", bound="ResourceEvent") TResource = TypeVar("TResource", bound="Resource") diff --git a/golem_core/core/resources/exceptions.py b/golem/resources/exceptions.py similarity index 79% rename from golem_core/core/resources/exceptions.py rename to golem/resources/exceptions.py index 24cb64d2..ff8176bb 100644 --- a/golem_core/core/resources/exceptions.py +++ b/golem/resources/exceptions.py @@ -1,16 +1,16 @@ from typing import TYPE_CHECKING -from golem_core.core.exceptions import BaseCoreException +from golem.exceptions import GolemException if TYPE_CHECKING: - from golem_core.core.resources.base import Resource + from golem.resources.resources.base import Resource -class BaseResourceException(BaseCoreException): +class ResourceException(GolemException): pass -class MissingConfiguration(BaseResourceException): +class MissingConfiguration(ResourceException): def __init__(self, key: str, description: str): self._key = key self._description = description @@ -19,7 +19,7 @@ def __str__(self) -> str: return f"Missing configuration for {self._description}. Please set env var {self._key}." -class ResourceNotFound(BaseResourceException): +class ResourceNotFound(ResourceException): """Raised on an attempt to interact with a resource that doesn't exist. Example:: diff --git a/golem/resources/invoice/__init__.py b/golem/resources/invoice/__init__.py new file mode 100644 index 00000000..812b1657 --- /dev/null +++ b/golem/resources/invoice/__init__.py @@ -0,0 +1,10 @@ +from golem.resources.invoice.events import NewInvoice, InvoiceDataChanged, InvoiceClosed +from golem.resources.invoice.invoice import Invoice + + +__all__ = ( + 'Invoice', + 'NewInvoice', + 'InvoiceDataChanged', + 'InvoiceClosed', +) diff --git a/golem/resources/invoice/event_collectors.py b/golem/resources/invoice/event_collectors.py new file mode 100644 index 00000000..351e45a6 --- /dev/null +++ b/golem/resources/invoice/event_collectors.py @@ -0,0 +1,20 @@ +from typing import Callable, Tuple, TYPE_CHECKING + +from golem.resources.event_collectors import PaymentEventCollector, InvoiceEvent +from golem.resources.invoice.invoice import Invoice + +if TYPE_CHECKING: + from golem_core.core.market_api import Agreement + + +class InvoiceEventCollector(PaymentEventCollector): + @property + def _collect_events_func(self) -> Callable: + return Invoice._get_api(self.node).get_invoice_events + + async def _get_event_resources(self, event: InvoiceEvent) -> Tuple[Invoice, "Agreement"]: + assert event.invoice_id is not None + invoice = self.node.invoice(event.invoice_id) + await invoice.get_data() + agreement = self.node.agreement(invoice.data.agreement_id) + return invoice, agreement diff --git a/golem/resources/invoice/events.py b/golem/resources/invoice/events.py new file mode 100644 index 00000000..d18cdbcc --- /dev/null +++ b/golem/resources/invoice/events.py @@ -0,0 +1,13 @@ +from golem.resources.resources import NewResource, ResourceClosed, ResourceDataChanged + + +class NewInvoice(NewResource["Invoice"]): + pass + + +class InvoiceDataChanged(ResourceDataChanged["Invoice"]): + pass + + +class InvoiceClosed(ResourceClosed["Invoice"]): + pass diff --git a/golem_core/core/payment_api/resources/invoice.py b/golem/resources/invoice/invoice.py similarity index 75% rename from golem_core/core/payment_api/resources/invoice.py rename to golem/resources/invoice/invoice.py index 43d17a80..967bbfee 100644 --- a/golem_core/core/payment_api/resources/invoice.py +++ b/golem/resources/invoice/invoice.py @@ -4,14 +4,14 @@ from ya_payment import RequestorApi, models -from golem_core.core.payment_api.events import NewInvoice -from golem_core.core.payment_api.resources.allocation import Allocation -from golem_core.core.resources import _NULL, Resource, api_call_wrapper -from golem_core.core.resources.base import TModel +from golem.resources.payment.events import NewInvoice +from golem.resources.allocation.allocation import Allocation +from golem.resources.resources import _NULL, Resource, api_call_wrapper +from golem.resources.resources.base import TModel if TYPE_CHECKING: - from golem_core.core.golem_node import GolemNode - from golem_core.core.market_api.resources.agreement import Agreement # noqa + from golem.resources.golem_node import GolemNode + from golem.resources.agreement.agreement import Agreement # noqa class Invoice(Resource[RequestorApi, models.Invoice, "Agreement", _NULL, _NULL]): diff --git a/golem/resources/network/__init__.py b/golem/resources/network/__init__.py new file mode 100644 index 00000000..b88265d5 --- /dev/null +++ b/golem/resources/network/__init__.py @@ -0,0 +1,13 @@ +from golem.resources.network.events import NewNetwork, NetworkDataChanged, NetworkClosed +from golem.resources.network.exceptions import NetworkFull, NetworkException +from golem.resources.network.network import Network + + +__all__ = ( + "Network", + "NetworkException", + "NetworkFull", + "NewNetwork", + "NetworkDataChanged", + "NetworkClosed", +) diff --git a/golem_core/core/network_api/events.py b/golem/resources/network/events.py similarity index 80% rename from golem_core/core/network_api/events.py rename to golem/resources/network/events.py index 92f31c55..9d6aafad 100644 --- a/golem_core/core/network_api/events.py +++ b/golem/resources/network/events.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING -from golem_core.core.resources import NewResource, ResourceClosed, ResourceDataChanged +from golem.resources.resources import NewResource, ResourceClosed, ResourceDataChanged if TYPE_CHECKING: pass diff --git a/golem_core/core/network_api/exceptions.py b/golem/resources/network/exceptions.py similarity index 65% rename from golem_core/core/network_api/exceptions.py rename to golem/resources/network/exceptions.py index b67ecab8..2c0c7bd2 100644 --- a/golem_core/core/network_api/exceptions.py +++ b/golem/resources/network/exceptions.py @@ -1,16 +1,16 @@ from typing import TYPE_CHECKING -from golem_core.core.exceptions import BaseCoreException +from golem.resources.exceptions import ResourceException if TYPE_CHECKING: - from golem_core.core.network_api.resources import Network + from golem.resources.network.network import Network -class BaseNetworkApiException(BaseCoreException): +class NetworkException(ResourceException): pass -class NetworkFull(BaseNetworkApiException): +class NetworkFull(NetworkException): """Raised when we need a new free ip but there are no free ips left in the :any:`Network`.""" def __init__(self, network: "Network"): diff --git a/golem_core/core/network_api/resources/network.py b/golem/resources/network/network.py similarity index 94% rename from golem_core/core/network_api/resources/network.py rename to golem/resources/network/network.py index 33a4381d..fe38e0bb 100644 --- a/golem_core/core/network_api/resources/network.py +++ b/golem/resources/network/network.py @@ -4,12 +4,12 @@ from ya_net import RequestorApi, models -from golem_core.core.network_api.events import NewNetwork -from golem_core.core.network_api.exceptions import NetworkFull -from golem_core.core.resources import _NULL, Resource, ResourceClosed, api_call_wrapper +from golem.resources.network.events import NewNetwork +from golem.resources.network.exceptions import NetworkFull +from golem.resources.resources import _NULL, Resource, ResourceClosed, api_call_wrapper if TYPE_CHECKING: - from golem_core.core.golem_node import GolemNode + from golem.resources.golem_node import GolemNode IpAddress = Union[IPv4Address, IPv6Address] IpNetwork = Union[IPv4Network, IPv6Network] @@ -83,7 +83,7 @@ async def create_node(self, provider_id: str, node_ip: Optional[str] = None) -> """ # Q: Why is there no `Node` class? # A: Mostly because yagna nodes don't have proper IDs (they are just provider_ids), and - # this is strongly against the current golem_core object model (e.g. what if we want to + # this is strongly against the current golem object model (e.g. what if we want to # have the same provider in muliple networks? Nodes would share the same id, but are # totally diferent objects). We could bypass this by having some internal ids # (e.g. network_id-provider_id, or just uuid), but this would not be pretty and there's diff --git a/golem/resources/pooling_batch/__init__.py b/golem/resources/pooling_batch/__init__.py new file mode 100644 index 00000000..64016284 --- /dev/null +++ b/golem/resources/pooling_batch/__init__.py @@ -0,0 +1,20 @@ +from golem.resources.pooling_batch.events import NewPoolingBatch, BatchFinished +from golem.resources.pooling_batch.exceptions import ( + PoolingBatchException, + BatchError, + CommandFailed, + CommandCancelled, + BatchTimeoutError, +) +from golem.resources.pooling_batch.pooling_batch import PoolingBatch + +__all__ = ( + PoolingBatch, + NewPoolingBatch, + BatchFinished, + PoolingBatchException, + BatchError, + CommandFailed, + CommandCancelled, + BatchTimeoutError, +) \ No newline at end of file diff --git a/golem/resources/pooling_batch/events.py b/golem/resources/pooling_batch/events.py new file mode 100644 index 00000000..46d2502d --- /dev/null +++ b/golem/resources/pooling_batch/events.py @@ -0,0 +1,9 @@ +class NewPoolingBatch(NewResource["PoolingBatch"]): + pass + + +class BatchFinished(ResourceEvent["PoolingBatch"]): + """Emitted when the execution of a :any:`PoolingBatch` finishes. + + The same event is emitted for successful and failed batches. + """ diff --git a/golem_core/core/activity_api/exceptions.py b/golem/resources/pooling_batch/exceptions.py similarity index 87% rename from golem_core/core/activity_api/exceptions.py rename to golem/resources/pooling_batch/exceptions.py index 1c0d3344..a9b15fec 100644 --- a/golem_core/core/activity_api/exceptions.py +++ b/golem/resources/pooling_batch/exceptions.py @@ -1,16 +1,16 @@ from typing import TYPE_CHECKING, Optional -from golem_core.core.exceptions import BaseCoreException +from golem.resources.exceptions import ResourceException if TYPE_CHECKING: - from golem_core.core.activity_api.resources import PoolingBatch + from golem.resources.pooling_batch.pooling_batch import PoolingBatch -class BaseActivityApiException(BaseCoreException): +class PoolingBatchException(ResourceException): pass -class BatchError(BaseActivityApiException): +class BatchError(PoolingBatchException): """Unspecified exception related to the execution of a batch.""" def __init__(self, batch: "PoolingBatch", msg: Optional[str] = None): @@ -52,7 +52,7 @@ def __init__(self, batch: "PoolingBatch"): super().__init__(batch, msg) -class BatchTimeoutError(BaseActivityApiException): +class BatchTimeoutError(PoolingBatchException): """Raised in :any:`PoolingBatch.wait()` when the batch execution timed out.""" def __init__(self, batch: "PoolingBatch", timeout: float): diff --git a/golem_core/core/activity_api/resources/pooling_batch.py b/golem/resources/pooling_batch/pooling_batch.py similarity index 92% rename from golem_core/core/activity_api/resources/pooling_batch.py rename to golem/resources/pooling_batch/pooling_batch.py index 77b205a3..3d598611 100644 --- a/golem_core/core/activity_api/resources/pooling_batch.py +++ b/golem/resources/pooling_batch/pooling_batch.py @@ -4,18 +4,18 @@ from ya_activity import models -from golem_core.core.activity_api.events import BatchFinished, NewPoolingBatch -from golem_core.core.activity_api.exceptions import ( +from golem.resources.pooling_batch.events import NewPoolingBatch, BatchFinished +from golem.resources.pooling_batch.exceptions import ( BatchError, BatchTimeoutError, CommandCancelled, CommandFailed, ) -from golem_core.core.resources import _NULL, ActivityApi, Resource, YagnaEventCollector +from golem.resources.resources import _NULL, ActivityApi, Resource, YagnaEventCollector if TYPE_CHECKING: - from golem_core.core.activity_api.resources.activity import Activity # noqa - from golem_core.core.golem_node import GolemNode + from golem.resources.activity.activity import Activity # noqa + from golem.resources.golem_node import GolemNode class PoolingBatch( @@ -110,7 +110,7 @@ async def _collect_yagna_events(self) -> None: try: await super()._collect_yagna_events() except Exception: - # This happens when activity_api is destroyed when we're waiting for batch results + # This happens when activity is destroyed when we're waiting for batch results # (I'm not sure if always - for sure when provider destroys activity because # agreement timed out). Maybe some other scenarios are also possible. await self._set_finished() diff --git a/golem/resources/proposal/__init__.py b/golem/resources/proposal/__init__.py new file mode 100644 index 00000000..220556ff --- /dev/null +++ b/golem/resources/proposal/__init__.py @@ -0,0 +1,13 @@ +from golem.resources.proposal.events import NewProposal, ProposalDataChanged, ProposalClosed +from golem.resources.proposal.pipeline import default_negotiate, default_create_agreement +from golem.resources.proposal.proposal import Proposal, ProposalData + +__all__ = ( + Proposal, + ProposalData, + NewProposal, + ProposalDataChanged, + ProposalClosed, + default_negotiate, + default_create_agreement, +) diff --git a/golem/resources/proposal/events.py b/golem/resources/proposal/events.py new file mode 100644 index 00000000..686e2fe6 --- /dev/null +++ b/golem/resources/proposal/events.py @@ -0,0 +1,13 @@ +from golem.resources.resources import NewResource, ResourceClosed, ResourceDataChanged + + +class NewProposal(NewResource["Proposal"]): + pass + + +class ProposalDataChanged(ResourceDataChanged["Proposal"]): + pass + + +class ProposalClosed(ResourceClosed["Proposal"]): + pass diff --git a/golem_core/core/market_api/pipeline.py b/golem/resources/proposal/pipeline.py similarity index 70% rename from golem_core/core/market_api/pipeline.py rename to golem/resources/proposal/pipeline.py index c27f9488..e6f525a0 100644 --- a/golem_core/core/market_api/pipeline.py +++ b/golem/resources/proposal/pipeline.py @@ -1,5 +1,4 @@ -from golem_core.core.activity_api.resources import Activity -from golem_core.core.market_api.resources import Agreement, Proposal +from golem.resources.market.resources import Agreement, Proposal async def default_negotiate(proposal: Proposal) -> Proposal: @@ -21,6 +20,3 @@ async def default_create_agreement(proposal: Proposal) -> Agreement: raise Exception(f"Agreement {agreement} created from {proposal} was not approved") -async def default_create_activity(agreement: Agreement) -> Activity: - """Create a new :any:`Activity` for a given :any:`Agreement`.""" - return await agreement.create_activity() diff --git a/golem_core/core/market_api/resources/proposal.py b/golem/resources/proposal/proposal.py similarity index 92% rename from golem_core/core/market_api/resources/proposal.py rename to golem/resources/proposal/proposal.py index 1eeb5cd5..5dab54e6 100644 --- a/golem_core/core/market_api/resources/proposal.py +++ b/golem/resources/proposal/proposal.py @@ -6,16 +6,16 @@ from ya_market import RequestorApi from ya_market import models as models -from golem_core.core.market_api.events import NewProposal -from golem_core.core.market_api.resources.agreement import Agreement -from golem_core.core.props_cons.constraints import Constraints -from golem_core.core.props_cons.properties import Properties -from golem_core.core.resources import Resource -from golem_core.core.resources.base import TModel, api_call_wrapper +from golem.resources.market.events import NewProposal +from golem.resources.agreement.agreement import Agreement +from golem.payload.constraints import Constraints +from golem.payload.properties import Properties +from golem.resources.resources import Resource +from golem.resources.resources.base import TModel, api_call_wrapper if TYPE_CHECKING: - from golem_core.core.golem_node import GolemNode - from golem_core.core.market_api.resources.demand import Demand + from golem.resources.golem_node import GolemNode + from golem.resources.market.resources.demand import Demand ProposalState = Literal["Initial", "Draft", "Rejected", "Accepted", "Expired"] @@ -91,7 +91,7 @@ def demand(self) -> "Demand": # and then _demand is always set, or a Proposal-parent or a Demand-parent. # FIXME: remove local import - from golem_core.core.market_api.resources.demand import Demand + from golem.resources.market.resources.demand import Demand if self._demand is not None: return self._demand diff --git a/golem_core/core/props_cons/parsers/__init__.py b/golem/utils/__init__.py similarity index 100% rename from golem_core/core/props_cons/parsers/__init__.py rename to golem/utils/__init__.py diff --git a/golem_core/utils/asyncio.py b/golem/utils/asyncio.py similarity index 100% rename from golem_core/utils/asyncio.py rename to golem/utils/asyncio.py diff --git a/golem_core/utils/http.py b/golem/utils/http.py similarity index 100% rename from golem_core/utils/http.py rename to golem/utils/http.py diff --git a/golem_core/utils/logging.py b/golem/utils/logging.py similarity index 91% rename from golem_core/utils/logging.py rename to golem/utils/logging.py index 7a838219..2fbc1e5e 100644 --- a/golem_core/utils/logging.py +++ b/golem/utils/logging.py @@ -3,8 +3,7 @@ from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: - from golem_core.core.events import Event - + from golem.event_bus import Event DEFAULT_LOGGING = { "version": 1, @@ -25,16 +24,16 @@ "asyncio": { "level": "DEBUG", }, - "golem_core": { + "golem": { "level": "INFO", }, - "golem_core.managers": { + "golem.managers": { "level": "DEBUG", }, - "golem_core.managers.negotiation": { + "golem.managers.negotiation": { "level": "INFO", }, - "golem_core.managers.proposal": { + "golem.managers.proposal": { "level": "INFO", }, }, @@ -85,7 +84,7 @@ def logger(self) -> logging.Logger: return self._logger def _prepare_logger(self) -> logging.Logger: - logger = logging.getLogger("golem_core") + logger = logging.getLogger("golem") logger.setLevel(logging.DEBUG) format_ = "[%(asctime)s %(levelname)s %(name)s] %(message)s" diff --git a/golem/utils/storage/__init__.py b/golem/utils/storage/__init__.py new file mode 100644 index 00000000..4d217c90 --- /dev/null +++ b/golem/utils/storage/__init__.py @@ -0,0 +1,7 @@ +from golem.utils.storage.gftp import GftpProvider + +__all__ = ( + "Destination", + "Source", + "GftpProvider", +) diff --git a/golem_core/utils/storage/common.py b/golem/utils/storage/base.py similarity index 100% rename from golem_core/utils/storage/common.py rename to golem/utils/storage/base.py diff --git a/golem_core/core/props_cons/parsers/textx/__init__.py b/golem/utils/storage/gftp/__init__.py similarity index 100% rename from golem_core/core/props_cons/parsers/textx/__init__.py rename to golem/utils/storage/gftp/__init__.py diff --git a/golem_core/utils/storage/gftp.py b/golem/utils/storage/gftp/provider.py similarity index 99% rename from golem_core/utils/storage/gftp.py rename to golem/utils/storage/gftp/provider.py index 48f676a7..dfe041c1 100644 --- a/golem_core/utils/storage/gftp.py +++ b/golem/utils/storage/gftp/provider.py @@ -29,8 +29,8 @@ from async_exit_stack import AsyncExitStack from typing_extensions import Literal, Protocol, TypedDict -from golem_core.utils.storage.common import Content, Destination, Source, StorageProvider -from golem_core.utils.storage.utils import strtobool +from golem.storage.base import Content, Destination, Source, StorageProvider +from golem.storage.utils import strtobool _logger = logging.getLogger(__name__) diff --git a/golem_core/utils/storage/utils.py b/golem/utils/storage/utils.py similarity index 100% rename from golem_core/utils/storage/utils.py rename to golem/utils/storage/utils.py diff --git a/golem_core/utils/typing.py b/golem/utils/typing.py similarity index 100% rename from golem_core/utils/typing.py rename to golem/utils/typing.py diff --git a/golem_core/cli/__init__.py b/golem_core/cli/__init__.py deleted file mode 100644 index b521017f..00000000 --- a/golem_core/cli/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from golem_core.cli.cli import cli - -__all__ = ("cli",) diff --git a/golem_core/core/activity_api/__init__.py b/golem_core/core/activity_api/__init__.py deleted file mode 100644 index 334644b3..00000000 --- a/golem_core/core/activity_api/__init__.py +++ /dev/null @@ -1,38 +0,0 @@ -from golem_core.core.activity_api.commands import ( - Command, - Deploy, - DownloadFile, - Run, - Script, - SendFile, - Start, -) -from golem_core.core.activity_api.events import BatchFinished -from golem_core.core.activity_api.exceptions import ( - BaseActivityApiException, - BatchError, - BatchTimeoutError, - CommandCancelled, - CommandFailed, -) -from golem_core.core.activity_api.pipeline import default_prepare_activity -from golem_core.core.activity_api.resources import Activity, PoolingBatch - -__all__ = ( - "Activity", - "PoolingBatch", - "BaseActivityApiException", - "BatchError", - "CommandFailed", - "CommandCancelled", - "BatchTimeoutError", - "BatchFinished", - "Command", - "Script", - "Deploy", - "Start", - "Run", - "SendFile", - "DownloadFile", - "default_prepare_activity", -) diff --git a/golem_core/core/activity_api/events.py b/golem_core/core/activity_api/events.py deleted file mode 100644 index 1d0920c2..00000000 --- a/golem_core/core/activity_api/events.py +++ /dev/null @@ -1,29 +0,0 @@ -from golem_core.core.resources import ( - NewResource, - ResourceClosed, - ResourceDataChanged, - ResourceEvent, -) - - -class NewActivity(NewResource["Activity"]): - pass - - -class ActivityDataChanged(ResourceDataChanged["Activity"]): - pass - - -class ActivityClosed(ResourceClosed["Activity"]): - pass - - -class NewPoolingBatch(NewResource["PoolingBatch"]): - pass - - -class BatchFinished(ResourceEvent["PoolingBatch"]): - """Emitted when the execution of a :any:`PoolingBatch` finishes. - - The same event is emitted for successful and failed batches. - """ diff --git a/golem_core/core/activity_api/resources/__init__.py b/golem_core/core/activity_api/resources/__init__.py deleted file mode 100644 index b0c0208a..00000000 --- a/golem_core/core/activity_api/resources/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from golem_core.core.activity_api.resources.activity import Activity -from golem_core.core.activity_api.resources.pooling_batch import PoolingBatch - -__all__ = ( - "Activity", - "PoolingBatch", -) diff --git a/golem_core/core/events/__init__.py b/golem_core/core/events/__init__.py deleted file mode 100644 index 0ea35171..00000000 --- a/golem_core/core/events/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from golem_core.core.events.base import Event, EventBus, TEvent -from golem_core.core.events.event_bus import InMemoryEventBus - -__all__ = ( - "Event", - "TEvent", - "EventBus", - "InMemoryEventBus", -) diff --git a/golem_core/core/exceptions.py b/golem_core/core/exceptions.py deleted file mode 100644 index f419cea1..00000000 --- a/golem_core/core/exceptions.py +++ /dev/null @@ -1,5 +0,0 @@ -from golem_core.exceptions import BaseGolemException - - -class BaseCoreException(BaseGolemException): - pass diff --git a/golem_core/core/market_api/__init__.py b/golem_core/core/market_api/__init__.py deleted file mode 100644 index 9a225d35..00000000 --- a/golem_core/core/market_api/__init__.py +++ /dev/null @@ -1,68 +0,0 @@ -from golem_core.core.market_api.events import AgreementClosed, AgreementDataChanged, NewAgreement -from golem_core.core.market_api.exceptions import BaseMarketApiException -from golem_core.core.market_api.pipeline import ( - default_create_activity, - default_create_agreement, - default_negotiate, -) -from golem_core.core.market_api.resources import ( - INF_CPU_THREADS, - INF_MEM, - INF_STORAGE, - RUNTIME_CAPABILITIES, - RUNTIME_NAME, - ActivityInfo, - Agreement, - BaseDemandOfferBaseException, - ConstraintException, - Demand, - DemandBuilder, - DemandOfferBaseModel, - InvalidPropertiesError, - ManifestVmPayload, - NodeInfo, - Payload, - Proposal, - RepositoryVmPayload, - TDemandOfferBaseModel, - VmPayloadException, - constraint, - prop, -) - -__all__ = ( - "default_negotiate", - "default_create_agreement", - "default_create_activity", - "Agreement", - "Proposal", - "Demand", - "Payload", - "ManifestVmPayload", - "VmPayloadException", - "RepositoryVmPayload", - "DemandBuilder", - "Payload", - "ManifestVmPayload", - "VmPayloadException", - "RepositoryVmPayload", - "RUNTIME_NAME", - "RUNTIME_CAPABILITIES", - "INF_CPU_THREADS", - "INF_MEM", - "INF_STORAGE", - "NodeInfo", - "ActivityInfo", - "TDemandOfferBaseModel", - "DemandOfferBaseModel", - "constraint", - "prop", - "BaseDemandOfferBaseException", - "ConstraintException", - "InvalidPropertiesError", - "BaseMarketApiException", - "NewAgreement", - "AgreementDataChanged", - "AgreementClosed", - "PaymentInfo", -) diff --git a/golem_core/core/market_api/events.py b/golem_core/core/market_api/events.py deleted file mode 100644 index eeaa1286..00000000 --- a/golem_core/core/market_api/events.py +++ /dev/null @@ -1,37 +0,0 @@ -from golem_core.core.resources import NewResource, ResourceClosed, ResourceDataChanged - - -class NewDemand(NewResource["Demand"]): - pass - - -class DemandDataChanged(ResourceDataChanged["Demand"]): - pass - - -class DemandClosed(ResourceClosed["Demand"]): - pass - - -class NewAgreement(NewResource["Agreement"]): - pass - - -class AgreementDataChanged(ResourceDataChanged["Agreement"]): - pass - - -class AgreementClosed(ResourceClosed["Agreement"]): - pass - - -class NewProposal(NewResource["Proposal"]): - pass - - -class ProposalDataChanged(ResourceDataChanged["Proposal"]): - pass - - -class ProposalClosed(ResourceClosed["Proposal"]): - pass diff --git a/golem_core/core/market_api/exceptions.py b/golem_core/core/market_api/exceptions.py deleted file mode 100644 index 338bb034..00000000 --- a/golem_core/core/market_api/exceptions.py +++ /dev/null @@ -1,5 +0,0 @@ -from golem_core.core.exceptions import BaseCoreException - - -class BaseMarketApiException(BaseCoreException): - pass diff --git a/golem_core/core/market_api/resources/__init__.py b/golem_core/core/market_api/resources/__init__.py deleted file mode 100644 index 7c536e65..00000000 --- a/golem_core/core/market_api/resources/__init__.py +++ /dev/null @@ -1,55 +0,0 @@ -from golem_core.core.market_api.resources.agreement import Agreement -from golem_core.core.market_api.resources.demand import ( - INF_CPU_THREADS, - INF_MEM, - INF_STORAGE, - RUNTIME_CAPABILITIES, - RUNTIME_NAME, - ActivityInfo, - BaseDemandOfferBaseException, - ConstraintException, - Demand, - DemandBuilder, - DemandOfferBaseModel, - InvalidPropertiesError, - ManifestVmPayload, - NodeInfo, - Payload, - PaymentInfo, - RepositoryVmPayload, - TDemandOfferBaseModel, - VmPayloadException, - constraint, - prop, -) -from golem_core.core.market_api.resources.proposal import Proposal - -__all__ = ( - "Agreement", - "Proposal", - "Demand", - "Payload", - "ManifestVmPayload", - "VmPayloadException", - "RepositoryVmPayload", - "DemandBuilder", - "Payload", - "ManifestVmPayload", - "VmPayloadException", - "RepositoryVmPayload", - "RUNTIME_NAME", - "RUNTIME_CAPABILITIES", - "INF_CPU_THREADS", - "INF_MEM", - "INF_STORAGE", - "NodeInfo", - "ActivityInfo", - "TDemandOfferBaseModel", - "DemandOfferBaseModel", - "constraint", - "prop", - "BaseDemandOfferBaseException", - "ConstraintException", - "InvalidPropertiesError", - "PaymentInfo", -) diff --git a/golem_core/core/market_api/resources/demand/__init__.py b/golem_core/core/market_api/resources/demand/__init__.py deleted file mode 100644 index 5108f598..00000000 --- a/golem_core/core/market_api/resources/demand/__init__.py +++ /dev/null @@ -1,51 +0,0 @@ -from golem_core.core.market_api.resources.demand.demand import Demand -from golem_core.core.market_api.resources.demand.demand_builder import DemandBuilder -from golem_core.core.market_api.resources.demand.demand_offer_base import ( - INF_CPU_THREADS, - INF_MEM, - INF_STORAGE, - RUNTIME_CAPABILITIES, - RUNTIME_NAME, - ActivityInfo, - BaseDemandOfferBaseException, - ConstraintException, - DemandOfferBaseModel, - InvalidPropertiesError, - ManifestVmPayload, - NodeInfo, - Payload, - PaymentInfo, - RepositoryVmPayload, - TDemandOfferBaseModel, - VmPayloadException, - constraint, - prop, -) - -__all__ = ( - "Demand", - "Payload", - "ManifestVmPayload", - "VmPayloadException", - "RepositoryVmPayload", - "DemandBuilder", - "Payload", - "ManifestVmPayload", - "VmPayloadException", - "RepositoryVmPayload", - "RUNTIME_NAME", - "RUNTIME_CAPABILITIES", - "INF_CPU_THREADS", - "INF_MEM", - "INF_STORAGE", - "NodeInfo", - "ActivityInfo", - "TDemandOfferBaseModel", - "DemandOfferBaseModel", - "constraint", - "prop", - "BaseDemandOfferBaseException", - "ConstraintException", - "InvalidPropertiesError", - "PaymentInfo", -) diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/__init__.py b/golem_core/core/market_api/resources/demand/demand_offer_base/__init__.py deleted file mode 100644 index 1f321ed7..00000000 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/__init__.py +++ /dev/null @@ -1,49 +0,0 @@ -from golem_core.core.market_api.resources.demand.demand_offer_base.defaults import ( - INF_CPU_THREADS, - INF_MEM, - INF_STORAGE, - RUNTIME_CAPABILITIES, - RUNTIME_NAME, - ActivityInfo, - NodeInfo, - PaymentInfo, -) -from golem_core.core.market_api.resources.demand.demand_offer_base.exceptions import ( - BaseDemandOfferBaseException, - ConstraintException, - InvalidPropertiesError, -) -from golem_core.core.market_api.resources.demand.demand_offer_base.model import ( - DemandOfferBaseModel, - TDemandOfferBaseModel, - constraint, - prop, -) -from golem_core.core.market_api.resources.demand.demand_offer_base.payload import ( - ManifestVmPayload, - Payload, - RepositoryVmPayload, - VmPayloadException, -) - -__all__ = ( - "Payload", - "ManifestVmPayload", - "VmPayloadException", - "RepositoryVmPayload", - "RUNTIME_NAME", - "RUNTIME_CAPABILITIES", - "INF_CPU_THREADS", - "INF_MEM", - "INF_STORAGE", - "NodeInfo", - "ActivityInfo", - "TDemandOfferBaseModel", - "DemandOfferBaseModel", - "constraint", - "prop", - "BaseDemandOfferBaseException", - "ConstraintException", - "InvalidPropertiesError", - "PaymentInfo", -) diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/exceptions.py b/golem_core/core/market_api/resources/demand/demand_offer_base/exceptions.py deleted file mode 100644 index 29e39700..00000000 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/exceptions.py +++ /dev/null @@ -1,13 +0,0 @@ -from golem_core.core.market_api.exceptions import BaseMarketApiException - - -class BaseDemandOfferBaseException(BaseMarketApiException): - pass - - -class ConstraintException(BaseDemandOfferBaseException): - pass - - -class InvalidPropertiesError(BaseDemandOfferBaseException): - """`properties` given to `DemandOfferBaseModel.from_properties(cls, properties)` are invalid.""" diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/payload/__init__.py b/golem_core/core/market_api/resources/demand/demand_offer_base/payload/__init__.py deleted file mode 100644 index dde9f292..00000000 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/payload/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from golem_core.core.market_api.resources.demand.demand_offer_base.payload.base import Payload -from golem_core.core.market_api.resources.demand.demand_offer_base.payload.vm import ( - ManifestVmPayload, - RepositoryVmPayload, - VmPayloadException, -) - -__all__ = ( - "Payload", - "ManifestVmPayload", - "VmPayloadException", - "RepositoryVmPayload", -) diff --git a/golem_core/core/market_api/resources/demand/demand_offer_base/payload/base.py b/golem_core/core/market_api/resources/demand/demand_offer_base/payload/base.py deleted file mode 100644 index 8c3fcced..00000000 --- a/golem_core/core/market_api/resources/demand/demand_offer_base/payload/base.py +++ /dev/null @@ -1,39 +0,0 @@ -from abc import ABC - -from golem_core.core.market_api.resources.demand.demand_offer_base.model import DemandOfferBaseModel - - -class Payload(DemandOfferBaseModel, ABC): - r"""Base class for descriptions of the payload required by the requestor. - - example usage:: - - import asyncio - - from dataclasses import dataclass - from golem_core.core.market_api import DemandBuilder, prop, constraint, Payload, RUNTIME_NAME, INF_MEM, INF_STORAGE - - CUSTOM_RUNTIME_NAME = "my-runtime" - CUSTOM_PROPERTY = "golem.srv.app.myprop" - - - @dataclass - class MyPayload(Payload): - myprop: str = prop(CUSTOM_PROPERTY, default="myvalue") - runtime: str = constraint(RUNTIME_NAME, default=CUSTOM_RUNTIME_NAME) - min_mem_gib: float = constraint(INF_MEM, ">=", default=16) - min_storage_gib: float = constraint(INF_STORAGE, ">=", default=1024) - - - async def main(): - builder = DemandBuilder() - payload = MyPayload(myprop="othervalue", min_mem_gib=32) - await builder.add(payload) - print(builder) - - asyncio.run(main()) - - output:: - - {'properties': {'golem.srv.app.myprop': 'othervalue'}, 'constraints': ['(&(golem.runtime.name=my-runtime)\n\t(golem.inf.mem.gib>=32)\n\t(golem.inf.storage.gib>=1024))']} - """ # noqa: E501 diff --git a/golem_core/core/network_api/__init__.py b/golem_core/core/network_api/__init__.py deleted file mode 100644 index b94f3566..00000000 --- a/golem_core/core/network_api/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from golem_core.core.network_api.exceptions import BaseNetworkApiException, NetworkFull -from golem_core.core.network_api.resources import Network - -__all__ = ( - "Network", - "BaseNetworkApiException", - "NetworkFull", -) diff --git a/golem_core/core/network_api/resources/__init__.py b/golem_core/core/network_api/resources/__init__.py deleted file mode 100644 index ece14223..00000000 --- a/golem_core/core/network_api/resources/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from golem_core.core.network_api.resources.network import Network - -__all__ = ("Network",) diff --git a/golem_core/core/payment_api/__init__.py b/golem_core/core/payment_api/__init__.py deleted file mode 100644 index 54651731..00000000 --- a/golem_core/core/payment_api/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -from golem_core.core.payment_api.events import DebitNoteClosed, DebitNoteDataChanged, NewDebitNote -from golem_core.core.payment_api.exceptions import BasePaymentApiException, NoMatchingAccount -from golem_core.core.payment_api.resources import ( - Allocation, - DebitNote, - DebitNoteEventCollector, - Invoice, - InvoiceEventCollector, -) - -__all__ = ( - "Allocation", - "DebitNote", - "Invoice", - "DebitNoteEventCollector", - "InvoiceEventCollector", - "BasePaymentApiException", - "NoMatchingAccount", - "NewDebitNote", - "DebitNoteClosed", - "DebitNoteDataChanged", -) diff --git a/golem_core/core/payment_api/events.py b/golem_core/core/payment_api/events.py deleted file mode 100644 index f42a3fe1..00000000 --- a/golem_core/core/payment_api/events.py +++ /dev/null @@ -1,37 +0,0 @@ -from golem_core.core.resources import NewResource, ResourceClosed, ResourceDataChanged - - -class NewAllocation(NewResource["Allocation"]): - pass - - -class AllocationDataChanged(ResourceDataChanged["Allocation"]): - pass - - -class AllocationClosed(ResourceClosed["Allocation"]): - pass - - -class NewDebitNote(NewResource["DebitNote"]): - pass - - -class DebitNoteDataChanged(ResourceDataChanged["DebitNote"]): - pass - - -class DebitNoteClosed(ResourceClosed["DebitNote"]): - pass - - -class NewInvoice(NewResource["Invoice"]): - pass - - -class InvoiceDataChanged(ResourceDataChanged["Invoice"]): - pass - - -class InvoiceClosed(ResourceClosed["Invoice"]): - pass diff --git a/golem_core/core/payment_api/resources/__init__.py b/golem_core/core/payment_api/resources/__init__.py deleted file mode 100644 index 66dc5990..00000000 --- a/golem_core/core/payment_api/resources/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -from golem_core.core.payment_api.resources.allocation import Allocation -from golem_core.core.payment_api.resources.debit_note import DebitNote -from golem_core.core.payment_api.resources.event_collectors import ( - DebitNoteEventCollector, - InvoiceEventCollector, - PaymentEventCollector, -) -from golem_core.core.payment_api.resources.invoice import Invoice - -__all__ = ( - "Allocation", - "DebitNote", - "Invoice", - "InvoiceEventCollector", - "DebitNoteEventCollector", - "PaymentEventCollector", -) diff --git a/golem_core/core/payment_api/resources/event_collectors.py b/golem_core/core/payment_api/resources/event_collectors.py deleted file mode 100644 index e9ac2912..00000000 --- a/golem_core/core/payment_api/resources/event_collectors.py +++ /dev/null @@ -1,78 +0,0 @@ -from abc import ABC, abstractmethod -from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Callable, Dict, Tuple, Union - -from ya_payment import models - -from golem_core.core.payment_api.resources.debit_note import DebitNote -from golem_core.core.payment_api.resources.invoice import Invoice -from golem_core.core.resources import Resource, YagnaEventCollector - -if TYPE_CHECKING: - from golem_core.core.activity_api import Activity - from golem_core.core.golem_node.golem_node import GolemNode - from golem_core.core.market_api import Agreement - -InvoiceEvent = Union[ - models.InvoiceReceivedEvent, - models.InvoiceAcceptedEvent, - models.InvoiceReceivedEvent, - models.InvoiceFailedEvent, - models.InvoiceSettledEvent, - models.InvoiceCancelledEvent, -] - -DebitNoteEvent = Union[ - models.DebitNoteReceivedEvent, - models.DebitNoteAcceptedEvent, - models.DebitNoteReceivedEvent, - models.DebitNoteFailedEvent, - models.DebitNoteSettledEvent, - models.DebitNoteCancelledEvent, -] - - -class PaymentEventCollector(YagnaEventCollector, ABC): - def __init__(self, node: "GolemNode"): - self.node = node - self.min_ts = datetime.now(timezone.utc) - - def _collect_events_kwargs(self) -> Dict: - return {"after_timestamp": self.min_ts, "app_session_id": self.node.app_session_id} - - async def _process_event(self, event: Union[InvoiceEvent, DebitNoteEvent]) -> None: - self.min_ts = max(event.event_date, self.min_ts) - resource, parent_resource = await self._get_event_resources(event) - resource.add_event(event) - if resource._parent is None: - parent_resource.add_child(resource) - - @abstractmethod - async def _get_event_resources(self, event: Any) -> Tuple[Resource, Resource]: - raise NotImplementedError - - -class DebitNoteEventCollector(PaymentEventCollector): - @property - def _collect_events_func(self) -> Callable: - return DebitNote._get_api(self.node).get_debit_note_events - - async def _get_event_resources(self, event: DebitNoteEvent) -> Tuple[DebitNote, "Activity"]: - assert event.debit_note_id is not None - debit_note = self.node.debit_note(event.debit_note_id) - await debit_note.get_data() - activity = self.node.activity(debit_note.data.activity_id) - return debit_note, activity - - -class InvoiceEventCollector(PaymentEventCollector): - @property - def _collect_events_func(self) -> Callable: - return Invoice._get_api(self.node).get_invoice_events - - async def _get_event_resources(self, event: InvoiceEvent) -> Tuple[Invoice, "Agreement"]: - assert event.invoice_id is not None - invoice = self.node.invoice(event.invoice_id) - await invoice.get_data() - agreement = self.node.agreement(invoice.data.agreement_id) - return invoice, agreement diff --git a/golem_core/core/props_cons/parsers/base.py b/golem_core/core/props_cons/parsers/base.py deleted file mode 100644 index 9f0667f6..00000000 --- a/golem_core/core/props_cons/parsers/base.py +++ /dev/null @@ -1,13 +0,0 @@ -from abc import ABC, abstractmethod - -from golem_core.core.props_cons.constraints import Constraints - - -class SyntaxException(Exception): - pass - - -class DemandOfferSyntaxParser(ABC): - @abstractmethod - def parse(self, syntax: str) -> Constraints: - ... diff --git a/golem_core/core/resources/__init__.py b/golem_core/core/resources/__init__.py deleted file mode 100644 index 2ea9cb03..00000000 --- a/golem_core/core/resources/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -from golem_core.core.resources.base import _NULL, Resource, TResource, api_call_wrapper -from golem_core.core.resources.event_collectors import YagnaEventCollector -from golem_core.core.resources.events import ( - NewResource, - ResourceClosed, - ResourceDataChanged, - ResourceEvent, - TResourceEvent, -) -from golem_core.core.resources.exceptions import ( - BaseResourceException, - MissingConfiguration, - ResourceNotFound, -) -from golem_core.core.resources.low import ActivityApi, ApiConfig, ApiFactory - -__all__ = ( - "Resource", - "api_call_wrapper", - "_NULL", - "TResource", - "ResourceEvent", - "NewResource", - "ResourceDataChanged", - "ResourceClosed", - "TResourceEvent", - "ApiConfig", - "ApiFactory", - "YagnaEventCollector", - "ActivityApi", - "ResourceNotFound", - "BaseResourceException", - "MissingConfiguration", -) diff --git a/golem_core/core/resources/event_collectors/__init__.py b/golem_core/core/resources/event_collectors/__init__.py deleted file mode 100644 index a5afff8c..00000000 --- a/golem_core/core/resources/event_collectors/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from golem_core.core.resources.event_collectors.base import YagnaEventCollector -from golem_core.core.resources.event_collectors.utils import ( - is_gsb_endpoint_not_found_error, - is_intermittent_error, -) - -__all__ = ( - "YagnaEventCollector", - "is_gsb_endpoint_not_found_error", - "is_intermittent_error", -) diff --git a/golem_core/exceptions.py b/golem_core/exceptions.py deleted file mode 100644 index 9d13ae46..00000000 --- a/golem_core/exceptions.py +++ /dev/null @@ -1,2 +0,0 @@ -class BaseGolemException(Exception): - pass diff --git a/golem_core/managers/__init__.py b/golem_core/managers/__init__.py deleted file mode 100644 index 3d00b277..00000000 --- a/golem_core/managers/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from golem_core.managers.payment import DefaultPaymentManager - -__all__ = ("DefaultPaymentManager",) diff --git a/golem_core/managers/agreement/__init__.py b/golem_core/managers/agreement/__init__.py deleted file mode 100644 index 887ba0b9..00000000 --- a/golem_core/managers/agreement/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from golem_core.managers.agreement.events import AgreementReleased -from golem_core.managers.agreement.single_use import SingleUseAgreementManager - -__all__ = ( - "AgreementReleased", - "SingleUseAgreementManager", -) diff --git a/golem_core/managers/negotiation/__init__.py b/golem_core/managers/negotiation/__init__.py deleted file mode 100644 index 55ec2fd6..00000000 --- a/golem_core/managers/negotiation/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from golem_core.managers.negotiation.sequential import SequentialNegotiationManager - -__all__ = ("SequentialNegotiationManager",) diff --git a/golem_core/managers/network/__init__.py b/golem_core/managers/network/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/golem_core/managers/payment/__init__.py b/golem_core/managers/payment/__init__.py deleted file mode 100644 index f1b02b7a..00000000 --- a/golem_core/managers/payment/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from golem_core.managers.payment.default import DefaultPaymentManager - -__all__ = ("DefaultPaymentManager",) diff --git a/golem_core/managers/proposal/__init__.py b/golem_core/managers/proposal/__init__.py deleted file mode 100644 index 9a163edc..00000000 --- a/golem_core/managers/proposal/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from golem_core.managers.proposal.stack import StackProposalManager - -__all__ = ("StackProposalManager",) diff --git a/golem_core/pipeline/__init__.py b/golem_core/pipeline/__init__.py deleted file mode 100644 index 7ab785de..00000000 --- a/golem_core/pipeline/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from golem_core.pipeline.buffer import Buffer -from golem_core.pipeline.chain import Chain -from golem_core.pipeline.limit import Limit -from golem_core.pipeline.map import Map -from golem_core.pipeline.sort import Sort -from golem_core.pipeline.zip import Zip - -__all__ = ( - "Chain", - "Limit", - "Map", - "Zip", - "Buffer", - "Sort", -) diff --git a/golem_core/utils/__init__.py b/golem_core/utils/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/golem_core/utils/storage/__init__.py b/golem_core/utils/storage/__init__.py deleted file mode 100644 index 4f24a070..00000000 --- a/golem_core/utils/storage/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from golem_core.utils.storage.common import Destination, Source -from golem_core.utils.storage.gftp import GftpProvider - -__all__ = ( - "Destination", - "Source", - "GftpProvider", -) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 9da9bb29..de6cd04e 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -1,15 +1,15 @@ from contextlib import asynccontextmanager from typing import AsyncGenerator, Optional -from golem_core.core.activity_api import Activity -from golem_core.core.golem_node import GolemNode -from golem_core.core.market_api import RepositoryVmPayload -from golem_core.core.market_api.pipeline import ( +from golem.resources.activity import Activity +from golem.resources.golem_node import GolemNode +from golem.resources.market import RepositoryVmPayload +from golem.resources.market.pipeline import ( default_create_activity, default_create_agreement, default_negotiate, ) -from golem_core.pipeline import Chain, Map +from golem.pipeline import Chain, Map ANY_PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/tests/integration/test_1.py b/tests/integration/test_1.py index 0e00deb1..2b42a570 100644 --- a/tests/integration/test_1.py +++ b/tests/integration/test_1.py @@ -5,10 +5,10 @@ import pytest import pytest_asyncio -from golem_core.core.golem_node import GolemNode -from golem_core.core.market_api import RepositoryVmPayload -from golem_core.core.payment_api import NoMatchingAccount -from golem_core.core.resources import ResourceNotFound +from golem.resources.golem_node import GolemNode +from golem.resources.market import RepositoryVmPayload +from golem.resources.payment import NoMatchingAccount +from golem.resources.resources import ResourceNotFound PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/tests/integration/test_app_session_id.py b/tests/integration/test_app_session_id.py index a91a5b27..e64a0dc6 100644 --- a/tests/integration/test_app_session_id.py +++ b/tests/integration/test_app_session_id.py @@ -2,9 +2,9 @@ import pytest -from golem_core.core.golem_node import GolemNode -from golem_core.core.payment_api import DebitNote, Invoice -from golem_core.core.resources import ResourceEvent +from golem.resources.golem_node import GolemNode +from golem.resources.payment import DebitNote, Invoice +from golem.resources.resources import ResourceEvent from .helpers import get_activity diff --git a/tests/unit/test_app_session_id.py b/tests/unit/test_app_session_id.py index 1331cf8f..ffeff20c 100644 --- a/tests/unit/test_app_session_id.py +++ b/tests/unit/test_app_session_id.py @@ -1,4 +1,4 @@ -from golem_core.core.golem_node import GolemNode +from golem.resources.golem_node import GolemNode def test_different_app_session_id() -> None: diff --git a/tests/unit/test_command.py b/tests/unit/test_command.py index 138bc253..109c55a4 100644 --- a/tests/unit/test_command.py +++ b/tests/unit/test_command.py @@ -2,7 +2,7 @@ import pytest -from golem_core.core.activity_api import Run +from golem.resources.activity import Run list_c = ["echo", "foo"] str_c = "echo foo" diff --git a/tests/unit/test_demand_builder.py b/tests/unit/test_demand_builder.py index 7775f8b7..7fa01c89 100644 --- a/tests/unit/test_demand_builder.py +++ b/tests/unit/test_demand_builder.py @@ -1,8 +1,8 @@ from dataclasses import dataclass -from golem_core.core.market_api import DemandBuilder, DemandOfferBaseModel, constraint, prop -from golem_core.core.props_cons.constraints import Constraint, ConstraintGroup, Constraints -from golem_core.core.props_cons.properties import Properties +from golem.resources.market import DemandBuilder, DemandOfferBaseModel, constraint, prop +from golem.payload.constraints import Constraint, Constraints, ConstraintGroup +from golem.payload.properties import Properties @dataclass @@ -162,7 +162,7 @@ async def test_create_demand(mocker): mocked_node = mocker.Mock() mocked_demand = mocker.patch( - "golem_core.core.market_api.resources.demand.demand_builder.Demand", + "golem.resources.market.resources.demand.demand_builder.Demand", **{"create_from_properties_constraints": mocker.AsyncMock(return_value="foobar")}, ) diff --git a/tests/unit/test_demand_builder_model.py b/tests/unit/test_demand_builder_model.py index 4342257d..a8def04c 100644 --- a/tests/unit/test_demand_builder_model.py +++ b/tests/unit/test_demand_builder_model.py @@ -5,14 +5,14 @@ import pytest -from golem_core.core.market_api import ( +from golem.resources.market import ( DemandOfferBaseModel, InvalidPropertiesError, constraint, prop, ) -from golem_core.core.props_cons.constraints import Constraint, Constraints -from golem_core.core.props_cons.properties import Properties +from golem.payload.constraints import Constraint, Constraints +from golem.payload.properties import Properties class ExampleEnum(Enum): diff --git a/tests/unit/test_demand_offer_cons.py b/tests/unit/test_demand_offer_cons.py index a831a141..df23abae 100644 --- a/tests/unit/test_demand_offer_cons.py +++ b/tests/unit/test_demand_offer_cons.py @@ -3,7 +3,7 @@ import pytest -from golem_core.core.props_cons.constraints import ( +from golem.payload.constraints import ( Constraint, ConstraintException, ConstraintGroup, diff --git a/tests/unit/test_demand_offer_parsers.py b/tests/unit/test_demand_offer_parsers.py index 1df96be3..86c2ccd2 100644 --- a/tests/unit/test_demand_offer_parsers.py +++ b/tests/unit/test_demand_offer_parsers.py @@ -1,8 +1,8 @@ import pytest -from golem_core.core.props_cons.constraints import Constraint, ConstraintException, ConstraintGroup -from golem_core.core.props_cons.parsers.base import SyntaxException -from golem_core.core.props_cons.parsers.textx.parser import TextXDemandOfferSyntaxParser +from golem.payload.constraints import Constraint, ConstraintException, ConstraintGroup +from golem.payload.parsers import SyntaxException +from golem.payload.parsers import TextXDemandOfferSyntaxParser @pytest.fixture(scope="module") @@ -12,7 +12,7 @@ def demand_offer_parser(): def test_parse_raises_exception_on_bad_syntax(demand_offer_parser): with pytest.raises(SyntaxException): - demand_offer_parser.parse("NOT VALID SYNTAX") + demand_offer_parser.parse_constraints("NOT VALID SYNTAX") @pytest.mark.parametrize( @@ -39,7 +39,7 @@ def test_parse_raises_exception_on_bad_syntax(demand_offer_parser): ), ) def test_single_constraint(demand_offer_parser, input_string, output): - result = demand_offer_parser.parse(input_string) + result = demand_offer_parser.parse_constraints(input_string) assert result == output @@ -72,7 +72,7 @@ def test_single_constraint(demand_offer_parser, input_string, output): ), ) def test_constraint_groups(demand_offer_parser, input_string, output): - result = demand_offer_parser.parse(input_string) + result = demand_offer_parser.parse_constraints(input_string) assert result == output @@ -86,14 +86,14 @@ def test_constraint_groups(demand_offer_parser, input_string, output): ), ) def test_constraint_groups_empty(demand_offer_parser, input_string, output): - result = demand_offer_parser.parse(input_string) + result = demand_offer_parser.parse_constraints(input_string) assert result == output def test_error_not_operator_with_multiple_items(demand_offer_parser): with pytest.raises(ConstraintException): - result = demand_offer_parser.parse("(! (foo=1) (bar=1))") + result = demand_offer_parser.parse_constraints("(! (foo=1) (bar=1))") @pytest.mark.parametrize( @@ -121,6 +121,6 @@ def test_error_not_operator_with_multiple_items(demand_offer_parser): ), ) def test_constraint_groups_nested(demand_offer_parser, input_string, output): - result = demand_offer_parser.parse(input_string) + result = demand_offer_parser.parse_constraints(input_string) assert result == output diff --git a/tests/unit/test_demand_offer_props.py b/tests/unit/test_demand_offer_props.py index 827e3586..bbed20ca 100644 --- a/tests/unit/test_demand_offer_props.py +++ b/tests/unit/test_demand_offer_props.py @@ -1,7 +1,7 @@ from datetime import datetime from enum import Enum -from golem_core.core.props_cons.properties import Properties +from golem.payload.properties import Properties class ExampleEnum(Enum): diff --git a/tests/unit/test_event_bus.py b/tests/unit/test_event_bus.py index 72de8c06..b1aad0cd 100644 --- a/tests/unit/test_event_bus.py +++ b/tests/unit/test_event_bus.py @@ -3,9 +3,9 @@ import pytest -from golem_core.core.events.base import Event, EventBusError -from golem_core.core.events.event_bus import InMemoryEventBus -from golem_core.utils.logging import DEFAULT_LOGGING +from golem.resources.events.base import Event, EventBusError +from golem.event_bus.in_memory import InMemoryEventBus +from golem.utils.logging import DEFAULT_LOGGING class ExampleEvent(Event): diff --git a/tests/unit/test_mid.py b/tests/unit/test_mid.py index 7a6fbdac..c719c2c1 100644 --- a/tests/unit/test_mid.py +++ b/tests/unit/test_mid.py @@ -2,7 +2,7 @@ import pytest -from golem_core.pipeline import Buffer, Chain, Limit, Map, Zip +from golem.pipeline import Buffer, Chain, Limit, Map, Zip async def src() -> AsyncIterator[int]: diff --git a/tests/unit/utils/test_storage.py b/tests/unit/utils/test_storage.py index 06ecdc42..82f35fa5 100644 --- a/tests/unit/utils/test_storage.py +++ b/tests/unit/utils/test_storage.py @@ -7,7 +7,7 @@ import pytest -from golem_core.utils.storage import gftp +from golem.utils.storage.gftp import provider @pytest.fixture(scope="function") From 30acf9e8201ea430d6513196fbd80085f3983f85 Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 28 Jun 2023 10:31:50 +0200 Subject: [PATCH 061/123] imports reorganized --- examples/attach.py | 5 +- examples/cli_example.sh | 16 +- examples/core_example.py | 12 +- examples/detached_activity.py | 7 +- .../exception_handling/exception_handling.py | 16 +- examples/managers/basic_composition.py | 6 +- examples/managers/ssh.py | 4 +- examples/rate_providers/rate_providers.py | 14 +- .../score_based_providers.py | 14 +- examples/service.py | 13 +- .../examples/blender/blender.py | 2 +- .../examples/execute_tasks_hello_world.py | 2 +- .../examples/pipeline_example.py | 14 +- .../task_api_draft/examples/redundance.py | 2 +- examples/task_api_draft/examples/yacat.py | 19 ++- .../examples/yacat_no_business_logic.py | 4 +- .../task_api_draft/task_api/activity_pool.py | 4 +- .../task_api_draft/task_api/execute_tasks.py | 15 +- .../task_api/redundance_manager.py | 3 +- golem/cli/utils.py | 4 +- golem/event_bus/__init__.py | 3 +- golem/event_bus/base.py | 3 +- golem/event_bus/in_memory/__init__.py | 4 +- golem/event_bus/in_memory/event_bus.py | 6 +- golem/exceptions.py | 9 + golem/managers/activity/__init__.py | 5 +- golem/managers/activity/single_use.py | 10 +- golem/managers/agreement/single_use.py | 6 +- golem/managers/base.py | 17 +- golem/managers/negotiation/plugins.py | 5 +- golem/managers/negotiation/sequential.py | 13 +- golem/managers/network/single.py | 6 +- golem/managers/payment/default.py | 8 +- golem/managers/payment/pay_all.py | 17 +- golem/managers/proposal/stack.py | 4 +- golem/managers/work/sequential.py | 2 +- golem/node/__init__.py | 15 +- golem/node/event_collectors/__init__.py | 8 - golem/node/event_collectors/utils.py | 41 ----- golem/node/events.py | 2 +- golem/node/node.py | 30 ++-- golem/payload/__init__.py | 37 ++-- golem/payload/base.py | 16 +- golem/payload/constraints.py | 8 +- golem/payload/defaults.py | 11 +- golem/payload/parsers/__init__.py | 7 +- golem/payload/parsers/textx/__init__.py | 4 +- golem/payload/vm.py | 5 +- golem/pipeline/__init__.py | 2 + golem/resources/__init__.py | 161 ++++++++++++++++++ golem/resources/activity/__init__.py | 10 +- golem/resources/activity/activity.py | 11 +- golem/resources/activity/commands.py | 3 +- golem/resources/activity/events.py | 8 + golem/resources/activity/pipeline.py | 2 +- golem/resources/agreement/__init__.py | 14 +- golem/resources/agreement/agreement.py | 10 +- golem/resources/agreement/events.py | 8 + golem/resources/agreement/pipeline.py | 4 + golem/resources/allocation/__init__.py | 15 +- golem/resources/allocation/allocation.py | 12 +- golem/resources/allocation/events.py | 8 + golem/resources/base.py | 2 +- golem/resources/debit_note/__init__.py | 13 +- golem/resources/debit_note/debit_note.py | 9 +- .../resources/debit_note/event_collectors.py | 4 +- golem/resources/debit_note/events.py | 8 + golem/resources/demand/__init__.py | 15 +- golem/resources/demand/demand.py | 19 +-- golem/resources/demand/demand_builder.py | 25 +-- golem/resources/demand/events.py | 8 + golem/resources/event_collectors.py | 7 +- golem/resources/events.py | 2 +- golem/resources/exceptions.py | 15 +- golem/resources/invoice/__init__.py | 13 +- golem/resources/invoice/event_collectors.py | 6 +- golem/resources/invoice/events.py | 7 +- golem/resources/invoice/invoice.py | 9 +- golem/resources/network/__init__.py | 8 +- golem/resources/network/events.py | 4 +- golem/resources/network/network.py | 8 +- golem/resources/pooling_batch/__init__.py | 26 +-- golem/resources/pooling_batch/events.py | 8 + .../resources/pooling_batch/pooling_batch.py | 7 +- golem/resources/proposal/__init__.py | 18 +- golem/resources/proposal/events.py | 7 +- golem/resources/proposal/pipeline.py | 5 +- golem/resources/proposal/proposal.py | 16 +- golem/utils/low/__init__.py | 11 ++ golem/{node => utils/low}/api.py | 6 +- .../base.py => utils/low/event_collector.py} | 44 ++++- golem/utils/storage/__init__.py | 3 +- golem/utils/storage/gftp/__init__.py | 3 + golem/utils/storage/gftp/provider.py | 4 +- tests/integration/helpers.py | 2 +- tests/unit/test_app_session_id.py | 2 +- tests/unit/test_command.py | 2 +- tests/unit/test_demand_builder.py | 17 +- tests/unit/test_event_bus.py | 2 +- ...emand_builder_model.py => test_payload.py} | 21 +-- ...and_offer_cons.py => test_payload_cons.py} | 7 +- ...fer_parsers.py => test_payload_parsers.py} | 7 +- ...d_offer_props.py => test_payload_props.py} | 2 +- tests/unit/utils/test_storage.py | 2 +- 104 files changed, 685 insertions(+), 465 deletions(-) delete mode 100644 golem/node/event_collectors/__init__.py delete mode 100644 golem/node/event_collectors/utils.py create mode 100644 golem/utils/low/__init__.py rename golem/{node => utils/low}/api.py (97%) rename golem/{node/event_collectors/base.py => utils/low/event_collector.py} (63%) rename tests/unit/{test_demand_builder_model.py => test_payload.py} (89%) rename tests/unit/{test_demand_offer_cons.py => test_payload_cons.py} (93%) rename tests/unit/{test_demand_offer_parsers.py => test_payload_parsers.py} (94%) rename tests/unit/{test_demand_offer_props.py => test_payload_props.py} (96%) diff --git a/examples/attach.py b/examples/attach.py index a8ef627b..6ac2f18f 100644 --- a/examples/attach.py +++ b/examples/attach.py @@ -1,11 +1,10 @@ import asyncio import sys +from golem.node import GolemNode +from golem.resources import DebitNote, NewResource from golem.resources.activity import commands -from golem.resources.golem_node import GolemNode -from golem.resources.payment import DebitNote from golem.resources.debit_note.events import NewDebitNote -from golem.resources.resources import NewResource ACTIVITY_ID = sys.argv[1].strip() diff --git a/examples/cli_example.sh b/examples/cli_example.sh index 05f8dc46..9a909c34 100644 --- a/examples/cli_example.sh +++ b/examples/cli_example.sh @@ -1,25 +1,25 @@ #!/usr/bin/env bash printf "*** STATUS ***\n" -python3 -m golem_core status +python3 -m golem status printf "\n*** FIND-NODE ***\n" -python3 -m golem_core find-node --runtime vm --timeout 1 +python3 -m golem find-node --runtime vm --timeout 1 printf "\n*** ALLOCATION LIST ***\n" -python3 -m golem_core allocation list +python3 -m golem allocation list printf "\n*** ALLOCATION NEW ***\n" -python3 -m golem_core allocation new 1 +python3 -m golem allocation new 1 printf "\n*** ALLOCATION NEW ***\n" -python3 -m golem_core allocation new 2 --driver erc20 --network goerli +python3 -m golem allocation new 2 --driver erc20 --network goerli printf "\n*** ALLOCATION LIST ***\n" -python3 -m golem_core allocation list +python3 -m golem allocation list printf "\n*** ALLOCATION CLEAN ***\n" -python3 -m golem_core allocation clean +python3 -m golem allocation clean printf "\n*** ALLOCATION LIST ***\n" -python3 -m golem_core allocation list +python3 -m golem allocation list diff --git a/examples/core_example.py b/examples/core_example.py index 6677d413..1bfc4460 100644 --- a/examples/core_example.py +++ b/examples/core_example.py @@ -4,17 +4,19 @@ from tempfile import TemporaryDirectory from typing import AsyncGenerator, Optional -from golem.resources.activity import ( +from golem.node import GolemNode +from golem.payload import RepositoryVmPayload +from golem.resources import ( Activity, BatchError, CommandCancelled, CommandFailed, + NewResource, + ResourceClosed, + ResourceEvent, Script, - commands, ) -from golem.resources.golem_node import GolemNode -from golem.resources.market import RepositoryVmPayload -from golem.resources.resources import NewResource, ResourceClosed, ResourceEvent +from golem.resources.activity import commands PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/examples/detached_activity.py b/examples/detached_activity.py index 7906269a..087d115c 100644 --- a/examples/detached_activity.py +++ b/examples/detached_activity.py @@ -1,9 +1,10 @@ import asyncio -from golem.resources.activity import Activity, commands -from golem.resources.golem_node import GolemNode -from golem.resources.market import Agreement, Proposal, RepositoryVmPayload, default_negotiate +from golem.node import GolemNode +from golem.payload import RepositoryVmPayload from golem.pipeline import Chain, Map +from golem.resources import Activity, Agreement, Proposal, default_negotiate +from golem.resources.activity import commands PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/examples/exception_handling/exception_handling.py b/examples/exception_handling/exception_handling.py index 0cdad688..d63a52a9 100644 --- a/examples/exception_handling/exception_handling.py +++ b/examples/exception_handling/exception_handling.py @@ -1,17 +1,19 @@ import asyncio from typing import Callable, Tuple -from golem.resources.activity import Activity, BatchError, BatchTimeoutError, commands -from golem.resources.events.base import Event -from golem.resources.golem_node import GolemNode -from golem.resources.market import ( - RepositoryVmPayload, +from golem.event_bus import Event +from golem.managers import DefaultPaymentManager +from golem.node import GolemNode +from golem.payload import RepositoryVmPayload +from golem.pipeline import Buffer, Chain, Limit, Map +from golem.resources import ( + BatchError, + BatchTimeoutError, default_create_activity, default_create_agreement, default_negotiate, ) -from golem.managers import DefaultPaymentManager -from golem.pipeline import Buffer, Chain, Limit, Map +from golem.resources.activity import Activity, commands from golem.utils.logging import DefaultLogger PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index d76ffbc5..8d3d6145 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -3,8 +3,6 @@ from random import randint from typing import List -from golem.resources.golem_node.golem_node import GolemNode -from golem.resources.market import RepositoryVmPayload from golem.managers.activity.single_use import SingleUseActivityManager from golem.managers.agreement.single_use import SingleUseAgreementManager from golem.managers.base import WorkContext, WorkResult @@ -18,6 +16,8 @@ work_decorator, ) from golem.managers.work.sequential import SequentialWorkManager +from golem.node import GolemNode +from golem.payload import RepositoryVmPayload from golem.utils.logging import DEFAULT_LOGGING @@ -80,7 +80,7 @@ async def main(): async with golem: async with payment_manager, negotiation_manager, proposal_manager: results: List[WorkResult] = await work_manager.do_work_list(work_list) - print(f"\nWORK MANAGER RESULTS:{[result.result for result in results]}\n") + print(f"\nWORK MANAGER RESULTS:{[result.result for result in results]}\n", flush=True) if __name__ == "__main__": diff --git a/examples/managers/ssh.py b/examples/managers/ssh.py index 006bcf97..808e09a1 100644 --- a/examples/managers/ssh.py +++ b/examples/managers/ssh.py @@ -4,8 +4,6 @@ import string from uuid import uuid4 -from golem.resources.golem_node.golem_node import GolemNode -from golem.resources.market import RepositoryVmPayload from golem.managers.activity.single_use import SingleUseActivityManager from golem.managers.agreement.single_use import SingleUseAgreementManager from golem.managers.base import WorkContext, WorkResult @@ -14,6 +12,8 @@ from golem.managers.payment.pay_all import PayAllPaymentManager from golem.managers.proposal import StackProposalManager from golem.managers.work.sequential import SequentialWorkManager +from golem.node import GolemNode +from golem.payload import RepositoryVmPayload from golem.utils.logging import DEFAULT_LOGGING diff --git a/examples/rate_providers/rate_providers.py b/examples/rate_providers/rate_providers.py index c193a87c..9989db2f 100644 --- a/examples/rate_providers/rate_providers.py +++ b/examples/rate_providers/rate_providers.py @@ -4,18 +4,18 @@ from pathlib import Path from typing import Any, AsyncIterator, Callable, Dict, Optional, Tuple -from golem.resources.activity import Activity, commands -from golem.resources.events.base import Event -from golem.resources.golem_node import GolemNode -from golem.resources.market import ( +from golem.event_bus import Event +from golem.managers import DefaultPaymentManager +from golem.node import GolemNode +from golem.payload import RepositoryVmPayload +from golem.pipeline import Buffer, Chain, Limit, Map +from golem.resources import ( Proposal, - RepositoryVmPayload, default_create_activity, default_create_agreement, default_negotiate, ) -from golem.managers import DefaultPaymentManager -from golem.pipeline import Buffer, Chain, Limit, Map +from golem.resources.activity import Activity, commands from golem.utils.logging import DefaultLogger FRAME_CONFIG_TEMPLATE = json.loads(Path(__file__).with_name("frame_params.json").read_text()) diff --git a/examples/score_based_providers/score_based_providers.py b/examples/score_based_providers/score_based_providers.py index f1a5cc63..493b43fc 100644 --- a/examples/score_based_providers/score_based_providers.py +++ b/examples/score_based_providers/score_based_providers.py @@ -2,18 +2,18 @@ from datetime import timedelta from typing import Callable, Dict, Optional, Tuple -from golem.resources.activity import Activity, commands -from golem.resources.events.base import Event -from golem.resources.golem_node import GolemNode -from golem.resources.market import ( +from golem.event_bus import Event +from golem.managers import DefaultPaymentManager +from golem.node import GolemNode +from golem.payload import RepositoryVmPayload +from golem.pipeline import Buffer, Chain, Limit, Map, Sort +from golem.resources import ( Proposal, - RepositoryVmPayload, default_create_activity, default_create_agreement, default_negotiate, ) -from golem.managers import DefaultPaymentManager -from golem.pipeline import Buffer, Chain, Limit, Map, Sort +from golem.resources.activity import Activity, commands from golem.utils.logging import DefaultLogger PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/examples/service.py b/examples/service.py index 96a6ce98..edbf93b0 100644 --- a/examples/service.py +++ b/examples/service.py @@ -5,17 +5,18 @@ from urllib.parse import urlparse from uuid import uuid4 -from golem.resources.activity import Activity, commands -from golem.resources.events.base import Event -from golem.resources.golem_node import GolemNode -from golem.resources.market import ( - RepositoryVmPayload, +from golem.event_bus import Event +from golem.node import GolemNode +from golem.payload import RepositoryVmPayload +from golem.pipeline import Buffer, Chain, Limit, Map +from golem.resources import ( + Activity, default_create_activity, default_create_agreement, default_negotiate, ) +from golem.resources.activity import commands from golem.resources.network import Network -from golem.pipeline import Buffer, Chain, Limit, Map from golem.utils.logging import DefaultLogger PAYLOAD = RepositoryVmPayload( diff --git a/examples/task_api_draft/examples/blender/blender.py b/examples/task_api_draft/examples/blender/blender.py index eef1e5b3..4d2ffd34 100644 --- a/examples/task_api_draft/examples/blender/blender.py +++ b/examples/task_api_draft/examples/blender/blender.py @@ -3,8 +3,8 @@ from pathlib import Path from examples.task_api_draft.task_api.execute_tasks import execute_tasks +from golem.payload import RepositoryVmPayload from golem.resources.activity import Activity, commands -from golem.resources.market import RepositoryVmPayload FRAME_CONFIG_TEMPLATE = json.loads(Path(__file__).with_name("frame_params.json").read_text()) FRAMES = list(range(0, 60, 10)) diff --git a/examples/task_api_draft/examples/execute_tasks_hello_world.py b/examples/task_api_draft/examples/execute_tasks_hello_world.py index 27659367..6f39113a 100644 --- a/examples/task_api_draft/examples/execute_tasks_hello_world.py +++ b/examples/task_api_draft/examples/execute_tasks_hello_world.py @@ -1,8 +1,8 @@ import asyncio from examples.task_api_draft.task_api.execute_tasks import execute_tasks +from golem.payload import RepositoryVmPayload from golem.resources.activity import Activity, commands -from golem.resources.market import RepositoryVmPayload TASK_DATA = list(range(7)) BUDGET = 1 diff --git a/examples/task_api_draft/examples/pipeline_example.py b/examples/task_api_draft/examples/pipeline_example.py index 3450c54d..b184133c 100644 --- a/examples/task_api_draft/examples/pipeline_example.py +++ b/examples/task_api_draft/examples/pipeline_example.py @@ -4,18 +4,18 @@ from typing import AsyncIterator, Callable, Tuple from examples.task_api_draft.task_api.activity_pool import ActivityPool -from golem.resources.activity import Activity, commands -from golem.resources.events.base import Event -from golem.resources.golem_node import GolemNode -from golem.resources.market import ( +from golem.events_bus import Event +from golem.managers import DefaultPaymentManager +from golem.node import GolemNode +from golem.payload import RepositoryVmPayload +from golem.pipeline import Buffer, Chain, Map, Sort, Zip +from golem.resources import ( Proposal, - RepositoryVmPayload, default_create_activity, default_create_agreement, default_negotiate, ) -from golem.managers import DefaultPaymentManager -from golem.pipeline import Buffer, Chain, Map, Sort, Zip +from golem.resources.activity import Activity, commands from golem.utils.logging import DefaultLogger PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/examples/task_api_draft/examples/redundance.py b/examples/task_api_draft/examples/redundance.py index 4feae5b4..0fc35ebc 100644 --- a/examples/task_api_draft/examples/redundance.py +++ b/examples/task_api_draft/examples/redundance.py @@ -2,8 +2,8 @@ import random from examples.task_api_draft.task_api.execute_tasks import execute_tasks +from golem.payload import RepositoryVmPayload from golem.resources.activity import Activity, commands -from golem.resources.market import RepositoryVmPayload PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/examples/task_api_draft/examples/yacat.py b/examples/task_api_draft/examples/yacat.py index cf53dc78..5d55ee73 100644 --- a/examples/task_api_draft/examples/yacat.py +++ b/examples/task_api_draft/examples/yacat.py @@ -23,20 +23,23 @@ tasks_queue, ) from examples.task_api_draft.task_api.activity_pool import ActivityPool -from golem.resources.activity import Activity, PoolingBatch, default_prepare_activity from golem.event_bus import Event -from golem.resources.golem_node import GolemNode -from golem.resources.market import ( +from golem.managers import DefaultPaymentManager +from golem.node import GolemNode +from golem.pipeline import Buffer, Chain, Map, Sort, Zip +from golem.resources import ( + Activity, + DebitNote, + NewDebitNote, + NewResource, + PoolingBatch, Proposal, + ResourceClosed, default_create_activity, default_create_agreement, default_negotiate, + default_prepare_activity, ) -from golem.resources.payment import DebitNote -from golem.resources.debit_note.events import NewDebitNote -from golem.resources.resources import NewResource, ResourceClosed -from golem.managers import DefaultPaymentManager -from golem.pipeline import Buffer, Chain, Map, Sort, Zip from golem.utils.logging import DefaultLogger ########################### diff --git a/examples/task_api_draft/examples/yacat_no_business_logic.py b/examples/task_api_draft/examples/yacat_no_business_logic.py index 139d2060..cd77f87f 100644 --- a/examples/task_api_draft/examples/yacat_no_business_logic.py +++ b/examples/task_api_draft/examples/yacat_no_business_logic.py @@ -5,8 +5,8 @@ from typing import List, Union from examples.task_api_draft.task_api.execute_tasks import execute_tasks -from golem.resources.activity import Activity, Run -from golem.resources.market import RepositoryVmPayload +from golem.payload import RepositoryVmPayload +from golem.resources import Activity, Run PAYLOAD = RepositoryVmPayload("055911c811e56da4d75ffc928361a78ed13077933ffa8320fb1ec2db") PASSWORD_LENGTH = 3 diff --git a/examples/task_api_draft/task_api/activity_pool.py b/examples/task_api_draft/task_api/activity_pool.py index 4ed11e9c..df814fa3 100644 --- a/examples/task_api_draft/task_api/activity_pool.py +++ b/examples/task_api_draft/task_api/activity_pool.py @@ -2,8 +2,8 @@ import inspect from typing import AsyncIterator, Awaitable, List, Union -from golem.resources.activity import Activity -from golem.pipeline.exceptions import InputStreamExhausted +from golem.pipeline import InputStreamExhausted +from golem.resources import Activity class ActivityPool: diff --git a/examples/task_api_draft/task_api/execute_tasks.py b/examples/task_api_draft/task_api/execute_tasks.py index e588a74b..8e605249 100644 --- a/examples/task_api_draft/task_api/execute_tasks.py +++ b/examples/task_api_draft/task_api/execute_tasks.py @@ -2,19 +2,20 @@ from random import random from typing import AsyncIterator, Awaitable, Callable, Iterable, Optional, Tuple, TypeVar -from golem.resources.activity import Activity, default_prepare_activity -from golem.resources.events.base import Event -from golem.resources.golem_node import GolemNode -from golem.resources.market import ( +from golem.event_bus import Event +from golem.managers import DefaultPaymentManager +from golem.node import GolemNode +from golem.payload import Payload +from golem.pipeline import Buffer, Chain, Map, Sort, Zip +from golem.resources import ( + Activity, Demand, - Payload, Proposal, default_create_activity, default_create_agreement, default_negotiate, + default_prepare_activity, ) -from golem.managers import DefaultPaymentManager -from golem.pipeline import Buffer, Chain, Map, Sort, Zip from golem.utils.logging import DefaultLogger from .activity_pool import ActivityPool diff --git a/examples/task_api_draft/task_api/redundance_manager.py b/examples/task_api_draft/task_api/redundance_manager.py index 13c11492..c50fe125 100644 --- a/examples/task_api_draft/task_api/redundance_manager.py +++ b/examples/task_api_draft/task_api/redundance_manager.py @@ -12,8 +12,7 @@ TypeVar, ) -from golem.resources.activity import Activity -from golem.resources.market import Proposal +from golem.resources import Activity, Proposal from .task_data_stream import TaskDataStream diff --git a/golem/cli/utils.py b/golem/cli/utils.py index 99ae7efe..fca14003 100644 --- a/golem/cli/utils.py +++ b/golem/cli/utils.py @@ -8,9 +8,7 @@ from prettytable import PrettyTable from typing_extensions import Concatenate, ParamSpec -from golem.resources.golem_node import GolemNode -from golem.resources.market import RUNTIME_NAME, Demand, Payload, Proposal, constraint -from golem.resources.payment import Allocation +from golem.node import GolemNode def format_allocations(allocations: List[Allocation]) -> str: diff --git a/golem/event_bus/__init__.py b/golem/event_bus/__init__.py index c281f026..dc2db152 100644 --- a/golem/event_bus/__init__.py +++ b/golem/event_bus/__init__.py @@ -1,7 +1,8 @@ -from golem.event_bus.base import EventBus, TEvent, Event +from golem.event_bus.base import Event, EventBus, EventBusError, TEvent __all__ = ( "EventBus", + "EventBusError", "TEvent", "Event", ) diff --git a/golem/event_bus/base.py b/golem/event_bus/base.py index 576b0dc7..676ed82e 100644 --- a/golem/event_bus/base.py +++ b/golem/event_bus/base.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import TypeVar, Type, Callable, Awaitable, Optional +from typing import Awaitable, Callable, Optional, Type, TypeVar from golem.exceptions import GolemException @@ -7,6 +7,7 @@ TEvent = TypeVar("TEvent", bound="Event") + class Event(ABC): """Base class for all events.""" diff --git a/golem/event_bus/in_memory/__init__.py b/golem/event_bus/in_memory/__init__.py index bb27dbe1..26a838a0 100644 --- a/golem/event_bus/in_memory/__init__.py +++ b/golem/event_bus/in_memory/__init__.py @@ -1,5 +1,3 @@ from golem.event_bus.in_memory.event_bus import InMemoryEventBus -__all__ = ( - 'InMemoryEventBus', -) \ No newline at end of file +__all__ = ("InMemoryEventBus",) diff --git a/golem/event_bus/in_memory/event_bus.py b/golem/event_bus/in_memory/event_bus.py index adda58f0..6e2e0f5b 100644 --- a/golem/event_bus/in_memory/event_bus.py +++ b/golem/event_bus/in_memory/event_bus.py @@ -4,9 +4,7 @@ from dataclasses import dataclass from typing import Awaitable, Callable, DefaultDict, List, Optional, Type -from golem.resources.events.base import Event -from golem.resources.events.base import EventBus as BaseEventBus -from golem.resources.events.base import EventBusError, TCallbackHandler, TEvent +from golem.event_bus.base import Event, EventBus, EventBusError, TCallbackHandler, TEvent from golem.utils.asyncio import create_task_with_logging logger = logging.getLogger(__name__) @@ -19,7 +17,7 @@ class _CallbackInfo: once: bool -class InMemoryEventBus(BaseEventBus): +class InMemoryEventBus(EventBus): def __init__(self): self._callbacks: DefaultDict[Type[TEvent], List[_CallbackInfo]] = defaultdict(list) self._event_queue = asyncio.Queue() diff --git a/golem/exceptions.py b/golem/exceptions.py index 6c67d3c3..04ef5ce9 100644 --- a/golem/exceptions.py +++ b/golem/exceptions.py @@ -1,2 +1,11 @@ class GolemException(Exception): pass + + +class MissingConfiguration(GolemException): + def __init__(self, key: str, description: str): + self._key = key + self._description = description + + def __str__(self) -> str: + return f"Missing configuration for {self._description}. Please set env var {self._key}." diff --git a/golem/managers/activity/__init__.py b/golem/managers/activity/__init__.py index b203330f..45b0e3aa 100644 --- a/golem/managers/activity/__init__.py +++ b/golem/managers/activity/__init__.py @@ -1,7 +1,4 @@ -from golem.managers.activity.defaults import ( - default_on_activity_start, - default_on_activity_stop, -) +from golem.managers.activity.defaults import default_on_activity_start, default_on_activity_stop from golem.managers.activity.single_use import SingleUseActivityManager __all__ = ( diff --git a/golem/managers/activity/single_use.py b/golem/managers/activity/single_use.py index 0f5e62ee..7b03f29c 100644 --- a/golem/managers/activity/single_use.py +++ b/golem/managers/activity/single_use.py @@ -2,15 +2,11 @@ from contextlib import asynccontextmanager from typing import Awaitable, Callable, Optional -from golem.resources.activity import Activity -from golem.resources.golem_node.golem_node import GolemNode -from golem.resources.market import Agreement -from golem.managers.activity.defaults import ( - default_on_activity_start, - default_on_activity_stop, -) +from golem.managers.activity.defaults import default_on_activity_start, default_on_activity_stop from golem.managers.agreement import AgreementReleased from golem.managers.base import ActivityManager, Work, WorkContext, WorkResult +from golem.node import GolemNode +from golem.resources import Activity, Agreement logger = logging.getLogger(__name__) diff --git a/golem/managers/agreement/single_use.py b/golem/managers/agreement/single_use.py index 3a2f6dda..6dd0e93d 100644 --- a/golem/managers/agreement/single_use.py +++ b/golem/managers/agreement/single_use.py @@ -1,10 +1,10 @@ import logging from typing import Awaitable, Callable -from golem.resources.golem_node.golem_node import GolemNode -from golem.resources.market import Agreement, Proposal from golem.managers.agreement.events import AgreementReleased from golem.managers.base import AgreementManager +from golem.node import GolemNode +from golem.resources import Agreement, Proposal logger = logging.getLogger(__name__) @@ -46,7 +46,7 @@ async def get_agreement(self) -> Agreement: await self._event_bus.on_once( AgreementReleased, self._terminate_agreement, - lambda e: e.resource.id == agreement.id, + lambda event: event.resource.id == agreement.id, ) logger.debug(f"Getting agreement done with `{agreement}`") diff --git a/golem/managers/base.py b/golem/managers/base.py index c1a285bf..6e2c0b90 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -2,13 +2,18 @@ from dataclasses import dataclass, field from typing import Any, Awaitable, Callable, Dict, List, Optional, Union -from golem.resources.activity import Activity, Script, commands -from golem.resources.market import Agreement, Proposal -from golem.resources.demand.demand import DemandData -from golem.resources.market.proposal import ProposalData -from golem.resources.payment import Allocation -from golem.resources.resources import ResourceEvent from golem.exceptions import GolemException +from golem.resources import ( + Activity, + Agreement, + Allocation, + DemandData, + Proposal, + ProposalData, + ResourceEvent, + Script, +) +from golem.resources.activity import commands class Batch: diff --git a/golem/managers/negotiation/plugins.py b/golem/managers/negotiation/plugins.py index d2d8e94a..bfa75b32 100644 --- a/golem/managers/negotiation/plugins.py +++ b/golem/managers/negotiation/plugins.py @@ -1,11 +1,10 @@ import logging from typing import Sequence, Set -from golem.resources.demand.demand import DemandData -from golem.resources.market.proposal import ProposalData -from golem.payload.properties import Properties from golem.managers.base import NegotiationPlugin from golem.managers.negotiation.sequential import RejectProposal +from golem.payload import Properties +from golem.resources import DemandData, ProposalData logger = logging.getLogger(__name__) diff --git a/golem/managers/negotiation/sequential.py b/golem/managers/negotiation/sequential.py index 2e4b8a17..dca5fa08 100644 --- a/golem/managers/negotiation/sequential.py +++ b/golem/managers/negotiation/sequential.py @@ -6,14 +6,11 @@ from ya_market import ApiException -from golem.resources.golem_node.golem_node import GolemNode -from golem.resources.market import Demand, DemandBuilder, Payload, Proposal -from golem.resources.demand.demand import DemandData -from golem.resources.market.proposal import ProposalData -from golem.resources.payment import Allocation -from golem.payload.parsers import TextXDemandOfferSyntaxParser -from golem.payload.properties import Properties from golem.managers.base import ManagerException, NegotiationManager, NegotiationPlugin +from golem.node import GolemNode +from golem.payload import Payload, Properties +from golem.payload.parsers.textx import TextXPayloadSyntaxParser +from golem.resources import Allocation, Demand, DemandBuilder, DemandData, Proposal, ProposalData from golem.utils.asyncio import create_task_with_logging logger = logging.getLogger(__name__) @@ -38,7 +35,7 @@ def __init__( self._negotiation_loop_task: Optional[asyncio.Task] = None self._plugins: List[NegotiationPlugin] = list(plugins) if plugins is not None else [] self._eligible_proposals: asyncio.Queue[Proposal] = asyncio.Queue() - self._demand_offer_parser = TextXDemandOfferSyntaxParser() + self._demand_offer_parser = TextXPayloadSyntaxParser() def register_plugin(self, plugin: NegotiationPlugin): self._plugins.append(plugin) diff --git a/golem/managers/network/single.py b/golem/managers/network/single.py index ae78794b..b2d06ad3 100644 --- a/golem/managers/network/single.py +++ b/golem/managers/network/single.py @@ -3,11 +3,9 @@ from typing import Dict from urllib.parse import urlparse -from golem.resources.golem_node.golem_node import GolemNode -from golem.resources.agreement.events import NewAgreement -from golem.resources.network import Network -from golem.resources.network.network import DeployArgsType from golem.managers.base import NetworkManager +from golem.node import GolemNode +from golem.resources import DeployArgsType, Network, NewAgreement logger = logging.getLogger(__name__) diff --git a/golem/managers/payment/default.py b/golem/managers/payment/default.py index c3717146..57f00224 100644 --- a/golem/managers/payment/default.py +++ b/golem/managers/payment/default.py @@ -2,13 +2,11 @@ from datetime import datetime, timedelta from typing import TYPE_CHECKING, Set -from golem.resources.debit_note.events import NewDebitNote -from golem.resources.payment.events import NewInvoice +from golem.resources import NewDebitNote, NewInvoice if TYPE_CHECKING: - from golem.resources.golem_node import GolemNode - from golem.resources.agreement.events import NewAgreement - from golem.resources.payment import Allocation + from golem.node import GolemNode + from golem.resources import Allocation, NewAgreement class DefaultPaymentManager: diff --git a/golem/managers/payment/pay_all.py b/golem/managers/payment/pay_all.py index d37aeb14..e3189f7b 100644 --- a/golem/managers/payment/pay_all.py +++ b/golem/managers/payment/pay_all.py @@ -3,14 +3,17 @@ from decimal import Decimal from typing import Optional -from golem.resources.golem_node.golem_node import PAYMENT_DRIVER, PAYMENT_NETWORK, GolemNode -from golem.resources.agreement.events import NewAgreement, AgreementClosed -from golem.resources.payment.events import NewInvoice -from golem.resources.debit_note.events import NewDebitNote -from golem.resources.allocation.allocation import Allocation -from golem.resources.debit_note.debit_note import DebitNote -from golem.resources.payment.resources.invoice import Invoice from golem.managers.base import PaymentManager +from golem.node import PAYMENT_DRIVER, PAYMENT_NETWORK, GolemNode +from golem.resources import ( + AgreementClosed, + Allocation, + DebitNote, + Invoice, + NewAgreement, + NewDebitNote, + NewInvoice, +) logger = logging.getLogger(__name__) diff --git a/golem/managers/proposal/stack.py b/golem/managers/proposal/stack.py index 9659e863..a048e279 100644 --- a/golem/managers/proposal/stack.py +++ b/golem/managers/proposal/stack.py @@ -2,9 +2,9 @@ import logging from typing import Optional -from golem.resources.golem_node.golem_node import GolemNode -from golem.resources.market import Proposal from golem.managers.base import ManagerException, ProposalManager +from golem.node import GolemNode +from golem.resources import Proposal from golem.utils.asyncio import create_task_with_logging logger = logging.getLogger(__name__) diff --git a/golem/managers/work/sequential.py b/golem/managers/work/sequential.py index 411df8a1..5dee56ca 100644 --- a/golem/managers/work/sequential.py +++ b/golem/managers/work/sequential.py @@ -1,8 +1,8 @@ import logging from typing import List -from golem.resources.golem_node.golem_node import GolemNode from golem.managers.base import DoWorkCallable, Work, WorkManager, WorkResult +from golem.node import GolemNode logger = logging.getLogger(__name__) diff --git a/golem/node/__init__.py b/golem/node/__init__.py index a5cf7d46..a97e1890 100644 --- a/golem/node/__init__.py +++ b/golem/node/__init__.py @@ -1,12 +1,12 @@ -from golem.node.events import ( - GolemNodeEvent, - SessionStarted, - ShutdownFinished, - ShutdownStarted, +from golem.node.events import GolemNodeEvent, SessionStarted, ShutdownFinished, ShutdownStarted +from golem.node.node import ( + DEFAULT_EXPIRATION_TIMEOUT, + PAYMENT_DRIVER, + PAYMENT_NETWORK, + SUBNET, + GolemNode, ) -from golem.node.node import GolemNode, PAYMENT_NETWORK, PAYMENT_DRIVER, SUBNET - __all__ = ( "GolemNode", "GolemNodeEvent", @@ -16,4 +16,5 @@ "PAYMENT_DRIVER", "PAYMENT_NETWORK", "SUBNET", + "DEFAULT_EXPIRATION_TIMEOUT", ) diff --git a/golem/node/event_collectors/__init__.py b/golem/node/event_collectors/__init__.py deleted file mode 100644 index 581d3159..00000000 --- a/golem/node/event_collectors/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from golem.node.event_collectors.base import YagnaEventCollector -from golem.node.event_collectors.utils import is_intermittent_error, is_gsb_endpoint_not_found_error - -__all__ = ( - "YagnaEventCollector", - "is_gsb_endpoint_not_found_error", - "is_intermittent_error", -) diff --git a/golem/node/event_collectors/utils.py b/golem/node/event_collectors/utils.py deleted file mode 100644 index 1e836af4..00000000 --- a/golem/node/event_collectors/utils.py +++ /dev/null @@ -1,41 +0,0 @@ -import asyncio -import json -from typing import Union - -import aiohttp -import ya_activity -import ya_market -import ya_payment - - -def is_intermittent_error(e: Exception) -> bool: - """Check if `e` indicates an intermittent communication failure such as network timeout.""" - - is_timeout_exception = isinstance(e, asyncio.TimeoutError) or ( - isinstance( - e, - (ya_activity.ApiException, ya_market.ApiException, ya_payment.ApiException), - ) - and e.status in (408, 504) - ) - - return ( - is_timeout_exception - or isinstance(e, aiohttp.ServerDisconnectedError) - # OS error with errno 32 is "Broken pipe" - or (isinstance(e, aiohttp.ClientOSError) and e.errno == 32) - ) - - -def is_gsb_endpoint_not_found_error( - err: Union[ya_activity.ApiException, ya_market.ApiException, ya_payment.ApiException] -) -> bool: - """Check if `err` is caused by "Endpoint address not found" GSB error.""" - - if err.status != 500: - return False - try: - msg = json.loads(err.body)["message"] - return "GSB error" in msg and "endpoint address not found" in msg - except Exception: - return False diff --git a/golem/node/events.py b/golem/node/events.py index 4121f118..b9fdf792 100644 --- a/golem/node/events.py +++ b/golem/node/events.py @@ -4,7 +4,7 @@ from golem.event_bus import Event if TYPE_CHECKING: - from golem.resources.golem_node import GolemNode + from golem.node import GolemNode class GolemNodeEvent(Event, ABC): diff --git a/golem/node/node.py b/golem/node/node.py index 8e2c6960..e29c14bc 100644 --- a/golem/node/node.py +++ b/golem/node/node.py @@ -6,21 +6,29 @@ from typing import Any, DefaultDict, Dict, Iterable, List, Optional, Set, Type, Union from uuid import uuid4 -from golem.resources.activity import Activity, PoolingBatch -from golem.event_bus import EventBus, InMemoryEventBus +from golem.event_bus import EventBus +from golem.event_bus.in_memory import InMemoryEventBus from golem.node.events import SessionStarted, ShutdownFinished, ShutdownStarted -from golem.resources.market import Agreement, Demand, DemandBuilder, Payload, Proposal -from golem.resources.market.resources.demand.demand_offer_base import defaults as dobm_defaults -from golem.resources.network import Network -from golem.resources.payment import ( +from golem.payload import Payload +from golem.payload import defaults as payload_defaults +from golem.payload.parsers.textx import TextXPayloadSyntaxParser +from golem.resources import ( + Activity, + Agreement, Allocation, DebitNote, DebitNoteEventCollector, + Demand, + DemandBuilder, Invoice, InvoiceEventCollector, + Network, + PoolingBatch, + Proposal, + Resource, + TResource, ) -from golem.payload.parsers import TextXDemandOfferSyntaxParser -from golem.resources.resources import ApiConfig, ApiFactory, Resource, TResource +from golem.utils.low import ApiConfig, ApiFactory PAYMENT_DRIVER: str = os.getenv("YAGNA_PAYMENT_DRIVER", "erc20").lower() PAYMENT_NETWORK: str = os.getenv("YAGNA_PAYMENT_NETWORK", "goerli").lower() @@ -85,7 +93,7 @@ def __init__( self._resources: DefaultDict[Type[Resource], Dict[str, Resource]] = defaultdict(dict) self._autoclose_resources: Set[Resource] = set() self._event_bus = InMemoryEventBus() - self._demand_offer_syntax_parser = TextXDemandOfferSyntaxParser() + self._demand_offer_syntax_parser = TextXPayloadSyntaxParser() self._invoice_event_collector = InvoiceEventCollector(self) self._debit_note_event_collector = DebitNoteEventCollector(self) @@ -228,8 +236,8 @@ async def create_demand( expiration = datetime.now(timezone.utc) + DEFAULT_EXPIRATION_TIMEOUT builder = DemandBuilder() - await builder.add(dobm_defaults.ActivityInfo(expiration=expiration, multi_activity=True)) - await builder.add(dobm_defaults.NodeInfo(subnet_tag=subnet)) + await builder.add(payload_defaults.ActivityInfo(expiration=expiration, multi_activity=True)) + await builder.add(payload_defaults.NodeInfo(subnet_tag=subnet)) await builder.add(payload) await self._add_builder_allocations(builder, allocations) diff --git a/golem/payload/__init__.py b/golem/payload/__init__.py index 02063e8a..7d55acf4 100644 --- a/golem/payload/__init__.py +++ b/golem/payload/__init__.py @@ -1,18 +1,25 @@ -from golem.payload.base import Payload -from golem.payload.constraints import Constraints, ConstraintException -from golem.payload.exceptions import PayloadException, InvalidProperties +from golem.payload.base import Payload, constraint, prop +from golem.payload.constraints import Constraint, ConstraintException, ConstraintGroup, Constraints +from golem.payload.exceptions import InvalidProperties, PayloadException +from golem.payload.parsers import PayloadSyntaxParser, SyntaxException from golem.payload.properties import Properties -from golem.payload.vm import VmPayload, RepositoryVmPayload, ManifestVmPayload, VmPayloadException +from golem.payload.vm import ManifestVmPayload, RepositoryVmPayload, VmPayload, VmPayloadException __all__ = ( - 'Payload', - 'VmPayload', - 'RepositoryVmPayload', - 'ManifestVmPayload', - 'VmPayloadException', - 'Constraints', - 'Properties', - 'PayloadException', - 'ConstraintException', - 'InvalidProperties', -) \ No newline at end of file + "Payload", + "prop", + "constraint", + "VmPayload", + "RepositoryVmPayload", + "ManifestVmPayload", + "VmPayloadException", + "Constraints", + "Constraint", + "ConstraintGroup", + "Properties", + "PayloadException", + "ConstraintException", + "InvalidProperties", + "SyntaxException", + "PayloadSyntaxParser", +) diff --git a/golem/payload/base.py b/golem/payload/base.py index 43436a75..0f978a55 100644 --- a/golem/payload/base.py +++ b/golem/payload/base.py @@ -1,19 +1,14 @@ import abc import dataclasses import enum -from typing import Any, Dict, Final, List, Tuple, Type, TypeVar, TypeAlias +from typing import Any, Dict, Final, List, Tuple, Type, TypeVar -from golem.payload.exceptions import ( - InvalidProperties, -) from golem.payload.constraints import Constraint, ConstraintOperator, Constraints +from golem.payload.exceptions import InvalidProperties from golem.payload.properties import Properties TPayload = TypeVar("TPayload", bound="Payload") -PropertyName: TypeAlias = str -PropertyValue: TypeAlias = Any - PROP_KEY: Final[str] = "key" PROP_OPERATOR: Final[str] = "operator" PROP_MODEL_FIELD_TYPE: Final[str] = "model_field_type" @@ -106,9 +101,7 @@ def _get_fields(cls, field_type: PayloadFieldType) -> List[dataclasses.Field]: ] @classmethod - def from_properties( - cls: Type[TPayload], props: Dict[str, Any] - ) -> TPayload: + def from_properties(cls: Type[TPayload], props: Dict[str, Any]) -> TPayload: """Initialize the model with properties from given dictionary. Only properties defined in model will be picked up from given dictionary, ignoring other @@ -116,8 +109,7 @@ def from_properties( will only load their own data. """ field_map = { - field.metadata[PROP_KEY]: field - for field in cls._get_fields(PayloadFieldType.property) + field.metadata[PROP_KEY]: field for field in cls._get_fields(PayloadFieldType.property) } data = { field_map[key].name: Properties._deserialize_value(val, field_map[key]) diff --git a/golem/payload/constraints.py b/golem/payload/constraints.py index da1218d1..3a664caa 100644 --- a/golem/payload/constraints.py +++ b/golem/payload/constraints.py @@ -1,10 +1,12 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import Any, Literal, MutableSequence, Union +from typing import Any, Literal, MutableSequence, TypeAlias, Union -from golem.payload.base import PropertyName from golem.payload.mixins import PropsConsSerializerMixin +PropertyName: TypeAlias = str +PropertyValue: TypeAlias = Any + class ConstraintException(Exception): pass @@ -36,7 +38,7 @@ def _validate(self) -> None: class Constraint(PayloadSyntaxElement): property_name: PropertyName operator: ConstraintOperator - value: Any + value: PropertyValue def _serialize(self) -> str: serialized_value = self._serialize_value(self.value) diff --git a/golem/payload/defaults.py b/golem/payload/defaults.py index c1672300..e390250c 100644 --- a/golem/payload/defaults.py +++ b/golem/payload/defaults.py @@ -3,10 +3,7 @@ from decimal import Decimal from typing import Optional -from golem.resources.market.resources.demand.demand_offer_base.model import ( - DemandOfferBaseModel, - prop, -) +from golem.payload.base import Payload, prop RUNTIME_NAME = "golem.runtime.name" RUNTIME_CAPABILITIES = "golem.runtime.capabilities" @@ -16,7 +13,7 @@ @dataclass -class NodeInfo(DemandOfferBaseModel): +class NodeInfo(Payload): """Properties describing the information regarding the node.""" name: Optional[str] = prop("golem.node.id.name", default=None) @@ -27,7 +24,7 @@ class NodeInfo(DemandOfferBaseModel): @dataclass -class ActivityInfo(DemandOfferBaseModel): +class ActivityInfo(Payload): """Activity-related Properties.""" cost_cap: Optional[Decimal] = prop("golem.activity.cost_cap", default=None) @@ -59,6 +56,6 @@ class ActivityInfo(DemandOfferBaseModel): @dataclass -class PaymentInfo(DemandOfferBaseModel): +class PaymentInfo(Payload): chosen_payment_platform: Optional[str] = prop("golem.com.payment.chosen-platform", default=None) """Payment platform selected to be used for this demand.""" diff --git a/golem/payload/parsers/__init__.py b/golem/payload/parsers/__init__.py index 8ebf7c5c..37123a96 100644 --- a/golem/payload/parsers/__init__.py +++ b/golem/payload/parsers/__init__.py @@ -1,7 +1,6 @@ from golem.payload.parsers.base import PayloadSyntaxParser, SyntaxException - __all__ = ( - 'PayloadSyntaxParser', - 'SyntaxException', -) \ No newline at end of file + "PayloadSyntaxParser", + "SyntaxException", +) diff --git a/golem/payload/parsers/textx/__init__.py b/golem/payload/parsers/textx/__init__.py index d2df8115..47e127c8 100644 --- a/golem/payload/parsers/textx/__init__.py +++ b/golem/payload/parsers/textx/__init__.py @@ -1,5 +1,3 @@ from golem.payload.parsers.textx.parser import TextXPayloadSyntaxParser -__all__ = ( - 'TextXPayloadSyntaxParser', -) \ No newline at end of file +__all__ = ("TextXPayloadSyntaxParser",) diff --git a/golem/payload/vm.py b/golem/payload/vm.py index fb38a53e..ced88dc6 100644 --- a/golem/payload/vm.py +++ b/golem/payload/vm.py @@ -9,9 +9,8 @@ from srvresolver.srv_record import SRVRecord from srvresolver.srv_resolver import SRVResolver -from golem.resources.market.resources.demand.demand_offer_base import defaults -from golem.resources.market.resources.demand.demand_offer_base.model import constraint, prop -from golem.resources.market.resources.demand.demand_offer_base.payload.base import Payload +from golem.payload import defaults +from golem.payload.base import Payload, constraint, prop from golem.payload.constraints import Constraints from golem.payload.properties import Properties from golem.utils.http import make_http_get_request, make_http_head_request diff --git a/golem/pipeline/__init__.py b/golem/pipeline/__init__.py index 0b0ee339..19624116 100644 --- a/golem/pipeline/__init__.py +++ b/golem/pipeline/__init__.py @@ -1,5 +1,6 @@ from golem.pipeline.buffer import Buffer from golem.pipeline.chain import Chain +from golem.pipeline.exceptions import InputStreamExhausted from golem.pipeline.limit import Limit from golem.pipeline.map import Map from golem.pipeline.sort import Sort @@ -12,4 +13,5 @@ "Zip", "Buffer", "Sort", + "InputStreamExhausted", ) diff --git a/golem/resources/__init__.py b/golem/resources/__init__.py index e69de29b..f05d3dc1 100644 --- a/golem/resources/__init__.py +++ b/golem/resources/__init__.py @@ -0,0 +1,161 @@ +from golem.resources.activity import ( + Activity, + ActivityClosed, + ActivityDataChanged, + Command, + Deploy, + DownloadFile, + NewActivity, + Run, + Script, + SendFile, + Start, + default_prepare_activity, +) +from golem.resources.agreement import ( + Agreement, + AgreementClosed, + AgreementDataChanged, + NewAgreement, + default_create_activity, +) +from golem.resources.allocation import ( + Allocation, + AllocationClosed, + AllocationDataChanged, + AllocationException, + NewAllocation, + NoMatchingAccount, +) +from golem.resources.base import Resource, TResource +from golem.resources.debit_note import ( + DebitNote, + DebitNoteClosed, + DebitNoteDataChanged, + DebitNoteEventCollector, + NewDebitNote, +) +from golem.resources.demand import ( + Demand, + DemandBuilder, + DemandClosed, + DemandData, + DemandDataChanged, + NewDemand, +) +from golem.resources.events import ( + NewResource, + ResourceClosed, + ResourceDataChanged, + ResourceEvent, + TResourceEvent, +) +from golem.resources.exceptions import ResourceException, ResourceNotFound +from golem.resources.invoice import ( + Invoice, + InvoiceClosed, + InvoiceDataChanged, + InvoiceEventCollector, + NewInvoice, +) +from golem.resources.network import ( + DeployArgsType, + Network, + NetworkClosed, + NetworkDataChanged, + NetworkException, + NetworkFull, + NewNetwork, +) +from golem.resources.pooling_batch import ( + BatchError, + BatchFinished, + BatchTimeoutError, + CommandCancelled, + CommandFailed, + NewPoolingBatch, + PoolingBatch, + PoolingBatchException, +) +from golem.resources.proposal import ( + NewProposal, + Proposal, + ProposalClosed, + ProposalData, + ProposalDataChanged, + default_create_agreement, + default_negotiate, +) + +__all__ = ( + "Activity", + "NewActivity", + "ActivityDataChanged", + "ActivityClosed", + "Command", + "Script", + "Deploy", + "Start", + "Run", + "SendFile", + "DownloadFile", + "default_prepare_activity", + "Agreement", + "NewAgreement", + "AgreementDataChanged", + "AgreementClosed", + "default_create_activity", + "Allocation", + "AllocationException", + "NoMatchingAccount", + "NewAllocation", + "AllocationDataChanged", + "AllocationClosed", + "DebitNote", + "NewDebitNote", + "DebitNoteEventCollector", + "DebitNoteDataChanged", + "DebitNoteClosed", + "Demand", + "DemandBuilder", + "DemandData", + "NewDemand", + "DemandDataChanged", + "DemandClosed", + "Invoice", + "InvoiceEventCollector", + "NewInvoice", + "InvoiceDataChanged", + "InvoiceClosed", + "Network", + "DeployArgsType", + "NetworkException", + "NetworkFull", + "NewNetwork", + "NetworkDataChanged", + "NetworkClosed", + "PoolingBatch", + "NewPoolingBatch", + "BatchFinished", + "PoolingBatchException", + "BatchError", + "CommandFailed", + "CommandCancelled", + "BatchTimeoutError", + "Proposal", + "ProposalData", + "NewProposal", + "ProposalDataChanged", + "ProposalClosed", + "default_negotiate", + "default_create_agreement", + "TResourceEvent", + "TResource", + "Resource", + "ResourceEvent", + "NewResource", + "ResourceDataChanged", + "ResourceClosed", + "ResourceException", + "ResourceNotFound", +) diff --git a/golem/resources/activity/__init__.py b/golem/resources/activity/__init__.py index c9375bea..b6e08bef 100644 --- a/golem/resources/activity/__init__.py +++ b/golem/resources/activity/__init__.py @@ -8,14 +8,14 @@ SendFile, Start, ) -from golem.resources.activity.events import NewActivity, ActivityDataChanged, ActivityClosed +from golem.resources.activity.events import ActivityClosed, ActivityDataChanged, NewActivity from golem.resources.activity.pipeline import default_prepare_activity __all__ = ( "Activity", - 'NewActivity', - 'ActivityDataChanged', - 'ActivityClosed', + "NewActivity", + "ActivityDataChanged", + "ActivityClosed", "Command", "Script", "Deploy", @@ -24,4 +24,4 @@ "SendFile", "DownloadFile", "default_prepare_activity", -) \ No newline at end of file +) diff --git a/golem/resources/activity/activity.py b/golem/resources/activity/activity.py index afb74b11..f65ae0e4 100644 --- a/golem/resources/activity/activity.py +++ b/golem/resources/activity/activity.py @@ -7,13 +7,14 @@ from golem.resources.activity.commands import Command, Script from golem.resources.activity.events import ActivityClosed, NewActivity -from golem.resources.pooling_batch.pooling_batch import PoolingBatch -from golem.resources.payment import DebitNote -from golem.resources.resources import _NULL, ActivityApi, Resource, api_call_wrapper +from golem.resources.base import _NULL, Resource, api_call_wrapper +from golem.resources.debit_note import DebitNote +from golem.resources.pooling_batch import PoolingBatch +from golem.utils.low import ActivityApi if TYPE_CHECKING: - from golem.resources.golem_node.golem_node import GolemNode - from golem.resources.market import Agreement # noqa + from golem.node import GolemNode + from golem.resources.agreement import Agreement # noqa class Activity(Resource[ActivityApi, _NULL, "Agreement", PoolingBatch, _NULL]): diff --git a/golem/resources/activity/commands.py b/golem/resources/activity/commands.py index 7633dabb..c14107bb 100644 --- a/golem/resources/activity/commands.py +++ b/golem/resources/activity/commands.py @@ -7,7 +7,8 @@ from ya_activity import models -from golem.storage import Destination, GftpProvider, Source +from golem.utils.storage import Destination, Source +from golem.utils.storage.gftp import GftpProvider ArgsDict = Mapping[str, Union[str, List, Dict[str, Any]]] diff --git a/golem/resources/activity/events.py b/golem/resources/activity/events.py index a740de61..59141095 100644 --- a/golem/resources/activity/events.py +++ b/golem/resources/activity/events.py @@ -1,3 +1,11 @@ +from typing import TYPE_CHECKING + +from golem.resources.events import NewResource, ResourceClosed, ResourceDataChanged + +if TYPE_CHECKING: + from golem.resources.activity.activity import Activity # noqa + + class NewActivity(NewResource["Activity"]): pass diff --git a/golem/resources/activity/pipeline.py b/golem/resources/activity/pipeline.py index d7270c51..b9c7f140 100644 --- a/golem/resources/activity/pipeline.py +++ b/golem/resources/activity/pipeline.py @@ -1,5 +1,5 @@ +from golem.resources.activity.activity import Activity from golem.resources.activity.commands import Deploy, Start -from golem.resources.activity.resources import Activity # TODO: Move default functions to Activity class diff --git a/golem/resources/agreement/__init__.py b/golem/resources/agreement/__init__.py index 74100277..5cfd7fca 100644 --- a/golem/resources/agreement/__init__.py +++ b/golem/resources/agreement/__init__.py @@ -1,11 +1,11 @@ -from golem.resources.agreement.events import NewAgreement, AgreementDataChanged, AgreementClosed -from golem.resources.agreement.pipeline import default_create_activity from golem.resources.agreement.agreement import Agreement +from golem.resources.agreement.events import AgreementClosed, AgreementDataChanged, NewAgreement +from golem.resources.agreement.pipeline import default_create_activity __all__ = ( - Agreement, - NewAgreement, - AgreementDataChanged, - AgreementClosed, - default_create_activity, + "Agreement", + "NewAgreement", + "AgreementDataChanged", + "AgreementClosed", + "default_create_activity", ) diff --git a/golem/resources/agreement/agreement.py b/golem/resources/agreement/agreement.py index 0e71f1b5..d34f5083 100644 --- a/golem/resources/agreement/agreement.py +++ b/golem/resources/agreement/agreement.py @@ -7,12 +7,12 @@ from ya_market.exceptions import ApiException from golem.resources.activity import Activity -from golem.resources.agreement.events import NewAgreement, AgreementClosed -from golem.resources.payment import Invoice -from golem.resources.resources import _NULL, Resource, api_call_wrapper -from golem.resources.resources.base import TModel +from golem.resources.agreement.events import AgreementClosed, NewAgreement +from golem.resources.base import _NULL, Resource, TModel, api_call_wrapper +from golem.resources.invoice import Invoice if TYPE_CHECKING: + from golem.node import GolemNode from golem.resources.market.proposal import Proposal # noqa @@ -70,7 +70,7 @@ async def create_activity( :param autoclose: Destroy the activity when the :any:`GolemNode` closes. :param timeout: Request timeout. """ - from golem.resources.activity.resources import Activity + from golem.resources import Activity activity = await Activity.create(self.node, self.id, timeout) if autoclose: diff --git a/golem/resources/agreement/events.py b/golem/resources/agreement/events.py index b6164e92..b9027b7d 100644 --- a/golem/resources/agreement/events.py +++ b/golem/resources/agreement/events.py @@ -1,3 +1,11 @@ +from typing import TYPE_CHECKING + +from golem.resources.events import NewResource, ResourceClosed, ResourceDataChanged + +if TYPE_CHECKING: + from golem.resources.agreement.agreement import Agreement # noqa + + class NewAgreement(NewResource["Agreement"]): pass diff --git a/golem/resources/agreement/pipeline.py b/golem/resources/agreement/pipeline.py index 5a442fc3..60039ccb 100644 --- a/golem/resources/agreement/pipeline.py +++ b/golem/resources/agreement/pipeline.py @@ -1,3 +1,7 @@ +from golem.resources.activity import Activity +from golem.resources.agreement.agreement import Agreement + + async def default_create_activity(agreement: Agreement) -> Activity: """Create a new :any:`Activity` for a given :any:`Agreement`.""" return await agreement.create_activity() diff --git a/golem/resources/allocation/__init__.py b/golem/resources/allocation/__init__.py index dafe3ace..4a3e803e 100644 --- a/golem/resources/allocation/__init__.py +++ b/golem/resources/allocation/__init__.py @@ -1,13 +1,12 @@ from golem.resources.allocation.allocation import Allocation -from golem.resources.allocation.events import NewAllocation, AllocationDataChanged, AllocationClosed +from golem.resources.allocation.events import AllocationClosed, AllocationDataChanged, NewAllocation from golem.resources.allocation.exceptions import AllocationException, NoMatchingAccount - __all__ = ( - 'Allocation', - 'AllocationException', - 'NoMatchingAccount', - 'NewAllocation', - 'AllocationDataChanged', - 'AllocationClosed', + "Allocation", + "AllocationException", + "NoMatchingAccount", + "NewAllocation", + "AllocationDataChanged", + "AllocationClosed", ) diff --git a/golem/resources/allocation/allocation.py b/golem/resources/allocation/allocation.py index 50e55ee7..d709f387 100644 --- a/golem/resources/allocation/allocation.py +++ b/golem/resources/allocation/allocation.py @@ -1,20 +1,20 @@ import asyncio from datetime import datetime, timedelta, timezone +from decimal import Decimal from typing import TYPE_CHECKING, Optional, Tuple -from decimal import Decimal from ya_payment import RequestorApi, models -from golem.resources.allocation.events import NewAllocation -from golem.resources.allocation.exceptions import NoMatchingAccount from golem.payload.constraints import Constraints from golem.payload.parsers.base import PayloadSyntaxParser from golem.payload.properties import Properties -from golem.resources.resources import _NULL, Resource, ResourceClosed, api_call_wrapper -from golem.resources.resources.base import TModel +from golem.resources.allocation.events import NewAllocation +from golem.resources.allocation.exceptions import NoMatchingAccount +from golem.resources.base import _NULL, Resource, TModel, api_call_wrapper +from golem.resources.events import ResourceClosed if TYPE_CHECKING: - from golem.resources.golem_node import GolemNode + from golem.node import GolemNode class Allocation(Resource[RequestorApi, models.Allocation, _NULL, _NULL, _NULL]): diff --git a/golem/resources/allocation/events.py b/golem/resources/allocation/events.py index b00d0dbd..0fd8d310 100644 --- a/golem/resources/allocation/events.py +++ b/golem/resources/allocation/events.py @@ -1,3 +1,11 @@ +from typing import TYPE_CHECKING + +from golem.resources.events import NewResource, ResourceClosed, ResourceDataChanged + +if TYPE_CHECKING: + from golem.resources.allocation.allocation import Allocation # noqa + + class NewAllocation(NewResource["Allocation"]): pass diff --git a/golem/resources/base.py b/golem/resources/base.py index bd6d7539..ce269e7b 100644 --- a/golem/resources/base.py +++ b/golem/resources/base.py @@ -22,7 +22,7 @@ from golem.resources.events import ResourceDataChanged from golem.resources.exceptions import ResourceNotFound -from golem.node import TRequestorApi, get_requestor_api +from golem.utils.low import TRequestorApi, get_requestor_api if TYPE_CHECKING: from golem.node import GolemNode diff --git a/golem/resources/debit_note/__init__.py b/golem/resources/debit_note/__init__.py index 45687890..4a39a7a1 100644 --- a/golem/resources/debit_note/__init__.py +++ b/golem/resources/debit_note/__init__.py @@ -1,10 +1,11 @@ from golem.resources.debit_note.debit_note import DebitNote -from golem.resources.debit_note.events import NewDebitNote, DebitNoteDataChanged, DebitNoteClosed - +from golem.resources.debit_note.event_collectors import DebitNoteEventCollector +from golem.resources.debit_note.events import DebitNoteClosed, DebitNoteDataChanged, NewDebitNote __all__ = ( - 'DebitNote', - 'NewDebitNote', - 'DebitNoteDataChanged', - 'DebitNoteClosed', + "DebitNote", + "DebitNoteEventCollector", + "NewDebitNote", + "DebitNoteDataChanged", + "DebitNoteClosed", ) diff --git a/golem/resources/debit_note/debit_note.py b/golem/resources/debit_note/debit_note.py index f527149b..faf5d33b 100644 --- a/golem/resources/debit_note/debit_note.py +++ b/golem/resources/debit_note/debit_note.py @@ -4,20 +4,19 @@ from _decimal import Decimal from ya_payment import RequestorApi, models +from golem.resources.allocation import Allocation +from golem.resources.base import _NULL, Resource, TModel, api_call_wrapper from golem.resources.debit_note.events import NewDebitNote -from golem.resources.allocation.allocation import Allocation -from golem.resources.resources import _NULL, Resource, api_call_wrapper -from golem.resources.resources.base import TModel if TYPE_CHECKING: + from golem.node import GolemNode from golem.resources.activity import Activity # noqa - from golem.resources.golem_node import GolemNode class DebitNote(Resource[RequestorApi, models.DebitNote, "Activity", _NULL, _NULL]): """A single debit note on the Golem Network. - Ususally created by a :any:`GolemNode` initialized with `collect_payment_events = True`. + Usually created by a :any:`GolemNode` initialized with `collect_payment_events = True`. """ def __init__(self, node: "GolemNode", id_: str, data: Optional[TModel] = None): diff --git a/golem/resources/debit_note/event_collectors.py b/golem/resources/debit_note/event_collectors.py index 0c2805d2..e90f70ec 100644 --- a/golem/resources/debit_note/event_collectors.py +++ b/golem/resources/debit_note/event_collectors.py @@ -1,7 +1,7 @@ -from typing import Callable, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING, Callable, Tuple from golem.resources.debit_note import DebitNote -from golem.resources.event_collectors import PaymentEventCollector, DebitNoteEvent +from golem.resources.event_collectors import DebitNoteEvent, PaymentEventCollector if TYPE_CHECKING: from golem.resources.activity import Activity diff --git a/golem/resources/debit_note/events.py b/golem/resources/debit_note/events.py index 19e16ce8..7ef6e890 100644 --- a/golem/resources/debit_note/events.py +++ b/golem/resources/debit_note/events.py @@ -1,3 +1,11 @@ +from typing import TYPE_CHECKING + +from golem.resources.events import NewResource, ResourceClosed, ResourceDataChanged + +if TYPE_CHECKING: + from golem.resources.debit_note.debit_note import DebitNote # noqa + + class NewDebitNote(NewResource["DebitNote"]): pass diff --git a/golem/resources/demand/__init__.py b/golem/resources/demand/__init__.py index d9fe88f4..f2b6e70f 100644 --- a/golem/resources/demand/__init__.py +++ b/golem/resources/demand/__init__.py @@ -1,11 +1,12 @@ from golem.resources.demand.demand import Demand, DemandData -from golem.resources.demand.events import NewDemand, DemandDataChanged, DemandClosed - +from golem.resources.demand.demand_builder import DemandBuilder +from golem.resources.demand.events import DemandClosed, DemandDataChanged, NewDemand __all__ = ( - 'Demand', - 'DemandData', - 'NewDemand', - 'DemandDataChanged', - 'DemandClosed', + "Demand", + "DemandBuilder", + "DemandData", + "NewDemand", + "DemandDataChanged", + "DemandClosed", ) diff --git a/golem/resources/demand/demand.py b/golem/resources/demand/demand.py index 35d183ef..41ceaccb 100644 --- a/golem/resources/demand/demand.py +++ b/golem/resources/demand/demand.py @@ -6,21 +6,14 @@ from ya_market import RequestorApi from ya_market import models as models -from golem.resources.demand.events import NewDemand, DemandClosed -from golem.resources.market.proposal import Proposal -from golem.payload.constraints import Constraints -from golem.payload.properties import Properties -from golem.resources.resources import ( - _NULL, - Resource, - ResourceNotFound, - YagnaEventCollector, - api_call_wrapper, -) -from golem.resources.resources.base import TModel +from golem.payload import Constraints, Properties +from golem.resources.base import _NULL, Resource, ResourceNotFound, TModel, api_call_wrapper +from golem.resources.demand.events import DemandClosed, NewDemand +from golem.resources.proposal import Proposal +from golem.utils.low import YagnaEventCollector if TYPE_CHECKING: - from golem.resources.golem_node import GolemNode + from golem.node import GolemNode @dataclass diff --git a/golem/resources/demand/demand_builder.py b/golem/resources/demand/demand_builder.py index 835cb224..69556618 100644 --- a/golem/resources/demand/demand_builder.py +++ b/golem/resources/demand/demand_builder.py @@ -2,16 +2,16 @@ from datetime import datetime, timezone from typing import TYPE_CHECKING, Iterable, Optional, Union -from golem.resources.demand.demand import Demand -from golem.resources.market.resources.demand.demand_offer_base import defaults as dobm_defaults -from golem.resources.market.resources.demand.demand_offer_base.model import DemandOfferBaseModel -from golem.resources.allocation.allocation import Allocation +from golem.payload import Payload +from golem.payload import defaults as payload_defaults from golem.payload.constraints import Constraint, ConstraintGroup, Constraints from golem.payload.parsers.base import PayloadSyntaxParser from golem.payload.properties import Properties +from golem.resources.allocation.allocation import Allocation +from golem.resources.demand.demand import Demand if TYPE_CHECKING: - from golem.resources.golem_node import GolemNode + from golem.node import GolemNode class DemandBuilder: @@ -21,7 +21,8 @@ class DemandBuilder: example usage: ```python - >>> from golem.resources.market import DemandBuilder, pipeline + >>> from golem.resources import DemandBuilder + >>> from golem.payload import defaults >>> from datetime import datetime, timezone >>> builder = DemandBuilder() >>> await builder.add(defaults.NodeInfo(name="a node", subnet_tag="testnet")) @@ -55,10 +56,10 @@ def __eq__(self, other): and self.constraints == other.constraints ) - async def add(self, model: DemandOfferBaseModel): - """Add properties and constraints from the given model to this demand definition.""" + async def add(self, payload: Payload): + """Add properties and constraints from the given payload to this demand definition.""" - properties, constraints = await model.build_properties_and_constraints() + properties, constraints = await payload.build_properties_and_constraints() self.add_properties(properties) self.add_constraints(constraints) @@ -92,7 +93,7 @@ async def add_default_parameters( :param allocations: Allocations that will be included in the description of this demand. """ # FIXME: get rid of local import - from golem.resources.golem_node.golem_node import DEFAULT_EXPIRATION_TIMEOUT, SUBNET + from golem.node import DEFAULT_EXPIRATION_TIMEOUT, SUBNET if subnet is None: subnet = SUBNET @@ -100,8 +101,8 @@ async def add_default_parameters( if expiration is None: expiration = datetime.now(timezone.utc) + DEFAULT_EXPIRATION_TIMEOUT - await self.add(dobm_defaults.ActivityInfo(expiration=expiration, multi_activity=True)) - await self.add(dobm_defaults.NodeInfo(subnet_tag=subnet)) + await self.add(payload_defaults.ActivityInfo(expiration=expiration, multi_activity=True)) + await self.add(payload_defaults.NodeInfo(subnet_tag=subnet)) for allocation in allocations: properties, constraints = await allocation.get_properties_and_constraints_for_demand( diff --git a/golem/resources/demand/events.py b/golem/resources/demand/events.py index 7cd3171d..7ae52466 100644 --- a/golem/resources/demand/events.py +++ b/golem/resources/demand/events.py @@ -1,3 +1,11 @@ +from typing import TYPE_CHECKING + +from golem.resources.events import NewResource, ResourceClosed, ResourceDataChanged + +if TYPE_CHECKING: + from golem.resources.demand.demand import Demand # noqa + + class NewDemand(NewResource["Demand"]): pass diff --git a/golem/resources/event_collectors.py b/golem/resources/event_collectors.py index 41ce0775..9fac642c 100644 --- a/golem/resources/event_collectors.py +++ b/golem/resources/event_collectors.py @@ -1,13 +1,14 @@ from abc import ABC, abstractmethod from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Dict, Tuple, Union, TypeAlias +from typing import TYPE_CHECKING, Any, Dict, Tuple, TypeAlias, Union from ya_payment import models -from golem.resources.resources import Resource, YagnaEventCollector +from golem.resources.base import Resource +from golem.utils.low import YagnaEventCollector if TYPE_CHECKING: - from golem_core.core.golem_node.golem_node import GolemNode + from golem.node import GolemNode InvoiceEvent: TypeAlias = Union[ models.InvoiceReceivedEvent, diff --git a/golem/resources/events.py b/golem/resources/events.py index 5064adb6..870b8673 100644 --- a/golem/resources/events.py +++ b/golem/resources/events.py @@ -4,7 +4,7 @@ from golem.event_bus import Event if TYPE_CHECKING: - from golem.resources.base import Resource + from golem.resources.base import Resource # noqa TResourceEvent = TypeVar("TResourceEvent", bound="ResourceEvent") TResource = TypeVar("TResource", bound="Resource") diff --git a/golem/resources/exceptions.py b/golem/resources/exceptions.py index ff8176bb..4c1fe238 100644 --- a/golem/resources/exceptions.py +++ b/golem/resources/exceptions.py @@ -3,22 +3,13 @@ from golem.exceptions import GolemException if TYPE_CHECKING: - from golem.resources.resources.base import Resource + from golem.resources.base import TResource class ResourceException(GolemException): pass -class MissingConfiguration(ResourceException): - def __init__(self, key: str, description: str): - self._key = key - self._description = description - - def __str__(self) -> str: - return f"Missing configuration for {self._description}. Please set env var {self._key}." - - class ResourceNotFound(ResourceException): """Raised on an attempt to interact with a resource that doesn't exist. @@ -35,13 +26,13 @@ class ResourceNotFound(ResourceException): """ - def __init__(self, resource: "Resource"): + def __init__(self, resource: "TResource"): self._resource = resource msg = f"{resource} doesn't exist" super().__init__(msg) @property - def resource(self) -> "Resource": + def resource(self) -> "TResource": """Resource that caused the exception.""" return self._resource diff --git a/golem/resources/invoice/__init__.py b/golem/resources/invoice/__init__.py index 812b1657..eda7a6ca 100644 --- a/golem/resources/invoice/__init__.py +++ b/golem/resources/invoice/__init__.py @@ -1,10 +1,11 @@ -from golem.resources.invoice.events import NewInvoice, InvoiceDataChanged, InvoiceClosed +from golem.resources.invoice.event_collectors import InvoiceEventCollector +from golem.resources.invoice.events import InvoiceClosed, InvoiceDataChanged, NewInvoice from golem.resources.invoice.invoice import Invoice - __all__ = ( - 'Invoice', - 'NewInvoice', - 'InvoiceDataChanged', - 'InvoiceClosed', + "Invoice", + "InvoiceEventCollector", + "NewInvoice", + "InvoiceDataChanged", + "InvoiceClosed", ) diff --git a/golem/resources/invoice/event_collectors.py b/golem/resources/invoice/event_collectors.py index 351e45a6..c26499e5 100644 --- a/golem/resources/invoice/event_collectors.py +++ b/golem/resources/invoice/event_collectors.py @@ -1,10 +1,10 @@ -from typing import Callable, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING, Callable, Tuple -from golem.resources.event_collectors import PaymentEventCollector, InvoiceEvent +from golem.resources.event_collectors import InvoiceEvent, PaymentEventCollector from golem.resources.invoice.invoice import Invoice if TYPE_CHECKING: - from golem_core.core.market_api import Agreement + from golem.resources.agreement import Agreement class InvoiceEventCollector(PaymentEventCollector): diff --git a/golem/resources/invoice/events.py b/golem/resources/invoice/events.py index d18cdbcc..6d21d762 100644 --- a/golem/resources/invoice/events.py +++ b/golem/resources/invoice/events.py @@ -1,4 +1,9 @@ -from golem.resources.resources import NewResource, ResourceClosed, ResourceDataChanged +from typing import TYPE_CHECKING + +from golem.resources.events import NewResource, ResourceClosed, ResourceDataChanged + +if TYPE_CHECKING: + from golem.resources.invoice.invoice import Invoice # noqa class NewInvoice(NewResource["Invoice"]): diff --git a/golem/resources/invoice/invoice.py b/golem/resources/invoice/invoice.py index 967bbfee..678aafd2 100644 --- a/golem/resources/invoice/invoice.py +++ b/golem/resources/invoice/invoice.py @@ -4,14 +4,13 @@ from ya_payment import RequestorApi, models -from golem.resources.payment.events import NewInvoice from golem.resources.allocation.allocation import Allocation -from golem.resources.resources import _NULL, Resource, api_call_wrapper -from golem.resources.resources.base import TModel +from golem.resources.base import _NULL, Resource, TModel, api_call_wrapper +from golem.resources.invoice.events import NewInvoice if TYPE_CHECKING: - from golem.resources.golem_node import GolemNode - from golem.resources.agreement.agreement import Agreement # noqa + from golem.node import GolemNode + from golem.resources.agreement import Agreement # noqa class Invoice(Resource[RequestorApi, models.Invoice, "Agreement", _NULL, _NULL]): diff --git a/golem/resources/network/__init__.py b/golem/resources/network/__init__.py index b88265d5..168b6c2c 100644 --- a/golem/resources/network/__init__.py +++ b/golem/resources/network/__init__.py @@ -1,10 +1,10 @@ -from golem.resources.network.events import NewNetwork, NetworkDataChanged, NetworkClosed -from golem.resources.network.exceptions import NetworkFull, NetworkException -from golem.resources.network.network import Network - +from golem.resources.network.events import NetworkClosed, NetworkDataChanged, NewNetwork +from golem.resources.network.exceptions import NetworkException, NetworkFull +from golem.resources.network.network import DeployArgsType, Network __all__ = ( "Network", + "DeployArgsType", "NetworkException", "NetworkFull", "NewNetwork", diff --git a/golem/resources/network/events.py b/golem/resources/network/events.py index 9d6aafad..6dd1fb1b 100644 --- a/golem/resources/network/events.py +++ b/golem/resources/network/events.py @@ -1,9 +1,9 @@ from typing import TYPE_CHECKING -from golem.resources.resources import NewResource, ResourceClosed, ResourceDataChanged +from golem.resources.events import NewResource, ResourceClosed, ResourceDataChanged if TYPE_CHECKING: - pass + from golem.resources.network.network import Network # noqa class NewNetwork(NewResource["Network"]): diff --git a/golem/resources/network/network.py b/golem/resources/network/network.py index fe38e0bb..c1146191 100644 --- a/golem/resources/network/network.py +++ b/golem/resources/network/network.py @@ -4,12 +4,12 @@ from ya_net import RequestorApi, models -from golem.resources.network.events import NewNetwork +from golem.resources.base import _NULL, Resource, api_call_wrapper +from golem.resources.network.events import NetworkClosed, NewNetwork from golem.resources.network.exceptions import NetworkFull -from golem.resources.resources import _NULL, Resource, ResourceClosed, api_call_wrapper if TYPE_CHECKING: - from golem.resources.golem_node import GolemNode + from golem.node import GolemNode IpAddress = Union[IPv4Address, IPv6Address] IpNetwork = Union[IPv4Network, IPv6Network] @@ -70,7 +70,7 @@ async def create( async def remove(self) -> None: """Remove the network.""" await self.api.remove_network(self.id) - await self.node.event_bus.emit(ResourceClosed(self)) + await self.node.event_bus.emit(NetworkClosed(self)) @api_call_wrapper() async def create_node(self, provider_id: str, node_ip: Optional[str] = None) -> str: diff --git a/golem/resources/pooling_batch/__init__.py b/golem/resources/pooling_batch/__init__.py index 64016284..a8a06ab6 100644 --- a/golem/resources/pooling_batch/__init__.py +++ b/golem/resources/pooling_batch/__init__.py @@ -1,20 +1,20 @@ -from golem.resources.pooling_batch.events import NewPoolingBatch, BatchFinished +from golem.resources.pooling_batch.events import BatchFinished, NewPoolingBatch from golem.resources.pooling_batch.exceptions import ( - PoolingBatchException, BatchError, - CommandFailed, - CommandCancelled, BatchTimeoutError, + CommandCancelled, + CommandFailed, + PoolingBatchException, ) from golem.resources.pooling_batch.pooling_batch import PoolingBatch __all__ = ( - PoolingBatch, - NewPoolingBatch, - BatchFinished, - PoolingBatchException, - BatchError, - CommandFailed, - CommandCancelled, - BatchTimeoutError, -) \ No newline at end of file + "PoolingBatch", + "NewPoolingBatch", + "BatchFinished", + "PoolingBatchException", + "BatchError", + "CommandFailed", + "CommandCancelled", + "BatchTimeoutError", +) diff --git a/golem/resources/pooling_batch/events.py b/golem/resources/pooling_batch/events.py index 46d2502d..a19f42d6 100644 --- a/golem/resources/pooling_batch/events.py +++ b/golem/resources/pooling_batch/events.py @@ -1,3 +1,11 @@ +from typing import TYPE_CHECKING + +from golem.resources.events import NewResource, ResourceEvent + +if TYPE_CHECKING: + from golem.resources.pooling_batch.pooling_batch import PoolingBatch # noqa + + class NewPoolingBatch(NewResource["PoolingBatch"]): pass diff --git a/golem/resources/pooling_batch/pooling_batch.py b/golem/resources/pooling_batch/pooling_batch.py index 3d598611..3ff94a69 100644 --- a/golem/resources/pooling_batch/pooling_batch.py +++ b/golem/resources/pooling_batch/pooling_batch.py @@ -4,18 +4,19 @@ from ya_activity import models -from golem.resources.pooling_batch.events import NewPoolingBatch, BatchFinished +from golem.resources.base import _NULL, Resource +from golem.resources.pooling_batch.events import BatchFinished, NewPoolingBatch from golem.resources.pooling_batch.exceptions import ( BatchError, BatchTimeoutError, CommandCancelled, CommandFailed, ) -from golem.resources.resources import _NULL, ActivityApi, Resource, YagnaEventCollector +from golem.utils.low import ActivityApi, YagnaEventCollector if TYPE_CHECKING: + from golem.node import GolemNode from golem.resources.activity.activity import Activity # noqa - from golem.resources.golem_node import GolemNode class PoolingBatch( diff --git a/golem/resources/proposal/__init__.py b/golem/resources/proposal/__init__.py index 220556ff..fde6ef7d 100644 --- a/golem/resources/proposal/__init__.py +++ b/golem/resources/proposal/__init__.py @@ -1,13 +1,13 @@ -from golem.resources.proposal.events import NewProposal, ProposalDataChanged, ProposalClosed -from golem.resources.proposal.pipeline import default_negotiate, default_create_agreement +from golem.resources.proposal.events import NewProposal, ProposalClosed, ProposalDataChanged +from golem.resources.proposal.pipeline import default_create_agreement, default_negotiate from golem.resources.proposal.proposal import Proposal, ProposalData __all__ = ( - Proposal, - ProposalData, - NewProposal, - ProposalDataChanged, - ProposalClosed, - default_negotiate, - default_create_agreement, + "Proposal", + "ProposalData", + "NewProposal", + "ProposalDataChanged", + "ProposalClosed", + "default_negotiate", + "default_create_agreement", ) diff --git a/golem/resources/proposal/events.py b/golem/resources/proposal/events.py index 686e2fe6..0344951a 100644 --- a/golem/resources/proposal/events.py +++ b/golem/resources/proposal/events.py @@ -1,4 +1,9 @@ -from golem.resources.resources import NewResource, ResourceClosed, ResourceDataChanged +from typing import TYPE_CHECKING + +from golem.resources.events import NewResource, ResourceClosed, ResourceDataChanged + +if TYPE_CHECKING: + from golem.resources.proposal.proposal import Proposal # noqa class NewProposal(NewResource["Proposal"]): diff --git a/golem/resources/proposal/pipeline.py b/golem/resources/proposal/pipeline.py index e6f525a0..67003a9e 100644 --- a/golem/resources/proposal/pipeline.py +++ b/golem/resources/proposal/pipeline.py @@ -1,4 +1,5 @@ -from golem.resources.market.resources import Agreement, Proposal +from golem.resources.agreement import Agreement +from golem.resources.proposal.proposal import Proposal async def default_negotiate(proposal: Proposal) -> Proposal: @@ -18,5 +19,3 @@ async def default_create_agreement(proposal: Proposal) -> Agreement: if approved: return agreement raise Exception(f"Agreement {agreement} created from {proposal} was not approved") - - diff --git a/golem/resources/proposal/proposal.py b/golem/resources/proposal/proposal.py index 5dab54e6..ed4a4c01 100644 --- a/golem/resources/proposal/proposal.py +++ b/golem/resources/proposal/proposal.py @@ -6,16 +6,14 @@ from ya_market import RequestorApi from ya_market import models as models -from golem.resources.market.events import NewProposal -from golem.resources.agreement.agreement import Agreement -from golem.payload.constraints import Constraints -from golem.payload.properties import Properties -from golem.resources.resources import Resource -from golem.resources.resources.base import TModel, api_call_wrapper +from golem.payload import Constraints, Properties +from golem.resources.agreement import Agreement +from golem.resources.base import Resource, TModel, api_call_wrapper +from golem.resources.proposal.events import NewProposal if TYPE_CHECKING: - from golem.resources.golem_node import GolemNode - from golem.resources.market.resources.demand import Demand + from golem.node import GolemNode + from golem.resources.demand import Demand ProposalState = Literal["Initial", "Draft", "Rejected", "Accepted", "Expired"] @@ -91,7 +89,7 @@ def demand(self) -> "Demand": # and then _demand is always set, or a Proposal-parent or a Demand-parent. # FIXME: remove local import - from golem.resources.market.resources.demand import Demand + from golem.resources import Demand if self._demand is not None: return self._demand diff --git a/golem/utils/low/__init__.py b/golem/utils/low/__init__.py new file mode 100644 index 00000000..7fb03368 --- /dev/null +++ b/golem/utils/low/__init__.py @@ -0,0 +1,11 @@ +from golem.utils.low.api import ActivityApi, ApiConfig, ApiFactory, TRequestorApi, get_requestor_api +from golem.utils.low.event_collector import YagnaEventCollector + +__all__ = ( + "YagnaEventCollector", + "TRequestorApi", + "ActivityApi", + "ApiConfig", + "ApiFactory", + "get_requestor_api", +) diff --git a/golem/node/api.py b/golem/utils/low/api.py similarity index 97% rename from golem/node/api.py rename to golem/utils/low/api.py index f77e4bdd..779e3f13 100644 --- a/golem/node/api.py +++ b/golem/utils/low/api.py @@ -8,11 +8,11 @@ import ya_net import ya_payment -from golem.resources.resources.exceptions import MissingConfiguration +from golem.exceptions import MissingConfiguration if TYPE_CHECKING: - from golem.resources.golem_node import GolemNode - from golem.resources.resources import Resource + from golem.node import GolemNode + from golem.resources import Resource TRequestorApi = TypeVar("TRequestorApi") diff --git a/golem/node/event_collectors/base.py b/golem/utils/low/event_collector.py similarity index 63% rename from golem/node/event_collectors/base.py rename to golem/utils/low/event_collector.py index bca4bd7c..ea164866 100644 --- a/golem/node/event_collectors/base.py +++ b/golem/utils/low/event_collector.py @@ -1,13 +1,14 @@ import asyncio +import json from abc import ABC, abstractmethod # TODO: replace Any here -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, Union -from golem.node.event_collectors.utils import ( - is_gsb_endpoint_not_found_error, - is_intermittent_error, -) +import aiohttp +import ya_activity +import ya_market +import ya_payment class YagnaEventCollector(ABC): @@ -65,3 +66,36 @@ def _collect_events_args(self) -> List: def _collect_events_kwargs(self) -> Dict: return {} + + +def is_intermittent_error(e: Exception) -> bool: + """Check if `e` indicates an intermittent communication failure such as network timeout.""" + + is_timeout_exception = isinstance(e, asyncio.TimeoutError) or ( + isinstance( + e, + (ya_activity.ApiException, ya_market.ApiException, ya_payment.ApiException), + ) + and e.status in (408, 504) + ) + + return ( + is_timeout_exception + or isinstance(e, aiohttp.ServerDisconnectedError) + # OS error with errno 32 is "Broken pipe" + or (isinstance(e, aiohttp.ClientOSError) and e.errno == 32) + ) + + +def is_gsb_endpoint_not_found_error( + err: Union[ya_activity.ApiException, ya_market.ApiException, ya_payment.ApiException] +) -> bool: + """Check if `err` is caused by "Endpoint address not found" GSB error.""" + + if err.status != 500: + return False + try: + msg = json.loads(err.body)["message"] + return "GSB error" in msg and "endpoint address not found" in msg + except Exception: + return False diff --git a/golem/utils/storage/__init__.py b/golem/utils/storage/__init__.py index 4d217c90..1d1d799a 100644 --- a/golem/utils/storage/__init__.py +++ b/golem/utils/storage/__init__.py @@ -1,7 +1,6 @@ -from golem.utils.storage.gftp import GftpProvider +from golem.utils.storage.base import Destination, Source __all__ = ( "Destination", "Source", - "GftpProvider", ) diff --git a/golem/utils/storage/gftp/__init__.py b/golem/utils/storage/gftp/__init__.py index e69de29b..06d4cd6d 100644 --- a/golem/utils/storage/gftp/__init__.py +++ b/golem/utils/storage/gftp/__init__.py @@ -0,0 +1,3 @@ +from golem.utils.storage.gftp.provider import GftpProvider + +__all__ = ("GftpProvider",) diff --git a/golem/utils/storage/gftp/provider.py b/golem/utils/storage/gftp/provider.py index dfe041c1..51c1057e 100644 --- a/golem/utils/storage/gftp/provider.py +++ b/golem/utils/storage/gftp/provider.py @@ -29,8 +29,8 @@ from async_exit_stack import AsyncExitStack from typing_extensions import Literal, Protocol, TypedDict -from golem.storage.base import Content, Destination, Source, StorageProvider -from golem.storage.utils import strtobool +from golem.utils.storage.base import Content, Destination, Source, StorageProvider +from golem.utils.storage.utils import strtobool _logger = logging.getLogger(__name__) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index de6cd04e..ce694d3b 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -1,6 +1,7 @@ from contextlib import asynccontextmanager from typing import AsyncGenerator, Optional +from golem.pipeline import Chain, Map from golem.resources.activity import Activity from golem.resources.golem_node import GolemNode from golem.resources.market import RepositoryVmPayload @@ -9,7 +10,6 @@ default_create_agreement, default_negotiate, ) -from golem.pipeline import Chain, Map ANY_PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/tests/unit/test_app_session_id.py b/tests/unit/test_app_session_id.py index ffeff20c..ffa92b16 100644 --- a/tests/unit/test_app_session_id.py +++ b/tests/unit/test_app_session_id.py @@ -1,4 +1,4 @@ -from golem.resources.golem_node import GolemNode +from golem.node import GolemNode def test_different_app_session_id() -> None: diff --git a/tests/unit/test_command.py b/tests/unit/test_command.py index 109c55a4..046aa3c0 100644 --- a/tests/unit/test_command.py +++ b/tests/unit/test_command.py @@ -2,7 +2,7 @@ import pytest -from golem.resources.activity import Run +from golem.resources import Run list_c = ["echo", "foo"] str_c = "echo foo" diff --git a/tests/unit/test_demand_builder.py b/tests/unit/test_demand_builder.py index 7fa01c89..b2ac00b9 100644 --- a/tests/unit/test_demand_builder.py +++ b/tests/unit/test_demand_builder.py @@ -1,12 +1,19 @@ from dataclasses import dataclass -from golem.resources.market import DemandBuilder, DemandOfferBaseModel, constraint, prop -from golem.payload.constraints import Constraint, Constraints, ConstraintGroup -from golem.payload.properties import Properties +from golem.payload import ( + Constraint, + ConstraintGroup, + Constraints, + Payload, + Properties, + constraint, + prop, +) +from golem.resources import DemandBuilder @dataclass -class ExampleModel(DemandOfferBaseModel): +class ExampleModel(Payload): prop1: int = prop("some.prop1.path") prop2: int = prop("some.prop2.path") con1: int = constraint("some.con1.path", "=") @@ -162,7 +169,7 @@ async def test_create_demand(mocker): mocked_node = mocker.Mock() mocked_demand = mocker.patch( - "golem.resources.market.resources.demand.demand_builder.Demand", + "golem.resources.demand.demand_builder.Demand", **{"create_from_properties_constraints": mocker.AsyncMock(return_value="foobar")}, ) diff --git a/tests/unit/test_event_bus.py b/tests/unit/test_event_bus.py index b1aad0cd..6c950d53 100644 --- a/tests/unit/test_event_bus.py +++ b/tests/unit/test_event_bus.py @@ -3,7 +3,7 @@ import pytest -from golem.resources.events.base import Event, EventBusError +from golem.event_bus import Event, EventBusError from golem.event_bus.in_memory import InMemoryEventBus from golem.utils.logging import DEFAULT_LOGGING diff --git a/tests/unit/test_demand_builder_model.py b/tests/unit/test_payload.py similarity index 89% rename from tests/unit/test_demand_builder_model.py rename to tests/unit/test_payload.py index a8def04c..6446a9ea 100644 --- a/tests/unit/test_demand_builder_model.py +++ b/tests/unit/test_payload.py @@ -5,14 +5,15 @@ import pytest -from golem.resources.market import ( - DemandOfferBaseModel, - InvalidPropertiesError, +from golem.payload import ( + Constraint, + Constraints, + InvalidProperties, + Payload, + Properties, constraint, prop, ) -from golem.payload.constraints import Constraint, Constraints -from golem.payload.properties import Properties class ExampleEnum(Enum): @@ -21,7 +22,7 @@ class ExampleEnum(Enum): @dataclass -class Foo(DemandOfferBaseModel): +class Foo(Payload): bar: str = prop("bar.dotted.path", default="cafebiba") max_baz: int = constraint("baz", "<=", default=100) min_baz: int = constraint("baz", ">=", default=1) @@ -29,7 +30,7 @@ class Foo(DemandOfferBaseModel): @dataclass -class FooToo(DemandOfferBaseModel): +class FooToo(Payload): text: str = prop("some.path") baz: int = constraint("baz", "=", default=21) en: ExampleEnum = prop("some_enum", default=ExampleEnum.TWO) @@ -46,7 +47,7 @@ def __post_init__(self): @dataclass -class FooZero(DemandOfferBaseModel): +class FooZero(Payload): pass @@ -117,7 +118,7 @@ def test_from_properties(): def test_from_properties_missing_key(): - with pytest.raises(InvalidPropertiesError, match="Missing key"): + with pytest.raises(InvalidProperties, match="Missing key"): FooToo.from_properties( { "some_enum": "one", @@ -128,7 +129,7 @@ def test_from_properties_missing_key(): def test_from_properties_custom_validation(): - with pytest.raises(InvalidPropertiesError, match="validation error"): + with pytest.raises(InvalidProperties, match="validation error"): FooToo.from_properties( { "some.path": "blow up please!", diff --git a/tests/unit/test_demand_offer_cons.py b/tests/unit/test_payload_cons.py similarity index 93% rename from tests/unit/test_demand_offer_cons.py rename to tests/unit/test_payload_cons.py index df23abae..fb996b41 100644 --- a/tests/unit/test_demand_offer_cons.py +++ b/tests/unit/test_payload_cons.py @@ -3,12 +3,7 @@ import pytest -from golem.payload.constraints import ( - Constraint, - ConstraintException, - ConstraintGroup, - Constraints, -) +from golem.payload import Constraint, ConstraintException, ConstraintGroup, Constraints class ExampleEnum(Enum): diff --git a/tests/unit/test_demand_offer_parsers.py b/tests/unit/test_payload_parsers.py similarity index 94% rename from tests/unit/test_demand_offer_parsers.py rename to tests/unit/test_payload_parsers.py index 86c2ccd2..b14d6d77 100644 --- a/tests/unit/test_demand_offer_parsers.py +++ b/tests/unit/test_payload_parsers.py @@ -1,13 +1,12 @@ import pytest -from golem.payload.constraints import Constraint, ConstraintException, ConstraintGroup -from golem.payload.parsers import SyntaxException -from golem.payload.parsers import TextXDemandOfferSyntaxParser +from golem.payload import Constraint, ConstraintException, ConstraintGroup, SyntaxException +from golem.payload.parsers.textx import TextXPayloadSyntaxParser @pytest.fixture(scope="module") def demand_offer_parser(): - return TextXDemandOfferSyntaxParser() + return TextXPayloadSyntaxParser() def test_parse_raises_exception_on_bad_syntax(demand_offer_parser): diff --git a/tests/unit/test_demand_offer_props.py b/tests/unit/test_payload_props.py similarity index 96% rename from tests/unit/test_demand_offer_props.py rename to tests/unit/test_payload_props.py index bbed20ca..6a8bbb9d 100644 --- a/tests/unit/test_demand_offer_props.py +++ b/tests/unit/test_payload_props.py @@ -1,7 +1,7 @@ from datetime import datetime from enum import Enum -from golem.payload.properties import Properties +from golem.payload import Properties class ExampleEnum(Enum): diff --git a/tests/unit/utils/test_storage.py b/tests/unit/utils/test_storage.py index 82f35fa5..54322b08 100644 --- a/tests/unit/utils/test_storage.py +++ b/tests/unit/utils/test_storage.py @@ -7,7 +7,7 @@ import pytest -from golem.utils.storage.gftp import provider +from golem.utils.storage.gftp import provider as gftp @pytest.fixture(scope="function") From b9cd6d56088c401bdc0c3c40575d5a60e99d7756 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 28 Jun 2023 12:57:49 +0200 Subject: [PATCH 062/123] Lambda and func negotiation plugin example --- examples/managers/basic_composition.py | 37 ++++++++++++++++++------ golem/managers/base.py | 8 ++++- golem/managers/negotiation/sequential.py | 19 ++++++++---- 3 files changed, 48 insertions(+), 16 deletions(-) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index 8d3d6145..4f1012b7 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -1,11 +1,11 @@ import asyncio import logging.config from random import randint -from typing import List +from typing import List, Optional from golem.managers.activity.single_use import SingleUseActivityManager from golem.managers.agreement.single_use import SingleUseAgreementManager -from golem.managers.base import WorkContext, WorkResult +from golem.managers.base import RejectProposal, WorkContext, WorkResult from golem.managers.negotiation import SequentialNegotiationManager from golem.managers.negotiation.plugins import AddChosenPaymentPlatform, BlacklistProviderId from golem.managers.payment.pay_all import PayAllPaymentManager @@ -18,8 +18,24 @@ from golem.managers.work.sequential import SequentialWorkManager from golem.node import GolemNode from golem.payload import RepositoryVmPayload +from golem.resources.demand.demand import DemandData +from golem.resources.proposal.proposal import ProposalData from golem.utils.logging import DEFAULT_LOGGING +BLACKLISTED_PROVIDERS = [ + "0x3b0f605fcb0690458064c10346af0c5f6b7202a5", + "0x7ad8ce2f95f69be197d136e308303d2395e68379", + "0x40f401ead13eabe677324bf50605c68caabb22c7", +] + + +async def blacklist_func( + demand_data: DemandData, proposal_data: ProposalData +) -> Optional[RejectProposal]: + provider_id = proposal_data.issuer_id + if provider_id in BLACKLISTED_PROVIDERS: + raise RejectProposal(f"Provider ID `{provider_id}` is blacklisted by the requestor") + async def commands_work_example(context: WorkContext) -> str: r = await context.run("echo 'hello golem'") @@ -63,13 +79,16 @@ async def main(): payload, plugins=[ AddChosenPaymentPlatform(), - BlacklistProviderId( - [ - "0x3b0f605fcb0690458064c10346af0c5f6b7202a5", - "0x7ad8ce2f95f69be197d136e308303d2395e68379", - "0x40f401ead13eabe677324bf50605c68caabb22c7", - ] - ), + # class based plugin + BlacklistProviderId(BLACKLISTED_PROVIDERS), + # func plugin + blacklist_func, + # lambda plugin + lambda _, proposal_data: proposal_data.issuer_id not in BLACKLISTED_PROVIDERS, + # lambda plugin with reject reason + lambda _, proposal_data: RejectProposal(f"Blacklisting {proposal_data.issuer_id}") + if proposal_data.issuer_id in BLACKLISTED_PROVIDERS + else None, ], ) proposal_manager = StackProposalManager(golem, negotiation_manager.get_proposal) diff --git a/golem/managers/base.py b/golem/managers/base.py index 6e2c0b90..77bd8018 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -152,7 +152,13 @@ class WorkManager(Manager, ABC): ... +class RejectProposal(Exception): + pass + + class NegotiationPlugin(ABC): @abstractmethod - async def __call__(self, demand_data: DemandData, proposal_data: ProposalData) -> None: + def __call__( + self, demand_data: DemandData, proposal_data: ProposalData + ) -> Union[Awaitable[Optional[RejectProposal]], Optional[RejectProposal]]: ... diff --git a/golem/managers/negotiation/sequential.py b/golem/managers/negotiation/sequential.py index dca5fa08..8a373e14 100644 --- a/golem/managers/negotiation/sequential.py +++ b/golem/managers/negotiation/sequential.py @@ -6,7 +6,12 @@ from ya_market import ApiException -from golem.managers.base import ManagerException, NegotiationManager, NegotiationPlugin +from golem.managers.base import ( + ManagerException, + NegotiationManager, + NegotiationPlugin, + RejectProposal, +) from golem.node import GolemNode from golem.payload import Payload, Properties from golem.payload.parsers.textx import TextXPayloadSyntaxParser @@ -16,10 +21,6 @@ logger = logging.getLogger(__name__) -class RejectProposal(Exception): - pass - - class SequentialNegotiationManager(NegotiationManager): def __init__( self, @@ -138,7 +139,13 @@ async def _negotiate_proposal( logger.debug(f"Applying plugins on `{offer_proposal}`...") for plugin in self._plugins: - await plugin(demand_data_after_plugins, proposal_data) + plugin_result = plugin(demand_data_after_plugins, proposal_data) + if asyncio.iscoroutine(plugin_result): + plugin_result = await plugin_result + if isinstance(plugin_result, RejectProposal): + raise plugin_result + if plugin_result is False: + raise RejectProposal() except RejectProposal as e: logger.debug( From 1af4128b6dcc3270cf9dfb08d7dd2f6ce967e235 Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 28 Jun 2023 13:22:25 +0200 Subject: [PATCH 063/123] work decorator -> work plugin --- docs/sphinx/api.rst | 86 +++++++++---------- examples/managers/basic_composition.py | 11 +-- examples/managers/ssh.py | 4 +- golem/cli/utils.py | 3 + golem/event_bus/in_memory/event_bus.py | 17 ++-- golem/managers/activity/single_use.py | 4 +- golem/managers/base.py | 27 +++++- golem/managers/negotiation/plugins.py | 2 +- golem/managers/negotiation/sequential.py | 5 +- golem/managers/payment/default.py | 2 +- golem/managers/work/__init__.py | 8 +- .../work/{decorators.py => plugins.py} | 14 +-- golem/managers/work/sequential.py | 52 ++++++++--- golem/payload/base.py | 4 +- golem/payload/mixins.py | 3 +- golem/resources/agreement/agreement.py | 2 +- golem/resources/demand/demand_builder.py | 3 +- golem/utils/logging.py | 5 +- pyproject.toml | 6 +- tests/integration/helpers.py | 8 +- tests/integration/test_1.py | 7 +- tests/integration/test_app_session_id.py | 5 +- tests/unit/test_payload_parsers.py | 2 +- 23 files changed, 162 insertions(+), 118 deletions(-) rename golem/managers/work/{decorators.py => plugins.py} (81%) diff --git a/docs/sphinx/api.rst b/docs/sphinx/api.rst index c5fb4e17..6dc33ef2 100644 --- a/docs/sphinx/api.rst +++ b/docs/sphinx/api.rst @@ -7,7 +7,7 @@ Golem Python Core API Reference GolemNode ========= -.. autoclass:: golem_core.core.golem_node.GolemNode +.. autoclass:: golem.node.GolemNode :members: __init__, __aenter__, __aexit__, create_allocation, create_demand, create_network, allocation, debit_note, invoice, @@ -19,7 +19,7 @@ GolemNode .. High-Level API .. ============== -.. .. autofunction:: golem_core.high.execute_tasks.execute_tasks +.. .. autofunction:: golem.high.execute_tasks.execute_tasks Mid-level API ============= @@ -33,15 +33,15 @@ General components Classes in this section know nothing about any Golem-specific logic. They should one day be extracted to a sparate library. -.. autoclass:: golem_core.pipeline.Sort +.. autoclass:: golem.pipeline.Sort :members: __init__, __call__ -.. autoclass:: golem_core.pipeline.Chain -.. autoclass:: golem_core.pipeline.Map +.. autoclass:: golem.pipeline.Chain +.. autoclass:: golem.pipeline.Map :members: __init__, __call__ -.. autoclass:: golem_core.pipeline.Zip -.. autoclass:: golem_core.pipeline.Buffer +.. autoclass:: golem.pipeline.Zip +.. autoclass:: golem.pipeline.Buffer :members: __init__, __call__ -.. autoclass:: golem_core.pipeline.Limit +.. autoclass:: golem.pipeline.Limit :members: __init__ @@ -49,10 +49,10 @@ Golem-specific components ------------------------------ Components in this section contain the common logic that is shared by various Golem applications. -.. autofunction:: golem_core.core.market_api.default_negotiate -.. autofunction:: golem_core.core.market_api.default_create_agreement -.. autofunction:: golem_core.core.market_api.default_create_activity -.. autofunction:: golem_core.core.activity_api.default_prepare_activity +.. autofunction:: golem.resources.default_negotiate +.. autofunction:: golem.resources.default_create_agreement +.. autofunction:: golem.resources.default_create_activity +.. autofunction:: golem.resources.default_prepare_activity Low-level API @@ -67,7 +67,7 @@ by performing operations on the low-level objects. Resource -------- -.. autoclass:: golem_core.core.resources.Resource +.. autoclass:: golem.resources.Resource :members: id, node, get_data, data, parent, children, child_aiter, @@ -76,103 +76,97 @@ Resource Market API ---------- -.. autoclass:: golem_core.core.market_api.Demand +.. autoclass:: golem.resources.Demand :members: initial_proposals, unsubscribe, proposal -.. autoclass:: golem_core.core.market_api.Proposal +.. autoclass:: golem.resources.Proposal :members: initial, draft, rejected, demand, respond, responses, reject, create_agreement -.. autoclass:: golem_core.core.market_api.Agreement +.. autoclass:: golem.resources.Agreement :members: confirm, wait_for_approval, terminate, create_activity, invoice, activities, close_all Payment API ----------- -.. autoclass:: golem_core.core.payment_api.Allocation +.. autoclass:: golem.resources.Allocation :members: release -.. autoclass:: golem_core.core.payment_api.DebitNote +.. autoclass:: golem.resources.DebitNote :members: accept_full -.. autoclass:: golem_core.core.payment_api.Invoice +.. autoclass:: golem.resources.Invoice :members: accept_full Activity API ------------ -.. autoclass:: golem_core.core.activity_api.Activity +.. autoclass:: golem.resources.Activity :members: execute_commands, execute_script, destroy, idle, destroyed, wait_busy, wait_idle, wait_destroyed, debit_notes, batch -.. autoclass:: golem_core.core.activity_api.PoolingBatch +.. autoclass:: golem.resources.PoolingBatch :members: wait, events, done, success -.. autoclass:: golem_core.core.activity_api.Script +.. autoclass:: golem.resources.Script :members: add_command Network API ----------- -.. autoclass:: golem_core.core.network_api.Network +.. autoclass:: golem.resources.Network :members: create_node, deploy_args, refresh_nodes, remove Commands -------- -.. autoclass:: golem_core.core.activity_api.commands.Command -.. autoclass:: golem_core.core.activity_api.commands.Deploy -.. autoclass:: golem_core.core.activity_api.commands.Start -.. autoclass:: golem_core.core.activity_api.commands.Run +.. autoclass:: golem.resources.Command +.. autoclass:: golem.resources.Deploy +.. autoclass:: golem.resources.Start +.. autoclass:: golem.resources.Run :members: __init__ -.. autoclass:: golem_core.core.activity_api.commands.SendFile +.. autoclass:: golem.resources.SendFile :members: __init__ -.. autoclass:: golem_core.core.activity_api.commands.DownloadFile +.. autoclass:: golem.resources.DownloadFile :members: __init__ Exceptions ---------- -.. autoclass:: golem_core.core.resources.ResourceNotFound +.. autoclass:: golem.resources.ResourceNotFound :members: resource -.. autoclass:: golem_core.core.payment_api.NoMatchingAccount +.. autoclass:: golem.resources.NoMatchingAccount :members: network, driver -.. autoclass:: golem_core.core.activity_api.BatchTimeoutError +.. autoclass:: golem.resources.BatchTimeoutError :members: batch, timeout -.. autoclass:: golem_core.core.activity_api.BatchError +.. autoclass:: golem.resources.BatchError :members: batch -.. autoclass:: golem_core.core.activity_api.CommandFailed +.. autoclass:: golem.resources.CommandFailed :members: batch -.. autoclass:: golem_core.core.activity_api.CommandCancelled +.. autoclass:: golem.resources.CommandCancelled :members: batch -.. autoclass:: golem_core.core.network_api.NetworkFull +.. autoclass:: golem.resources.NetworkFull :members: network Events ====== -.. autoclass:: golem_core.core.events.EventBus - :members: listen, resource_listen, emit - -.. automodule:: golem_core.core.events.event - :members: - -.. automodule:: golem_core.core.activity_api.events +.. automodule:: golem.event_bus.base :members: -.. automodule:: golem_core.core.resources.events +.. automodule:: golem.resources.events :members: Logging ======= -.. autoclass:: golem_core.utils.logging.DefaultLogger +.. autoclass:: golem.utils.logging.DefaultLogger :members: __init__, file_name, logger, on_event Managers ======== -.. autoclass:: golem_core.managers.DefaultPaymentManager +.. autoclass:: golem.managers.DefaultPaymentManager :members: __init__, terminate_agreements, wait_for_invoices diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index 4f1012b7..ed52be31 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -10,11 +10,7 @@ from golem.managers.negotiation.plugins import AddChosenPaymentPlatform, BlacklistProviderId from golem.managers.payment.pay_all import PayAllPaymentManager from golem.managers.proposal import StackProposalManager -from golem.managers.work.decorators import ( - redundancy_cancel_others_on_first_done, - retry, - work_decorator, -) +from golem.managers.work.plugins import redundancy_cancel_others_on_first_done, retry, work_plugin from golem.managers.work.sequential import SequentialWorkManager from golem.node import GolemNode from golem.payload import RepositoryVmPayload @@ -46,8 +42,7 @@ async def commands_work_example(context: WorkContext) -> str: return result -@work_decorator(redundancy_cancel_others_on_first_done(size=2)) -@work_decorator(retry(tries=5)) +@work_plugin(redundancy_cancel_others_on_first_done(size=2)) async def batch_work_example(context: WorkContext): if randint(0, 1): raise Exception("Random fail") @@ -94,7 +89,7 @@ async def main(): proposal_manager = StackProposalManager(golem, negotiation_manager.get_proposal) agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) activity_manager = SingleUseActivityManager(golem, agreement_manager.get_agreement) - work_manager = SequentialWorkManager(golem, activity_manager.do_work) + work_manager = SequentialWorkManager(golem, activity_manager.do_work, plugins=[retry(tries=5)]) async with golem: async with payment_manager, negotiation_manager, proposal_manager: diff --git a/examples/managers/ssh.py b/examples/managers/ssh.py index 808e09a1..7ebb37c5 100644 --- a/examples/managers/ssh.py +++ b/examples/managers/ssh.py @@ -45,7 +45,9 @@ async def _work(context: WorkContext) -> str: print( "Connect with:\n" - f" ssh -o ProxyCommand='websocat asyncstdio: {await get_provider_uri(context._activity.parent.parent.data.issuer_id, 'ws')} --binary " + " ssh -o ProxyCommand='websocat asyncstdio: " + f"{await get_provider_uri(context._activity.parent.parent.data.issuer_id, 'ws')}" + f" --binary " f'-H=Authorization:"Bearer {app_key}"\' root@{uuid4().hex} ' ) print(f"PASSWORD: {password}") diff --git a/golem/cli/utils.py b/golem/cli/utils.py index fca14003..c510e95d 100644 --- a/golem/cli/utils.py +++ b/golem/cli/utils.py @@ -9,6 +9,9 @@ from typing_extensions import Concatenate, ParamSpec from golem.node import GolemNode +from golem.payload import Payload, constraint +from golem.payload.defaults import RUNTIME_NAME +from golem.resources import Allocation, Demand, Proposal def format_allocations(allocations: List[Allocation]) -> str: diff --git a/golem/event_bus/in_memory/event_bus.py b/golem/event_bus/in_memory/event_bus.py index 6e2e0f5b..fd527346 100644 --- a/golem/event_bus/in_memory/event_bus.py +++ b/golem/event_bus/in_memory/event_bus.py @@ -65,7 +65,8 @@ async def on( filter_func: Optional[Callable[[TEvent], bool]] = None, ) -> TCallbackHandler: logger.debug( - f"Adding callback handler for `{event_type}` with callback `{callback}` and filter `{filter_func}`..." + f"Adding callback handler for `{event_type}` with callback `{callback}`" + f" and filter `{filter_func}`..." ) callback_info = _CallbackInfo( @@ -89,7 +90,8 @@ async def on_once( filter_func: Optional[Callable[[TEvent], bool]] = None, ) -> TCallbackHandler: logger.debug( - f"Adding one-time callback handler for `{event_type}` with callback `{callback}` and filter `{filter_func}`..." + f"Adding one-time callback handler for `{event_type}` with callback `{callback}`" + f" and filter `{filter_func}`..." ) callback_info = _CallbackInfo( @@ -158,7 +160,8 @@ async def _process_event( if not isinstance(event, event_type): logger.debug( - f"Processing event `{event}` on event type `{event_type}` ignored as event is not a instance of event type" + f"Processing event `{event}` on event type `{event_type}` ignored as event is" + f" not a instance of event type" ) return @@ -175,9 +178,10 @@ async def _process_event( if not callback_info.filter_func(event): logger.debug("Calling filter function done, ignoring callback") continue - except: + except Exception: logger.exception( - f"Encountered an error in `{callback_info.filter_func}` filter function while handling `{event}`!" + f"Encountered an error in `{callback_info.filter_func}` filter function" + f" while handling `{event}`!" ) continue else: @@ -192,7 +196,8 @@ async def _process_event( logger.debug(f"Calling {callback_info.callback} failed with `{e}") logger.exception( - f"Encountered an error in `{callback_info.callback}` callback while handling `{event}`!" + f"Encountered an error in `{callback_info.callback}` callback" + f" while handling `{event}`!" ) continue else: diff --git a/golem/managers/activity/single_use.py b/golem/managers/activity/single_use.py index 7b03f29c..de77bbce 100644 --- a/golem/managers/activity/single_use.py +++ b/golem/managers/activity/single_use.py @@ -53,9 +53,7 @@ async def _prepare_activity(self) -> Activity: break except Exception: - logger.exception( - f"Creating activity failed, but will be retried with new agreement" - ) + logger.exception("Creating activity failed, but will be retried with new agreement") finally: event = AgreementReleased(agreement) diff --git a/golem/managers/base.py b/golem/managers/base.py index 77bd8018..f0e6bb9a 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import Any, Awaitable, Callable, Dict, List, Optional, Union +from typing import Any, Awaitable, Callable, Dict, Generic, List, Optional, Sequence, TypeVar, Union from golem.exceptions import GolemException from golem.resources import ( @@ -78,11 +78,11 @@ class WorkResult: extras: Dict = field(default_factory=dict) -WorkDecorator = Callable[["DoWorkCallable"], "DoWorkCallable"] +WORK_PLUGIN_FIELD_NAME = "_work_plugins" class Work(ABC): - _work_decorators: Optional[List[WorkDecorator]] + _work_plugins: Optional[List["WorkPlugin"]] def __call__(self, context: WorkContext) -> Awaitable[Optional[WorkResult]]: ... @@ -91,6 +91,11 @@ def __call__(self, context: WorkContext) -> Awaitable[Optional[WorkResult]]: DoWorkCallable = Callable[[Work], Awaitable[WorkResult]] +class WorkPlugin(ABC): + def __call__(self, do_work: DoWorkCallable) -> DoWorkCallable: + ... + + class ManagerEvent(ResourceEvent, ABC): pass @@ -114,6 +119,22 @@ async def stop(self): ... +TPlugin = TypeVar("TPlugin") + + +class ManagerPluginsMixin(Generic[TPlugin]): + def __init__(self, plugins: Optional[Sequence[TPlugin]] = None, *args, **kwargs) -> None: + self._plugins: List[TPlugin] = list(plugins) if plugins is not None else [] + + super().__init__(*args, **kwargs) + + def register_plugin(self, plugin: TPlugin): + self._plugins.append(plugin) + + def unregister_plugin(self, plugin: TPlugin): + self._plugins.remove(plugin) + + class NetworkManager(Manager, ABC): ... diff --git a/golem/managers/negotiation/plugins.py b/golem/managers/negotiation/plugins.py index bfa75b32..0280c059 100644 --- a/golem/managers/negotiation/plugins.py +++ b/golem/managers/negotiation/plugins.py @@ -44,7 +44,7 @@ async def __call__(self, demand_data: DemandData, proposal_data: ProposalData) - common_platforms = list(demand_platforms.intersection(proposal_platforms)) if not common_platforms: - raise RejectProposal(f"No common payment platform!") + raise RejectProposal("No common payment platform!") chosen_platform = common_platforms[0] diff --git a/golem/managers/negotiation/sequential.py b/golem/managers/negotiation/sequential.py index 8a373e14..3d4304c1 100644 --- a/golem/managers/negotiation/sequential.py +++ b/golem/managers/negotiation/sequential.py @@ -108,7 +108,7 @@ async def _prepare_demand_builder(self, allocation: Allocation) -> DemandBuilder await demand_builder.add(self._payload) - logger.debug(f"Preparing demand done") + logger.debug("Preparing demand done") return demand_builder @@ -199,7 +199,8 @@ async def _negotiate_proposal( return offer_proposal async def _get_demand_data_from_demand(self, demand: Demand) -> DemandData: - # FIXME: Unnecessary serialisation from DemandBuilder to Demand, and from Demand to ProposalData + # FIXME: Unnecessary serialisation from DemandBuilder to Demand, + # and from Demand to ProposalData data = await demand.get_data() constraints = self._demand_offer_parser.parse_constraints(data.constraints) diff --git a/golem/managers/payment/default.py b/golem/managers/payment/default.py index 57f00224..441c445c 100644 --- a/golem/managers/payment/default.py +++ b/golem/managers/payment/default.py @@ -39,7 +39,7 @@ def __init__(self, node: "GolemNode", allocation: "Allocation"): """ # FIXME: Resolve local import due to cyclic imports - from golem.resources.market import Agreement + from golem.resources import Agreement self._node = node self.allocation = allocation diff --git a/golem/managers/work/__init__.py b/golem/managers/work/__init__.py index cbbd09a2..8913103d 100644 --- a/golem/managers/work/__init__.py +++ b/golem/managers/work/__init__.py @@ -1,13 +1,9 @@ -from golem.managers.work.decorators import ( - redundancy_cancel_others_on_first_done, - retry, - work_decorator, -) +from golem.managers.work.plugins import redundancy_cancel_others_on_first_done, retry, work_plugin from golem.managers.work.sequential import SequentialWorkManager __all__ = ( "SequentialWorkManager", - "work_decorator", + "work_plugin", "redundancy_cancel_others_on_first_done", "retry", ) diff --git a/golem/managers/work/decorators.py b/golem/managers/work/plugins.py similarity index 81% rename from golem/managers/work/decorators.py rename to golem/managers/work/plugins.py index 402d7a1b..ab266d34 100644 --- a/golem/managers/work/decorators.py +++ b/golem/managers/work/plugins.py @@ -1,19 +1,19 @@ import asyncio from functools import wraps -from golem.managers.base import DoWorkCallable, Work, WorkDecorator, WorkResult +from golem.managers.base import WORK_PLUGIN_FIELD_NAME, DoWorkCallable, Work, WorkPlugin, WorkResult -def work_decorator(decorator: WorkDecorator): - def _work_decorator(work: Work): - if not hasattr(work, "_work_decorators"): - work._work_decorators = [] +def work_plugin(plugin: WorkPlugin): + def _work_plugin(work: Work): + if not hasattr(work, WORK_PLUGIN_FIELD_NAME): + work._work_plugins = [] - work._work_decorators.append(decorator) + work._work_plugins.append(plugin) return work - return _work_decorator + return _work_plugin def retry(tries: int): diff --git a/golem/managers/work/sequential.py b/golem/managers/work/sequential.py index 5dee56ca..b5720cba 100644 --- a/golem/managers/work/sequential.py +++ b/golem/managers/work/sequential.py @@ -1,36 +1,62 @@ import logging from typing import List -from golem.managers.base import DoWorkCallable, Work, WorkManager, WorkResult +from golem.managers.base import ( + WORK_PLUGIN_FIELD_NAME, + DoWorkCallable, + ManagerPluginsMixin, + Work, + WorkManager, + WorkPlugin, + WorkResult, +) from golem.node import GolemNode logger = logging.getLogger(__name__) -class SequentialWorkManager(WorkManager): - def __init__(self, golem: GolemNode, do_work: DoWorkCallable): +class SequentialWorkManager(ManagerPluginsMixin[WorkPlugin], WorkManager): + def __init__(self, golem: GolemNode, do_work: DoWorkCallable, *args, **kwargs): self._do_work = do_work - def _apply_work_decorators(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: - logger.debug(f"Applying decorators on `{work}`...") + super().__init__(*args, **kwargs) - if not hasattr(work, "_work_decorators"): + def _apply_plugins_from_manager(self, do_work: DoWorkCallable) -> DoWorkCallable: + logger.debug("Applying plugins from manager...") + + do_work_with_plugins = do_work + + for plugin in self._plugins: + do_work_with_plugins = plugin(do_work_with_plugins) + + logger.debug("Applying plugins from manager done") + + return do_work_with_plugins + + def _apply_plugins_from_work(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: + logger.debug(f"Applying plugins from `{work}`...") + + work_plugins = getattr(work, WORK_PLUGIN_FIELD_NAME, []) + + if not work_plugins: return do_work - result = do_work - for dec in work._work_decorators: - result = dec(result) + do_work_with_plugins = do_work - logger.debug(f"Applying decorators on `{work}` done") + for plugin in work_plugins: + do_work_with_plugins = plugin(do_work_with_plugins) - return result + logger.debug(f"Applying plugins from `{work}` done") + + return do_work_with_plugins async def do_work(self, work: Work) -> WorkResult: logger.debug(f"Running work `{work}`...") - decorated_do_work = self._apply_work_decorators(self._do_work, work) + do_work_with_plugins = self._apply_plugins_from_manager(self._do_work) + do_work_with_plugins = self._apply_plugins_from_work(do_work_with_plugins, work) - result = await decorated_do_work(work) + result = await do_work_with_plugins(work) logger.debug(f"Running work `{work}` done") logger.info(f"Work `{work}` completed") diff --git a/golem/payload/base.py b/golem/payload/base.py index 0f978a55..a3bd8864 100644 --- a/golem/payload/base.py +++ b/golem/payload/base.py @@ -21,7 +21,7 @@ class PayloadFieldType(enum.Enum): @dataclasses.dataclass class Payload(abc.ABC): - """Base class for convenient declaration of Golem's property and constraints syntax. + r"""Base class for convenient declaration of Golem's property and constraints syntax. Provides helper methods to translate fields between python class and Golem's property and constraints syntax. @@ -61,7 +61,7 @@ async def main(): - """ + """ # noqa def __init__(self, **kwargs): # pragma: no cover pass diff --git a/golem/payload/mixins.py b/golem/payload/mixins.py index 3379cb3a..612ee099 100644 --- a/golem/payload/mixins.py +++ b/golem/payload/mixins.py @@ -10,7 +10,8 @@ class PropsConsSerializerMixin: @classmethod def _serialize_value(cls, value: Any) -> Any: - """Return value in primitive format compatible with Golem's property and constraint syntax.""" + """Return value in primitive format compatible with Golem's property \ + and constraint syntax.""" if isinstance(value, (list, tuple)): return type(value)(cls._serialize_value(v) for v in value) diff --git a/golem/resources/agreement/agreement.py b/golem/resources/agreement/agreement.py index d34f5083..fb4564db 100644 --- a/golem/resources/agreement/agreement.py +++ b/golem/resources/agreement/agreement.py @@ -13,7 +13,7 @@ if TYPE_CHECKING: from golem.node import GolemNode - from golem.resources.market.proposal import Proposal # noqa + from golem.resources.proposal import Proposal # noqa class Agreement(Resource[RequestorApi, models.Agreement, "Proposal", Activity, _NULL]): diff --git a/golem/resources/demand/demand_builder.py b/golem/resources/demand/demand_builder.py index 69556618..e091cd92 100644 --- a/golem/resources/demand/demand_builder.py +++ b/golem/resources/demand/demand_builder.py @@ -85,7 +85,8 @@ async def add_default_parameters( expiration: Optional[datetime] = None, allocations: Iterable[Allocation] = (), ) -> None: - """ + """Add default parameters for Demand. + :param payload: Details of the demand :param subnet: Subnet tag :param expiration: Timestamp when all agreements based on this demand will expire diff --git a/golem/utils/logging.py b/golem/utils/logging.py index 2fbc1e5e..a36559e1 100644 --- a/golem/utils/logging.py +++ b/golem/utils/logging.py @@ -28,7 +28,7 @@ "level": "INFO", }, "golem.managers": { - "level": "DEBUG", + "level": "INFO", }, "golem.managers.negotiation": { "level": "INFO", @@ -36,6 +36,9 @@ "golem.managers.proposal": { "level": "INFO", }, + "golem.managers.work": { + "level": "DEBUG", + }, }, } diff --git a/pyproject.toml b/pyproject.toml index 7f2bdf2e..81cf4a55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ classifiers = [ "Topic :: System :: Distributed Computing" ] repository = "https://github.com/golemfactory/golem-core-python" -packages = [{ include = "golem_core" }] +packages = [{ include = "golem" }] [build-system] requires = ["poetry_core>=1.0.0"] @@ -58,7 +58,7 @@ poethepoet = "^0.8" [tool.poe.tasks] checks = {sequence = ["checks_codestyle", "checks_typing", "checks_license"], help = "Run all available code checks"} checks_codestyle = {sequence = ["_checks_codestyle_flake8", "_checks_codestyle_isort", "_checks_codestyle_black"], help = "Run only code style checks"} -_checks_codestyle_flake8 = "flake8 golem_core tests examples" +_checks_codestyle_flake8 = "flake8 golem tests examples" _checks_codestyle_isort = "isort --check-only --diff ." _checks_codestyle_black = "black --check --diff ." checks_typing = {cmd = "mypy .", help = "Run only code typing checks" } @@ -149,7 +149,7 @@ target-version = ['py38'] [tool.pytest.ini_options] asyncio_mode = "auto" -addopts = "--cov golem_core --cov-report html --cov-report term-missing -sv" +addopts = "--cov golem --cov-report html --cov-report term-missing -sv" testspaths = [ "tests", ] diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index ce694d3b..02b15db2 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -1,11 +1,11 @@ from contextlib import asynccontextmanager from typing import AsyncGenerator, Optional +from golem.node import GolemNode +from golem.payload import RepositoryVmPayload from golem.pipeline import Chain, Map -from golem.resources.activity import Activity -from golem.resources.golem_node import GolemNode -from golem.resources.market import RepositoryVmPayload -from golem.resources.market.pipeline import ( +from golem.resources import ( + Activity, default_create_activity, default_create_agreement, default_negotiate, diff --git a/tests/integration/test_1.py b/tests/integration/test_1.py index 2b42a570..dc6f92ad 100644 --- a/tests/integration/test_1.py +++ b/tests/integration/test_1.py @@ -5,10 +5,9 @@ import pytest import pytest_asyncio -from golem.resources.golem_node import GolemNode -from golem.resources.market import RepositoryVmPayload -from golem.resources.payment import NoMatchingAccount -from golem.resources.resources import ResourceNotFound +from golem.node import GolemNode +from golem.payload import RepositoryVmPayload +from golem.resources import NoMatchingAccount, ResourceNotFound PAYLOAD = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") diff --git a/tests/integration/test_app_session_id.py b/tests/integration/test_app_session_id.py index e64a0dc6..ae566753 100644 --- a/tests/integration/test_app_session_id.py +++ b/tests/integration/test_app_session_id.py @@ -2,9 +2,8 @@ import pytest -from golem.resources.golem_node import GolemNode -from golem.resources.payment import DebitNote, Invoice -from golem.resources.resources import ResourceEvent +from golem.node import GolemNode +from golem.resources import DebitNote, Invoice, ResourceEvent from .helpers import get_activity diff --git a/tests/unit/test_payload_parsers.py b/tests/unit/test_payload_parsers.py index b14d6d77..740e195e 100644 --- a/tests/unit/test_payload_parsers.py +++ b/tests/unit/test_payload_parsers.py @@ -92,7 +92,7 @@ def test_constraint_groups_empty(demand_offer_parser, input_string, output): def test_error_not_operator_with_multiple_items(demand_offer_parser): with pytest.raises(ConstraintException): - result = demand_offer_parser.parse_constraints("(! (foo=1) (bar=1))") + demand_offer_parser.parse_constraints("(! (foo=1) (bar=1))") @pytest.mark.parametrize( From 3f7fa16e30d8d8af1e140d3ba0746b91745c3445 Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 28 Jun 2023 15:45:46 +0200 Subject: [PATCH 064/123] removed TypeAlias --- golem/payload/constraints.py | 6 +++--- golem/payload/vm.py | 4 ++-- golem/resources/event_collectors.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/golem/payload/constraints.py b/golem/payload/constraints.py index 3a664caa..3e50ba0f 100644 --- a/golem/payload/constraints.py +++ b/golem/payload/constraints.py @@ -1,11 +1,11 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import Any, Literal, MutableSequence, TypeAlias, Union +from typing import Any, Literal, MutableSequence, Type, Union from golem.payload.mixins import PropsConsSerializerMixin -PropertyName: TypeAlias = str -PropertyValue: TypeAlias = Any +PropertyName: Type[str] = str +PropertyValue: Type[Any] = Any class ConstraintException(Exception): diff --git a/golem/payload/vm.py b/golem/payload/vm.py index ced88dc6..d5ed5d14 100644 --- a/golem/payload/vm.py +++ b/golem/payload/vm.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from datetime import timedelta from enum import Enum -from typing import Final, List, Literal, Optional, Tuple, TypeAlias +from typing import Final, List, Literal, Optional, Tuple from dns.exception import DNSException from srvresolver.srv_record import SRVRecord @@ -19,7 +19,7 @@ DEFAULT_REPO_URL_FALLBACK: Final[str] = "http://girepo.dev.golem.network:8000" DEFAULT_REPO_URL_TIMEOUT: Final[timedelta] = timedelta(seconds=10) -VmCaps: TypeAlias = Literal["vpn", "inet", "manifest-support"] +VmCaps = Literal["vpn", "inet", "manifest-support"] logger = logging.getLogger(__name__) diff --git a/golem/resources/event_collectors.py b/golem/resources/event_collectors.py index 9fac642c..58662981 100644 --- a/golem/resources/event_collectors.py +++ b/golem/resources/event_collectors.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any, Dict, Tuple, TypeAlias, Union +from typing import TYPE_CHECKING, Any, Dict, Tuple, Union from ya_payment import models @@ -10,7 +10,7 @@ if TYPE_CHECKING: from golem.node import GolemNode -InvoiceEvent: TypeAlias = Union[ +InvoiceEvent = Union[ models.InvoiceReceivedEvent, models.InvoiceAcceptedEvent, models.InvoiceReceivedEvent, @@ -19,7 +19,7 @@ models.InvoiceCancelledEvent, ] -DebitNoteEvent: TypeAlias = Union[ +DebitNoteEvent = Union[ models.DebitNoteReceivedEvent, models.DebitNoteAcceptedEvent, models.DebitNoteReceivedEvent, From 8a355dabca6c51967bda2fa79837d1bc8e098784 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 29 Jun 2023 12:19:55 +0200 Subject: [PATCH 065/123] Basic blender managers example with acitivty pool --- examples/managers/blender/blender.py | 82 +++++++++++++++++ examples/managers/blender/cubes.blend | Bin 0 -> 558772 bytes examples/managers/blender/frame_params.json | 12 +++ golem/managers/activity/pool.py | 95 ++++++++++++++++++++ golem/managers/base.py | 6 ++ golem/managers/work/plugins.py | 7 ++ golem/utils/logging.py | 2 +- 7 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 examples/managers/blender/blender.py create mode 100644 examples/managers/blender/cubes.blend create mode 100644 examples/managers/blender/frame_params.json create mode 100644 golem/managers/activity/pool.py diff --git a/examples/managers/blender/blender.py b/examples/managers/blender/blender.py new file mode 100644 index 00000000..b6f6efcc --- /dev/null +++ b/examples/managers/blender/blender.py @@ -0,0 +1,82 @@ +import asyncio +import json +import logging.config +from pathlib import Path +from typing import List + +from golem.managers.activity.pool import ActivityPoolManager +from golem.managers.agreement.single_use import SingleUseAgreementManager +from golem.managers.base import WorkContext, WorkResult +from golem.managers.negotiation import SequentialNegotiationManager +from golem.managers.negotiation.plugins import AddChosenPaymentPlatform +from golem.managers.payment.pay_all import PayAllPaymentManager +from golem.managers.proposal import StackProposalManager +from golem.managers.work.plugins import retry +from golem.managers.work.sequential import SequentialWorkManager +from golem.node import GolemNode +from golem.payload import RepositoryVmPayload +from golem.utils.logging import DEFAULT_LOGGING + +FRAME_CONFIG_TEMPLATE = json.loads(Path(__file__).with_name("frame_params.json").read_text()) +FRAMES = list(range(0, 60, 10)) + + +async def load_blend_file(context: WorkContext): + batch = await context.create_batch() + batch.deploy() + batch.start() + batch.send_file(str(Path(__file__).with_name("cubes.blend")), "/golem/resource/scene.blend") + await batch() + + +def blender_frame_work(frame_ix: int): + async def _blender_frame_work(context: WorkContext) -> str: + frame_config = FRAME_CONFIG_TEMPLATE.copy() + frame_config["frames"] = [frame_ix] + fname = f"out{frame_ix:04d}.png" + fname_path = str(Path(__file__).parent / fname) + + print(f"BLENDER: Running {fname_path}") + + batch = await context.create_batch() + batch.run(f"echo '{json.dumps(frame_config)}' > /golem/work/params.json") + batch.run("/golem/entrypoints/run-blender.sh") + batch.download_file(f"/golem/output/{fname}", fname_path) + await batch() + return fname_path + + return _blender_frame_work + + +async def main(): + logging.config.dictConfig(DEFAULT_LOGGING) + payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") + + work_list = [blender_frame_work(frame_ix) for frame_ix in FRAMES] + + golem = GolemNode() + + payment_manager = PayAllPaymentManager(golem, budget=1.0) + negotiation_manager = SequentialNegotiationManager( + golem, + payment_manager.get_allocation, + payload, + plugins=[ + AddChosenPaymentPlatform(), + ], + ) + proposal_manager = StackProposalManager(golem, negotiation_manager.get_proposal) + agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) + activity_manager = ActivityPoolManager( + golem, agreement_manager.get_agreement, size=3, on_activity_start=load_blend_file + ) + work_manager = SequentialWorkManager(golem, activity_manager.do_work, plugins=[retry(tries=3)]) + + async with golem: + async with payment_manager, negotiation_manager, proposal_manager, activity_manager: + results: List[WorkResult] = await work_manager.do_work_list(work_list) + print(f"\nWORK MANAGER RESULTS:{[result.result for result in results]}\n", flush=True) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/managers/blender/cubes.blend b/examples/managers/blender/cubes.blend new file mode 100644 index 0000000000000000000000000000000000000000..4a6a4cbe9faa069f805c9922d4c29de7f335f8a8 GIT binary patch literal 558772 zcmeEv2VfM()&B`mj$$Lzt1Y?!QAH5~)FlK+NJxO_swX56l2D=9M#8w9!38&hE7-W; zmN<@+IEh1XixS(3Q|vhLm*9M#zkl!-$2b+oj@kcvJM-Jyo7-;smvwP>9XD`jdO=*u(H+?< zwOzSuwOf~av^~p9KS~)pDMyVTtbbI$K{Lw|<{yn4F?KQ2gokx-%^DH4IBLLpkY zBl-@QJz+>>Y2v78Z{h6Jr?)IvdV1T!e63;rl1~%IOjuDeH+|op{8H`aWyPl#PmS9W zOncN)nitbMbNkQw5R7b7t{gG>pr>AYWZh158a@3@sFnvu%-gi;= zA1-Uq*lb?7RErxOrNu{$|JlL`F{jFt7HD-Dc^YA*HyHEGa`b``Xnb)mzmsa~ePG{j5?|9-fp17i0kF%a1 zD_PoF|9f`r?%unq;-`now`lj3Y|>WGO4YKb%+m6v&Hd8du~X*>LnAlby|z+&fZhTB zJJ+n&3g^r-Om$k0cFW3g?K;w*T~zOHU%Ae}o^?mZzm#-9n(Uak=7g~mpG+M)@iz;i zr+!hHlBLxU55}I&)r(g6Vb6MSeQtpt?q(g}bh&S0bJv;;r&i5MZt8Pc`}JT<|G^1s zVv~MQox14XHEG!zwMRdS?A>poRrzyR#||CB7EO-5W@pZtFRoj>R%5ec(W+Av*3_em zJ1E^T^B&P}&@8H-aptg=+-<4<%&%x}`t^+e&fGQHoKa)7q^Jp6Ms$ppIdR6Ru>*!C zZpv8l()CLUUt#=N7YgU5X({6-Y14;|)aH&J$72HR+MLze&D1yST2idtw7k^7|GwfH z19#x>NnEJ0Su-b1qp^p!minDNL|dM?KwFnEU+2?E{80aQQ8}9NHoJG^tkL#So`64W z=)vWtu?)-FvrE@2uIkly_NaaXmyR7cWc`={gA1w8$s}FZ@uB!VXAT>^p)fA>71l{% zvdI=^V@lSiQ-==ER&lu8-L+fEob+))%#SH*2+))T9+M*R=$4c>h;<+ zSu2jlM2_-nH&vRp)c@gq2gJq>8?!Ba?370DvXa*xTfa>^RIyddpOK&~n;x&5sY6Fx z&uM+Fu_h9ZL$dTUV?Lz@F*r z7UXFwW+iB|Mvc*Cj2K1w7OmAUSfcGF9q{IR2>&u|H;p;2U9^(gf>qj$%StTqZzSw* zq_lv!zgG31^-gdyrYY@L1sr9rnd5<VMYc*qvN{8wlsTsk3#nW>)g=8JioH6dJg*4)FWj zsk8r0?P9;!QRDK=_-~c%nshZIkZ9Jw)L($wbdS<8O} z>>nn)8F!!AuI;7q0gpLgO5*1mZ2;>4@Nc60-ms!n+fC!Yo2bq=P}+N_o$%FpAMT9( zZ8RQX?AdTza){DpbD+FIyEbpNwjp(qX4Zw3R1b=%KiW%k7Ve8SlP;@xv!CY3+P2IU zG}bB7YUbs>d~MdsKedFl`mTxEtlEqv9}?dD1gywn>#^rnNJ2JOyO>$T<66SR`K^Yp%V3)QFFsLkeiz|~ZDpc`yY89Ka?`k-h7|MlCc z56{qXXYAQ5nmj{W5;I3Po2Wf6PtVjE7q8I{RBX{6Dy!G-rLlkyd)9;P`gjr8v%MxR zm1x#dn@?C%UT&cJ2p!m!U!v>4)zt3PQ=Pt3u1PT7W5*v`+`Z>4gj_UMb= z-7mXb#h=5j&ntMx4sW|~)`L*$Kacqd=eJk~Zd_WV_p^nw)3l|89j9?<-4@-fPs-F* z&P>#b=cH5JNIy;P-!v^=^Noi*^&dS>_&-Wxg}c^n;JLZBY}#A{d)9$FimP<9k1*Hq zCmnc@bU?*_E0rXyIVWnd!$;|c$CV4IUsx5FL?&4)j-UVQX~W0lWK5pXuxoh< z^&9!xHKYf7X)a>Mp4~hjs7Rr@Pi9-zN=xh+e;95{cs{dyR7x2JZvb2yJNlg4W3}ezjJuUFzR={VTW;R z;nWv@(J`z;v8fHPo&S&MH!${^MJr$BzK+LOj2~mqhWiBIf5)l{-OqFTN-E1G)DC2f zk9nQOy1fav`_UG#y_wn$)P1(Ep}E8()z@eTH*D1&;`&eK(E6<={I~1%pZoZ=vy*Av zVX784VvK=5>p{Zk@me~KFPBY=*RrO}JUw#2fI?bRPH&?1rN*U&G>%-RT|;&1=GE)8 z+oc}tqq@H_ZLxv>E!2l_-52$qY!mM6K1lg0nww$FWmyln%|bh{oB9~wt=i?uSx*xF zXIKYrp+3e=2YlH3@Si?x%xW|KG!Lq84P(3a+$VDVAJ8-63E*$$t-8~rV;b%t{pL1p zU%^INODolAtwPJEa;C8@t?`vHF|IL&K;JpmI6e%Q1KI&@@BOj&;h!^g_8#CeXXLnf!S?y8 z4sd^6Lu2Mj()o!)hn)catPkL?+VdvFY-Riz^P88J=YBza|y_VWRn#)q# zw^r8;)(Kz_`^ME}+T(;9bl|??O*;Ne$Mbj?o?kFcV${SH(L;xg<-82*)4yNd^tqL{ zt}6dI&+VS5yhhgn)&n+=kuCst*tOI~?OI-}uRZS6ao?ofN9#$tA^de6p!Ryl(gLlD z)}i&bpV~HG-S@Tk<~E>y;W9t`SH>p4O@04q;LqzCm88QJyhcTJfyW*^_T#o-=7_QR zn-?zo0kAjQ+;52Xo-tR=3T^G&w4e9t(PP48UAlIs{`(UZ*I@j2P+!e`7h}(OoAFmM zH~Uk@O{u#+ujq{E+g9uN-$v~uukBw)WxI;{{B;QnwEGEDUc2Klb~&xrFpgZex!s`t zL!+^Pf&T-91+C2+xHJC1p6O_f;V9>UHUm5Ky}W03{+#5Wu1-v&wmN&aia+Z>K8;I( zJ=+b0(?RM#ZeHQh>zax^uvw~z_y>(3M#WE&v?3U zZmO0rX1o>?Io#5uM^Dj4^zDE9irB>GcFR1O@xPwZ55!&Q0k;V`)8n-G=xA-x^jLkp zD;T`hI6Sst>|toFHmLf~I1KFBs~^o@i)sFFFYi2Hf_R(L8NA&HdP{n2~q`&GSCrOm$RWv!U_L>e%FfHX~}_kZUUD=lp9=(FX1C z=IgXaHeIVVEm=$T|7ySb&-fS6c+`wPtt)7I3o5l+R+rNnQ;|N#qcutGCTc&pkG`j{ z`qRFb^^90BX~s6{{~x5W{0B7NIAhoDGyc0tKg@0anz*!2Y2EFL2P!uI>WNJ|v>R8J zYUOE}+M<}5TI|TtmU_VYF_XqB*;8Xl|JEA)fS1})EBtxQiucB@&0azM*)&}TuAw%D zW#T%hVr=JUxq&i+0AtyI>6&W z#-8{0xNZY~8XKIZ^;TnlfY%9cAztV!U~fpzA>4QA*faiY?q63g@K^eP3L1w3|9sN1 zRkRjiWme2g;PnX}vpsK?lg85>#typjD(+`)TT}6hxy^@LZTmAP&3vj`r!F0y*wpy< zr|YlNo?+7<425a;cs6O9vzKctW+!ShhL6|Xrg?cZhZJK0UdvSR z-$i`_V-LeRkTq?lR*{;eHPJgZH_+NfYkI)z6IJQCe)#iR+`Q-*1OJx#e{(tUo&~L= z{DsyeJ|{!#D?$etdmbCIDW^4g+TS(mKJB#`*h2@r)COXI2zCX{AAvo$2h?W#Y;R$u zKIY;%_XCw%wI}MDEb-UZmZ;qi27AUoe&qOv&2p*kS)<0y*)%`zt^*aDPl>jFYrwca zclw;SM-CV?jIckW;{Qy;F4Bcv+Owp;Y@Vv$sVyhW&1UStq0{R!mVB3WfQ=b{6?@hJ zUZbLROJ8^3IgQ!eVACF$`+U^@%?tAN{@;whDt~K!T2t;gD5Cd}g%f8KKS2FH*MG*I z_wPz+eTde@^>sJay&Ea-=JucaioG=ce!x@brvuz}@w|btr?qRHr|2J+>*2KTr0p%O z{x7P#2ddL^ZZVGocwZ-2>>2-*F_XTj%GkA2=bm(qVKtpsG4{}S9XoHv+{#V!m%L$a z``Nvn_8s>YS89xXYxo}~y)|Rc?g_((?cwtXi(_VQp}nxvj6e7FR@n2L?H(F)?<4%# zTu1Gy8GpD{J=jLL0C%>z9pG~|x6?epjDN88UoA8C(>(TQ6V3e?f5y9p#(SjKI{w_w zvtd2t^}<2?5`b^<;7skx;OdB?O zLQYIv3AM#HGC!Cd)Iag~qbhBYzBj>`n+@^#D7-_5aOT6fj~{epV|w(|Q;TEbw90uo zZX!Bv;N#z9W*}j9y^u}eymO8+?z~h5mq$9kx0sPs01FbtT_Kdd~f3v)y zxS>Htm6EMBQe2*+@mP}e z1h})!>&m?5jP)kAvnS2`pxME)!nIc2ip7YPxG+Xc#TmtyjPt^bY}e7JvgFw zZ1mu(ZerYdPiZ~r%E3xn^PzR9g;QdUGicOK=jpgh?0F7)kkVoEIK>U7{@+tt$Ds4i~ljWckg(XgVxpe0dKR-?Yt^iLcHfLWyYDWj`i#zB71ADE<2Bi<*t@bq zYoLCF$Bbs|Y5)17jvd0odvtb(-&jZlt}(dYUWK*-FhzbFICrH|XO)#-Gkm96}zi=-wm3tOvX<&G?)9eBggO zmF+we{_W}ik-n8p`$RV|{&kf9Qd-}=n(8#;&+~>F${UXr&Gr9Q(oya+e5QuaMEU7J z%#|aXI4#n#uJ@JIo!M7XquoUF37UuN=TTJb+0W;Y%=jmdne-x`;h8ydZ1ydri#)gI zeW2~s-*8#i(V8Id>t~Fg`VAWEx_fr(?%t8R_GuOWr)XSfUH^F=NNs?u{~Ksc(2PH0 zNBzi;=)7MywcJRqzPVH5-d#v%D_b-1BgefSJ#gqvboQ)~&M0lAvFj|U2TeN`uQ|>5 zqt7?6NB?iaU(LhW^3&L0RzlRoE#rp_TQzFHkQ6#I@$?N#iofLbE*Ks=HqhG9Hfp1I z&A^AfPY=vGKy!=3o6>Whp?2aE9?R~gy0khriP{s|kEi_rEBp_YZ`J1&tOI4Fha>w9 z8qYEk%@M}g%50?l0G^N1Ube1pS9HIu8?FBzH|xNYwXNVUb)bOOxy|_BzN+H=0X=*5 zwJPVK5xwRuh>khk8unK2#|#|W0G^`<44yiEaAXpl&09t5D}Oier@DVTwIN?c{YPHR zHmxQ0-?@10kLVn!K6c@@dE3I}+Sh3Qa9G$+cphP;1H9+R_}{X2{Xgf9o0L9gz>owh+!01+Ty*?%CeJ!W;~IS|Fr-)C!Cx!c z^xx2dM=G~}wtG$aGY@av{sD~x&zRSMumvbwid>F;!y*y+zwu5d)%NaC(~H^N4gFW_UF3)H>YjorgN&X_ZQck=DvaZ_yXz& zcI6eG0{%QVpfhqBot4tp0=5!%9FOLZzc*vgZr+au{%mu-rL%Lq2Ym~zKR$bRQPo@2 zPyC+dhCdzCZ}4&&*XeC;OLd?3EO>ts>rqLgCp-r^;>S#wb-$L){+O!Yvo-uq(>a3CDGG~vD&ZT_ZLV8drjqU!Q%I9qP*UnCD;&t!YQIn1h zi0GX}=SIB1pU=|p*5v3r!)C}8eWJ+phrhMD z&torO4}0hpeG);RH6=DXg2n{!t9F0VvAJU>&dr`Nry^n8r23(BhFFcG`mgNOC&vog zV8Uo`S;v1w|3Pb1zE*y&|0e7Ut>T>PFg{n*@A513wH!`sNzCk8;LrODX6)IWI&R8! zG&X)##h)7$TGNZ!wrIs^>T~~1W9Zjt9=DIaJ9`c7ylPV#`g$X9hn*TV z>AT>0F6`VHbE}`E`Q#Hc_t;;$N#DEVHO4C12PvX6^m(*4Wu*(K`&|Dc>D!o=@;18P zpha~4@>kTye22!KH<@(cE3f}*K4Y#LIfL=%b%X6R=I3(?Fc}l4U&rw`rR6-M;?L(2 zRQ!|2OuQfb)5cG|lh1wc<#{I=UQglmWuBAr9%Q{T{$qZ?+-zq~nDH+0W?5#72?>)$h?Xh^T#J4W{(_+a#qD}Olos$uUwTWuzjvU>9A7oGr`qMpD|+vs_+N4z4@C!f{SnxceTL2l zT><{IMw&qV#p}S^Y@e_7A9=U3X`Mfu=7=9XMD>-=m)uV6xUL6^F3g`WeHYWyT3Q#{ zi+Y{gd#=}fHi7q^_rH5n!GjIt!pdj z{NUz5L+geQl+}O1d*>=G`>9P?x`58Z^Paele=9n`{Q}wo))nAS`-rDIg@wREoOYqXAO;LkeHiXN~IJht&_1OG)+X8n=)hK(D1Rh|{z<}j}R zG#5PY`hRn|Sh;y$f!2ghK?itEs5o){my2TJUK-YSV3L_;*|a&^fInj|>VKhuKkERU zrL@(72T0HKa|?8K=^moLm+C+DCqs}1ol&jNoG^XI(rI&c(>b>@H`0EqYWO@feJjWH zU!!lFEbpM8dk(_4=Bzx)8v{H|H4T#e?@r*IZ>8txYGHd6%RtJi77j#&L}J@MF~D zv4gKl*qpOMV{k<6*MGB&ExG9|!BL*;vqAsQHt#tDbCdl=hcJDBu9qK;>1HmR zJoEJGxD?t;SYzPNdcZnBYeWaFFgJ%?Lu+?t?BS;SBlNkcn)kYeOYb^d#rVUtt_S=* z663Gdeef>kHh}PNq_N3^bZ?JY2QKFN&+^mV2Z_L*ZT^Od_ZzBd{G)$oMtkxbg*me! zWzk=Wh94L2!z7KJ^g5j@)X%G+?wjqMIjc{Zv9@ya9u_b++oSsr4wyr7x}ygUiJ|k( zr|+V10G~MyrY+#_8idBIh1~x`wqR_!Yv>6+3(4PW z^I2FnTe4RCqk3NMJ1b`;KSXfZlQ)fNIu(a zg|`~U_Y|3Nhnv=AW+N@trnQ9Fn#`rY{ebD`oe6JioMqddxeQTIY*q>QP<7Qqf;x%SIm+&CX{TO#TpGW7Gg2SKp5P`Yb zP8dC5Ez+~HdtY{W-)%XoUMfnMw==SL|A4s-<-haft455TKY8XFvQw;hsA024k6X?< zu$$&HmrDPymO*R&-rX;cqVLA^yEN!*g1*MY>r60wHj(k?{X(^_TlrVfnGZAWaPxOl zv~G2j&OjvE<*D!G5tj1=yZFI`Pb6Kb&&vOk=mRdD`X5Z$a2!78M|(U?G-o`G^(A1> zHs4Rk_%~3$V1>6DM&C*X;LqQ|G5#>E=|Ql04>lgHkwtIIUimt=_bKD293i`qO|bNr zLc9eNrZ2I=pYJr}vnG7U3*RZkYlnP>?GCyZ=T16P#rMnHMf+1QJZ4y}v;iKPdjNa0 zP2X4>=l0HZ8S}Y|=pm6~fxWcPyZyg&T@H5qw){U=~;XY*bq zf9HU+o}Ji`M|6ZV-wA|!_D{H zR7(6=2k3rJ{VWl%wzKb`{(-)a)7Ozz{OKI(J%m#jo&CMCcaO`K(j3HcpNQ*xZAR|D z_*)9I!FP3JpP_HT|4iRay+q%+H`4xI3Y`bKg1&QP_I`66!rZC^MurqeV*mSyIyrFhp-edF)$?0qOn?^d9#NXNQy{0PW?B_dCN@$Ll5EcE)@q>q+ zm>N0!4Bsh8-^HAwK02U1h#nj{U)Fv78x{WMo$pjJ;cvM0@d)W0jZOYV?aObdZ7{x< zgq%+O_pcKu7ua;J`~lQ^lYNHvoc}O?e9Yanj+aF1M&XDLo6Z);()Z6t`5P~~Cs)6N z(u_Z!S6euFx;E*`NW<_w%Ij#HP)}zb+0dOz-=%x)qv`xpZr0?P^($v4-azY#ukrWQ zyobe@-%a1p^O+|8HgzMdP4M@Ue7_UFW{r$8u&4H3-<#(#A+Tqg@5`6?>vkaiKI|F)0=feg_`{~( z-ZAi}Z*GsPa|NgV2ajJUSX;%V@mH6zgND3?I?wfgam+0J{xklj2?jb~#=kY}8GpWy z7r4XbJx;X$bg$w*t9&{2|J>^U?ajo^9yRvIyr0lWXM*wj76<5j;a$|u1ADer{DZ-s z@#k>^aEHB?_N{?G-O&?p?#QYC=N@~mbF>FNi?OydP1$P#bb#xBAogOc$9SuT&z1pq zv(4w8SqJE>RH9S=zZ$%)%9-{NkJ{nS-!rIf0H0F`#6J*s=mD=onQ@1k$D1^k{Cr46 z@7`*@ocjOO>BH2kM#j?q@o769P}=~W-}&Qjk3H)E??(c2v(0l2nt%Sb6MX}v;?I7k z|NrWApkPi~qaFVATZ;O33}^@VK1Uz^Eo0C4^Y5scafh4lKcjKr-8R@e>wh84;jGFq zlra8ofxc}$Wv2ssXdeOi^S!_V|JJc*{P`|kU~aY-Oqll6sc)4mW+{ z{p9QsV?T)>HC|H<_y431CxCw_y6nVlTff2k?zDgSqt@C0-ecr?9h9MO%=lhwU~aY- zO_~{iyIBv+__IDZxV8<~P|A(IVNS5apU#(jPUi!DOXIe;gT?-C!k_m9%(%l%_i5PT z&v;Yc;8*{l1CA}~!nrI5I#>Kn=ARccdpp~4Bge+GO>2c6*e3kXNSnuLdRxFY@9VJ5 z_fWFUzlXziC7nrSo3S_BT;JKR+Vq<&^jn;|5*z z#MH=9-=(_>UyU0b{q~$O6W*CMYWz=TjEFiGGi>yWldc@`jp!l6o}PH+@NY~d|CFmn zzDfJNCr0-lbb{M>D?{s2C+Hq42mgyq|LF|L)%1PA`&O7EY#iwZWAEU9{?+^Tw8gZ@ zQ!=RM6)yv4E1S+1o#1xg!T;jIp7o|zw;l(qFgJ(sm=D-H_5b410Xi4_^bG3PeP+xF zns1z-@0d@}T_`6~#!fy#-!(e;Uu1ng`V9JhrIdb0;sq2SDrC7e`yr zy|cT5?gD7G$3OPO!bvku(A_o8`0pZZ2k5s^8t6A6e$|o=(C^;qI^fj%i=*DFI9}DO zUnAWI^_!M-fZG9;&hcM7`Y^2TfEzL=&wRfn9dPizc<{Eul72(=_Lg+OipB}MxO9Sk z=k;;BHo(F9;=Vt=@l#%*`N5CUqho$TzyJ28ReUGx;?)T{^;c*% z4%Qbh)(*A~wvG-sI^gJlqXUi(I6C0yfTIJB4mdjC=zyaGjt)3F;OKy(1C9^@!E}L08D`rL+`%~gp zFRk;`)~znCZ1Aie7Fk^BsVeooqpni70&!^L4__+1mz%!NT%D0PaPs&2m? zF~{ZFp2~EoNlTJ)$bt5$t6h5{hRfd0ct^+!`?CU3 z-mvW{TYkgkwVa}W?=3d^?ES5nhqSj8fGhHSyG?q)9r-eFlQWt2|TQ(_+Etrqh=&VQ3BygRwA zbb(c2%f{>)Pi;|sb!|>#xcvKW|>qMo(mB^}31@^<5}_l#iKaRtxnM@x1?#?W?q7 zvt;`!ZNv-4mV;M4D&xfC8SuBOr{3eDeQj?kaC(uH9`fH>Jw?6@+$2s0ZZ}BWB%Z|D z@5#^0s^-B=>dVzZ1B% zw-kUI^4%J4$d`ee#L2+zI*FUaQx!cwFR!GwqNbgr8SA!ib`ZFgza?;k{a(7j4Yr!! zmi*@AfE)PQ;pQ#P)#runEd}6)e7A-h@@3#AaWZh*DRGl{s&eP&EnOCwR@&ZSsAV5E z$6SzV!qV^kNVb#Gj+OI1*fwOA!Cg$DzzzKEaC5H{xV5(wfE)7N8g9s!ft$n$xIq-R z#1nql`FYDKNhR&*;oEr>of0ekwt`_ypD{bU67@OBK@EEvBzIM1ZdjxLnEd}6) ze7B05%vW9tt+5RB?OXne1h6Vg$OStuC9lL&<#E*)t@F5wD%w10_e-lOk=Q4?{!Jau z;UD;~yCYrikazX@1N`ouV!s6bw%L~JlO|Ta_sB2XaGN3;PVcLUe)|MtSNELD=B+p1 z)Ymy0U!3o=2*>vJV;Df!t5G{0!|5BQYFyA)geO8WO**-3hUr&R;eHoWnp*^i23-*-Tw1( zo(Oh-MuFxEaRvq}!+v}s!t=k#%|EZrV5n-45U(uiW=8W(MmP_h09M+Uj z_W09OwoM5gj3~gs2(WeN_?zB_>Nk*~AFMvc2mr?WI>Uitip^ zH?LB@gHQ&9zo+J5+ zAh3O)cl@3Vzia)RZLwRPETz^mk_b^J1-;94-@W(Jdi9%gxo#$s;k_yjqxby#oSd(C z@>4yIOcJ!s$y|jK$sn&xi#jgonv(^8kwf0l zE7q0QN0tDX+9|O<5u=(uxDxHOU6UKGH&pf zgZgP#UD3vxvz9Sa9@*KX_>nK<1Nl1ntSo1lU(Q!et!FcxATBr;`HQs47xIC8McpLw zh5T~98tOa+^|eJ+bu~q`o~rtSy7HpZ>MdK8$=_C-Q~-a=wa+&Ph?} z8B(jbkuRAK1B45a_8OL*xKf8P(AIKN#NtDbl*I)X?v$kMeWp!QMHvd9D2R@V+ z@_~FIDdd;)CG@?Zq>}1oK}A(*MJ*jUM8vafBVTB*)O?{`^R-_EB@MNk&sH;LD-Gu> zho&CbzkQQpvM!aUiSNV8R|-2ywijUZS*{2_St{)1@;fqbmGE<)CieG;8x3RSULXB5 zde|^^ro&jE-*tzHb1%&ugpKstYJ1;!QU5N(-AjCTryh~W80?=8%>kQOel}Lmht%+M zf6h+_R}TL{7bOBi{M*IViT=^|(JtQAHFkCr{QXUe-_=GxGMAAI7vaR?7(-qNgYSXd zZuPuDQ-T;Th?wE7h$bqJfa2nHk`ZLyd~KUP9%Fak=XdIE)(!GnKEo_S&wXH+E`c0z zDfxS<=mE|z!{mCu=X^cytglS+@{7;U<*@gA{+P`CM2=&AwjT7IA1W_O?X!`lzq@QYA&5XF7dnrNi`#!ljeKvS}Zr~GWi~RZ4v;Nj` z`cXE2mnP$RAqRL~+BTP;l?f~r#)r>Up!{AWFjQUe@e+7)I|RM?KDjx*`|0hXz1t^j z=Bw+SY{yl-JNY$hz4K1@*SqHCmfOYuA-zNGp==lr^bY*amEN7a*U>wp&@FWib=en- zUC=v}U$A-yy=h7Byu=^lhNI->_QkuWqu$PCJS)C$=|f@Wr#Z%7@3hj=CZ>Zi$7S~i zq<5_MP4Z9@;<4>=B_`>X#UJP;(mPjrnX%i^OJDI@wS_I~CCV=ty*x+>L9bXZ&AJ7> z^ZrAQ2cH&t=>2w2T_4>$5ib1RJ;LwZgZ5Ou-v>HjyN}E8Kgsw9ML1-3A787t$KHdI z&ra|Y>98IaQe=#$kCR*IE;v}*?xFtmgkv$C9qs?4dSZSF{Rf|OrT^~3LjRlJ{@mH; zYD2q>@(V`)3khP($6oe_n|#(!bN|rv=#Q*Nrj8zZ1U~0Vk23BTdgMOt=#g(6sJ5qp z7&AWlxKfs1uzG~~{42!Lu1ztg=It1o9-U-8;*MKB&?E3US9%on9jQlQUG;V!>0g?* zsz2}tmIm|)ub7nKaqY zJRkZzxmgd?bE1`=vUY7H7Gx6Bv+!^^aek3M*(Yt(crxgQaI59ce9UZ5Sh$tGWcXO6|}=(F#L`T!gC349T5$2ViGxU0dJH|trK znJ>4$$PaIj1A8BjK-aw7tfIPlPi`VN0zjLBeI zMxww-yBDp5bNTa{9Qe<%DJPVFCf^N8aiAC1+310qY-XOWvGv<3J?g8F{8ku#iB2#i zP8^xXcZrFTXF`W4p(E@Ua|J#dYkCrgq#LJVWH84S^vuipk^HS+USOfi*NXN4HtSD0 z$pC+yjXvb@dDFIE#U&2B5MBZoF0{xb{+mq*c?o~qMe;C(ae2Fy{*~jJpX}rmy4$U` z$ff_CIC=6E^V{IvR@k0tvp)LL<+k~Je7v2-+{2u=F!=>AhjDp)PR8#98NWDBpPP={ zm)v{dnd2*xI{DXizqy||^S(LZgPundrVMj`_w@yHj+Z?)qouGg zlAR3Qtnd1k2=dCl7;indFJ@8 zME6XVMdrWhxc0B(zwt&M|K#gWyc}3|Iz>o-C@;o3d?#HaUE%LiP?(E8vD7(rAj__t z^ay9|kGs4{-*Z1}3@Akz2>$n3W*y$Assk{C!KeUmw!fX{}PR3nQIA zi_gJCW@hwHvOjv5!dQNsWktWmam^3)PxR|Uu9|XBZMz0cdRdzplUt+ysabyGpu7s& zYsEKc;&T|$J?rCuvmNKMf6Dl%t6slhw^g+LFN(YV-@8uOT)r)rG3ttFgMI1Rh5KRz zwy`P?j%yAxyVZB#tHx<5UMO+FA2!pT75XO0k!>*)Ya(W0+c0%ElLxq?z9-hF z-2464S^h>MDvWDl6*Gt? zvni*->8*K7zi&0R0N=3%M7iJs&gEF_To*>KHx?&6^H5ph!`^S}^Il(i=#%|*(btDJ zCH(uv{G<2p?exO~xBT>okB(hdH2yLC#s7qGmgdcY3 z={gjFCBXeqIR&G=8%ZUvx6k9hiZN6B z&7Ufa22zI~@gE~1Mspwk`?wLV+UlN#`u;!V_r4SnaPIj+?jQEuT$H@F`JcyMDgJhn zf0?44VB7ro!vFW`(FytEi|79D|9)Z-N2M|Q*=^=)cDr0NKEC_Nz#HomKEAB@haB^F zWobueeLeHY@RtYa>5S4M68`dVLHwiD3w6K049Fi-njil(#2dRXecgB}#nIQ2X^nU) zuN#xD>ubqAu7aMz$B>yuamKJ*od4fbo;jZ!cL2%4`Tr~BnY>Lb7t8XP$bZvZ@o6gU`=b9h3WqCQ20Tp*r_O ze$Mt>z4MEx-p6k9lbxJXT~XEcvo+vlP77fjNI9cs9ctH}(D~Y*yC1!{`tvPIjwBb=Jaw#W z_`xLLCv5bKYM*BpZl<=PF~4WGIgZ(_z5|^a#~06E299SR7uB}qx`RkgCDF%ckrax2beH8;o+Hu^Adms9 z2a)7PJkWXb1s(Xw=&nY2fEIL!2Rg_PexP%?hSJtVn2}G!10CXlAN-A7d~{J1h;r>A z3W_<~-|2j~$W^uQNz#6)ckOKU^#@5A%0C%)?swn0L~YKC)@5Yan|0j-CyeZobi5gE|0lGeYiZI zqcq);Pk$*-##?RU7i#+dnfN`Z2JHOLBz=xj1&+WCIC_60a0I@3^2aJuMK<0}((TAk z{r#=vu}QW)^mBsrFl{sOfz4wneLO{{4Z_1a(k~o!bM5RhNH>9ut#ll*P%q#&+v0|B zUPUK`bqzAVx}e>_5x4=z=HCk(fv+xP)AK2=rt(*P0yibX-;o0Q$`=oZ3&(BJkFObI zunX_yYgoJnme`FVh0)gqKYZ!(0dG zHpq+R0gk{8IHH{fzPh|Rj!&C#OiXkYK1BY%zRX44M;6L9S$>a5KXB}zyq8{B0!OMQ zJ{|`7n%Cr zdg`g4$H#p0ts^f!v{M5^CXO$`yuZk7|*a@R39gP z2~5v4&rv*nIgNPTM)XX>{_hcA<~3f@#}}j~#cX}{Sl!$2Br^T;5B_ib#KBod>z`bF zl=5)2VP#{&r{8-m{-NpL<2=OQ@xsQJZr=W3!f!J^JVJTU^Td3IJzbeZc}Ob#^k{rl z{nCWQ@1;}vxi8esTXbao`>)Sk_~ZYj{CZ35FL`dxVPz>Wtg56<(SEP=mFyjj;1a~^^XWBoDHh3;-jSRFqzA@=(p9QpGbcOMD= z`2UWuPJz#~C%47H4I8&jz~53kg*a?;{e6#GCd_+&OJTvZOujg(to*wbneiP=V;Sxr zt0t{?(H&Ke&#YWYT|Y zntw+pjDAN)9Qjo^Tw>m`>RL}-{Y1~!+DlAuDQ#{0kUT#uV#l|B2Tk^JlX=>fJb_`-cw0VJ8$2+`M*jwv_{bHrCy+0M} zuNfZ-Tg|`OA4GEI=(?NxiAZuGA6VC1zF4HMN3!?>I>ZB=w@}c5A4oBV1|8yoE zexOs=Ul9*<`;%lj$#g+yUVn|GK*R%GhFtFkKhUMfLPS0h4|FF>M0()oqw}rza(ajd zy3eu%9r*d^-qX7^q=$H*JG@QMfgk7^yZF-~9_XUvdM)^Y&gHsAcOpH+1D$uPNDur# zm(tmv4)H+OJWtSpALyRz;7^Bm2AwPy@bl66muIt#2f6{9MS9=|I+yEuzw$&p&>dbU z=_EhUJ=ejX4)F{+sh{8nx|Gff^$`)u6ZIH$hzEY~H+J#S@%ldMJ=WJhlkEcHL*LYT z#pyCV;)DJ@r9AHPm zrA@y;Xu9d_iZIf5ZoUtjs_7qJG-ZBR=T&1Amzhqz^u7{t+Ma<=2Y*gD>hi z`VEyH@j*Xesh|g6(A$+i;)DJ(xep4ypts9E;)8yxOdovFe%X~D;)6b8zQ{lLqTR7e zAMru&T_oti7wLnKT7HNR`p?P*J@}&DsO=c^7x6*AHC@nyFUsAH9`Qk+A?d*v?GD;k zHUEeY`uAiz55B1XcJzo3`V2`AzM!+qKjMQP<2Ud{`P=C);)5RYfiLvW3P0Xo#W)fB ztM32uA1|W)0FEe+fcBKeKNze1ntIZXw1zMz>^*%M!fNdy5I994G&m)u;m#6wLN1i?KE8$jST$p~~2;6|9Tj~Y$(86%G2XsteSM5}a|Bbb=o2)j^i#*rHtw9~&QK@34# z5Q>`r3-VEA({>OM0{i>ODvRp&hLxDYdxLiDU z=KQcux>uCt95O}rUM=ok^geclzAx7tE$$@pj%nN7M5wRegZkhE!v>$;SupNf;mdQ?9sLbE^68s}?TW23l3BO`f% zAny=C8wO#LUSIN8SVSgyrIlK$I2YfG&G+MIJuDB&1v@6#9{+=RuosIR5 z2ZdEETQ|H@hUM~1^Idk8E zbZ1c{E@zI&CPLg*{-n!4{=QH3uE5!T0x{vx98DCIyioS>A zo9`LZ{iZuhb$(k#`_ELm|6bp^v>xB|+aCIR(>*HUz3;a<^!M`4ZvOT__sjf^_GL?V zaXIMyp6ut)?`8a6^n1p7h1|c#egXDfuwDVS-=t|AuT$Jg0r-yU&hWkATq0I$XYR1S z>fcSlZtg0-qbhD+zXv{OFWt-9#t(9x-}c4(9Tji`j@|}=Bk*xkW6wDw~<+MwjjCyg1CL*eK+6+ z9Gi0lj=)!l(lf=CnEt9ytez=aqwwEOfd|R(_+`ArOZ|?D{pR=NH9s_dq3@{ZkMH}N zgRbcC+g*b!I@y+RWGt^DnXJFRIYV#7cU0!YFLXC>1a81FL$=evSI9YnzK1I`;pp=p zq6jaLVVv%g<^G5ua$y|Ge)D_z8kPl)FSzawKjgYD1RS}KV)F!XwZ@Tqtc!QO3Ah1A z6fW?E|9r7qs*f$nk@e<{ABN>je0n%>HHRF4*>CRJ~H_Sk1X_kBn8N#B!4CcohOj%wmd1@VIteBV)x zOWm2Udx`&dR9(MUnM8R=Dv{q&<;d@-zR3K}kstqXQv8R#_^!MC-%6=$c=@|;zL!+~ zK&O{~b^5@uKgHK3&pz|c@dy79o;=fiFww7&9M4x?0-gTSGXR5N!+3q7eHjk?j*4{( zJU<>1J{xY>?f;HS%zrM{@2K|b1!ew@>hlzoowS|rsFVU}&Fg$e<$OoQ-%r8#^hIy~ z)HYE1#riFOKLsP&%dh4;Dy%bMT?y+<&GI~Hvs{0|IcA(|#ko}Vd@{@Y4Vr23I@2y1 zs&^pwa~)38+|Fj5=`Z3usT+2woPWA^w&`9*%!k~P5602nYpm9pb|~u+(Tbm?v42=j ze)RQ;W8;aI)+bPw znJ4(UT!X{>7ya!YDD$%TTNHy-{(^ek z@xA4}HbymD;4mltC{+}F&VO)mq!ZBXMcnsee zVIL6VGjrxx-*2FVcwLpxDL>a?zg{v{b;RwF=U$}U+*S0u&7;~kE(0I5AKrl${1VqI z@PMor2!qWWF)kBsyt%M#jLW@`j??YQ)ULW;7VqE}_3_uF|0l_q>zjF8#xxk0 z`PD}rmn9BKSI;R^ouN-jPlj>rv3l%qQFDrxod$LWB5-p#zl6K%aCf%&JAZ&nc0{L3$IL9*A z3$7dLBi2nW-utS78*qgDz!(1W#SR8XtMfVB*5NU`5q7@W?hive*CFHMTt}RCS)wY_ zliyqx5O!aZ|GAEguq^$q+27qWDPif3M~=838>8s)wBx)_bb{z-63sD+$M5rr*M~$OOEm27LVWiT zFVI(cP9=UV{_SH~+Ea;4KlR5iClsc8kN*AhF-Iv6M~^-F(}a7*-om&y?fwTr!l>gdwyLz{#(y4lHQLxNO^Fc z|6~dLqSv5AHT_`2#6G1;9g4px;V=C%5*9u3&m+<9l}ExyJWX{%tPwMp@V~_==Q=)3 z3c~)ekk z)Whn{PUJII#xv-)3p(%vU1Jx8PR0XW zxx8->{6IIYv;RG?hzGhT`Fn4%k{{@3$3<5;e*{eX=^e((jIUHw6P(7z||KLuZ-{ABy1rjPibNB;=ENZ+pf5g+v6 z556dOEByIff>-v#STA?WbE4pb^dUd$6Ur6wrc=9yzOae{`CP)nuxc8sgjwF7=H4RC zC1f0u_ot1L_oqd+?Y=Xv&)kwXOFmeibB}BrKge}{+hVo#{LW)&xdV>C4LD{Dm-nYh zd_&{f_Sk$bK|H1f0tB5k`y#$SP2vU|y|R7--?LVP=LMsS_gn&S1CA(e)Jxxa(LOJW zcM-?=J~jGPN9hMeXtr~IT4?2E7n|GFLh2vU?`D#l{=3ls+R)w!H5^a1-M-YtDgbT?iq-5x5yR zO1%KS7lJIo=kembFWxJ01CA&Y;4A!G|9tnwYmJYkoH<(hD!+@Tx|S{L8}9vCDE&AO zjQV&d?M+!cyzZWO9W{Wy`{Ead-4)IzFm`M8k=wTpR43KP%+2gp<6UTeu8-WYqrLt; z2_TEVMNuq@sKxl_%S%f|M##lj%@Cv^5^zVeZpK3J2>$2 zGx+G#>z*;r>kb_FO8j&G`e}UAC$G|Z-DBUXe)`)#ocO!v=embA(Rtk-F~6kqx_jxo zZWlVQdz{YezCXHm@~h8k$M2`}y3ac2byYR_%Kg{>;O+qZ4L0mtoYytq!}gCYFU0v5 z$U)2Vx~t=7Cd7XKgCl=_&ki0Z|R-UP?_SXPG21VGr=x8 zS4?N{;`v-Fp7Y?muG@TG_pLYIq%(qTJFm+(|2yY(1+vb0UFW6HlzHU&+w6s0} z+j4zkTIWGj8ayUeAG>ih$3uDk#Mj9fV;Y zGVarZAMfRBSlr&4_i0?6iDPaDHi+W0?Qm?O-v#FSXosWV#_JQDUn|bOG%h3;>LYLi zj*tQP(s=hcg;855?0;VK?N_covNF~6$$=yP>g`Icrt;%5l`Vc8ib;_b=nP zm)aoow|^#4EV`8d0jV4R${xnIy@{#!eDJxTh`vS27_2Gb$$yPXz45wNl3Pm;Unf=9 zE~+l|EUEQ)TBGoPA50mWDHtbxTqeh17?&Xq#%t<%X=5KyIls+x-Sz4s)>S)G8^+&~ zaK5bCI&b-p%el>k?Vc+3|C-~3&HO@n(0+gq+K*IeA2eqV~>7jNzULSOaq^|554rSUav%kgy&+WTV@_*m&UV)>gv$;`#*htTe<|)80XN`?b{hEV<#z$#2%KBS@gkll$^I2MqC9}F z@HbKYxt?;3i8}z zM53NJ`(%h;X7Kw%_`RWS6p7<`kBj?Y@w-HRMC8fKP4$!&HB{ECFGBPid5ZKA^0zj2 zPjl^CYYx%NsFm_}9OUmoe3mDFmt(Q8o8J}p{xzRy?e~Ym=(|SWlhMcjd!2(DcOO38 z=z9+rpIeP25AxAnmYaK-NFVWhc5mGYI>ZCrfLuWbexPgY;!lTopnDO&rxK8EV@F03 z=^-BIV&(7afS)fte>%hiom>813;2OA|B}W<9A+Ux;{Gp^bikpFUsE;0Y4wz z_w`fRhI}#}=)4O>d4eD48oT(@As*;nl=_5pLFaPq(49tlG9Ks<5B%WY+1W?OzXt?8 z#qW81CVxK*@qs_e9pRj=PA}tw-n&G|3BJ%PJ9@+ieTLLKNFVV*e^TZjd_e~s zRQwPh^!v+2`GGI!ZSybVgFfRLK@YyrKj@K~KH`IZK(?Sa@*(-79;oz)Z_w8WdhkX1 zw)8SS=ugV_8hk+yJyg?2e9+_l0Ld5WLJ#ns@#F79(J$cp(B`-N`v+D3fQO3bb<}C_ z@8R?zp5LREi0@ZF<9gh6o2~9wACMyG@m=8##SiY*_LnkP+$(wm(b9K?P13gfuJ8pr zvHSi7<#{k%l&4C`-1yE1mfy_zWX27-hbYYb6Ssa}z5AEKR_R0aH~(qzQs&!yLfEa* zN7ZNR`DiVk`rg1%;szX{7tj+ycaqBe>#z8Dkl$7Es?XoDYlwWkhsxOE(Z4Hf68Cb4 z(|rORJG+LuZV6`-7`r7LAELU!_2Pg99diqF@%`!%I(Kk(1IM{NS0AE4H|%fk6u3$o z@q@Laek<@h`9Hzon6bR=IG*Hu%Y62f<3Tl_?yGz1dWG9bpLj# zD8vkRxyfv~1ZTmZdA@9q#!EfU`jeiXdxQ9WEB6xX-z)R}yJr>W z>+pLkb^UOEDfAU~#;3yH{P*DWxO{Cp^;qT?dhA|joo^Id^3o+svKQ$}TvA&{3e9A5RSKZKiY)?>XMOL-0XmrML)xtM9#otw5OH7%FxaIL4R z)Du+QHS@@A6P1~R%e9o`=lUal(bfMQ`|<*V0zwS=rM!mx`BHuxJ@Cn2TCvenRYy%& zT>$eI0#e&V<KZ}=$1Z1ekrdZ|8^g^k;gU&?F9f31`sdXP83Zyw8ZJo?>O zU0P9A;i(O2@JD&cY9pOljCs`l|A7L&NA(}J`F*b0W8*gDm+~6&ZF~B-8}dte4f&T!`DM9yKeUs-q`I<_Dt?F);FLB| z`3urj%Fm(x_zU^x`A4_(8S+bc4f$)N{Lq8{wv)f2ibjA{B_U3M=C_H;UxdF<{|C8Y zbBvC1hd;wVx~0#MU&?F9pDN{-<>LLVo&0kCTT@hD9zqq6^WL1$sQg7(){6XD{?RRc zhWt`qL;iXxKlI?oBlLB7FiBaFl1XdWzHV@HMO{U4MP)_(HbiOBUKUz(OJ4D<$e(RV zK9Z$o|#Mg14!-!c9X zTksk3OS~b!J4u$8)C=$HcJh}r)YVsSO&%zOe8c@=JLQ`8P}XWx1%NKKYH2Z$V{s$%bI{*ckan zW`;oJpNrEp{=wd%`3>YxGY8tZ4f&tJ_bR;;A_UgaIy|G7%ImDiA8%4^6!LCP=7MWt2buWP6+D+*-;EH6|7H3Q_2k@DNg z8E-+O1{v~8c@6nTNcm;CsC0__MH`DNJXQ4t)g>jB4SX0tjj#HX1VLnWummat@}vJZ zxBoo;!@~#MT6+xnrM!mx)g=)L^abE zenWmKuOWX=DZeZil~j?x&Qo7cD?mXGeu!CKs03;T$X_JoFH_z@{v;*b%4^6kI*fF?G)_=Qs&I~ov7Jfs1DX$^_3@N`X z7nM}tFHSLW2cQ=~L3K^A>!7;)D$@}C4@>;{H>9nenO1>Hm?6KE*N{I}$}h`BA#+*F zUsF_97u-}nC+L`J%0BlwX#MLKc93si&%%mNbJt0+t$Ld^LUcuV_Vn z^SLl9dJZ$>m+~6&&zADba#88?QqmTsamBAI@l=Jj^{unYYa5wgHma!oVw4ml9cf_5XFkpXJ}HW={Gi+Ju{f4f&^dkp!dyoUVoQhr%3 zDzza$ZU0r*7SQpxqIDiR2pg;}(EeX_ZDdxcWPaILE9LhKBoEsE$?6LWzahVr*N{I- z$}h`BB{k$PYACI+JAkYbnfys@BQtSuxzbyae~MphYmXtnl-H1dtdw7ti%Mm*oOR4UPJzBAe1GbTqcW5CbCF<5kci2jQ^*!{-w%qri(FqY}|(YQeH#;E2R9gTvT#H z{emdru`qrAHT21QOx>3oda5V4Ed$JhWt~c{IXnBHtP7dj?c3PKm2p)=ift!ucpub z?@9TOTD_#OwKn1AU_*W>uOWYglwX#MnTFk}{Odf`8$I>4K`X$#5aX-qv;WgC1pePt zUPAv1lyEDrA-|N@kUvh!FUv)xRpqa&SXWNRz=J-{k;(BR+jacI^}3Y*6*Y6xUuY9< z4mRYM@*48TOZjEFm}%Ip${*tPFP;D7m4dc?_kp~8UCIwX+g$&Ff06lxjoXl4%4^6! zSjsQUMI~3|FRQ5BsK#l@pB`d-D_*Om{C4xaVk;sg%#dHoYsf!K$}h`BAq&7i)UQC! z#`hno7Wi{Mfq#jbX$!w0zm(UIf0C46mWxU%@YlCK{$t;NM7JV;shVjEzahVr*O0%9 zlwX#MN~-FAs9yn`jr0F~T9H4PWeG!mDX$^_KqkNguj;#C{F!Z}GxHuU<=?X^8&T`oCd8@H6C>@*47oOZjEF zsH_b6&H5igA^d;#z68Lls`~%UB0LZUmlUvMTvF6h6!(Q$kuQph11P2jBMc4-Gvmyl zXepyseoGCsG|*BMOLD{9!crEY`nsj|`Z%5@kvX+s3=h={d`rf^IJ5iX9%OvL<6rpo_&;9wM?Kga zef}4xUl%raw#4VsJ5O@$|J*H)Y(4&kUyuK4;a}vE+!)RO#dLb3dGV}*H$k7F@TyAM z2K@%<|JTIhZ2G|uvOGE%i3xlB3%?%!<-)(nC3$%?|8)CFd-E)M30&YTi0(gWZ?2-Z zzy;z3*=WhZ{~kf$n6Ssc@ayq^nDCE!@R?}-7hODyFTYyUk}Z~xc@JNWJEmvgMeL7s+9V#UV|5c7cBlt zvj4z0{NQuG+b);$_!oXX{)dVFQ4cPT=AU={(%b)nHF+Z8m0AOm!awuK{KH}V-xCR0 ze2;(O*W>?_!awT4MbZ4X^3^{J3fJJNMdAfnVf%;q=6&u#{sFT)0_3~rNPQL`k39Y* z9sJ`t{A6oZ{%18U>S~+Sxv;IRE5Bi|mf)+VFC@h|!Y z|L$obFR2&F`O*AK=Qpdfy`b&y&UD>rR!u7qFUUu`=%3Sq`bYgA83c|Ad;AN(9{*)h z|IrT4kLJIFHvJT1`7{=Z7i42*4*o|4fn&lR|H7}w|KY;F$R#;9nt#3hq41lW@cy@| zq4-On+H>@OV`6v?CVTt~zaIZzk$Y%S56;WRzdrpttAMJ|6Z7AhIry&%G8z;1_!oXX z{+9{=A{P@~&wqk%02VfNHFwZWh$dw3#V?WhU!3W+Qyy7+{0qMx|JMosA{UdC=KmDo zx$rfBo|ykn68_Dyq5s>(d=L@$_!oXX{;wAPMJ^_4CjW&Uf$E9#Pk)v9Z`}E(e8xhL zf8p2T{|Vt=BXNE=2hX}EF1Ly5%WPr+~Z&P_4xn3@Go*n{yC=q!}=%h zXNjQSn@PFm|J&!0xyQfo>+%0R;a}vEJbK@#`%k7!o|I6xfE9WGvM1XA<{abyutNA3 zem(vl7yd;q$?@6wSNosx$PXZD{}Uob`u~ynFYo_i`;Yyg+aVZ<348h%em(xz3je4F zhi2!$&=tU*SpQou{IlH;`#;F$j=@Mw*yCUL_4uz7`ych-$1&|cwEmxeC(sl2&)vd5 zvleCl50BwFnC$T{{CfOP6#h{UW^Nxk|9-~giD!;i2jH9N?yrlRXLYr;EtrM70OBkX zKj&J7RW+Tk{KNu7>!khjJ%7xp9iVsr2{I_c2mgHhg7sg- zF%19wm*^Y(yQhl0&<_4ED5QV7|KQ|lQ)}zh8L)*-^IHonKW+NIu+TLFGmF8Ze@>5T z|3giLh`6VJ;n(B8N%$AJBwviY|A=>fi3Rlj&wp+e{<*hd{>@&eB7$Et$>U%6_4q$o z_(wgcj2!n&3_>ipq`lj?3P3S`WCzx#wFR~ zU-eVoU|~1l^@RQRW)A*$Hkplxd;AN(9{-KPzsM!|-%+9Q&%ghP`u|S4_iKJ@ zK_^-D&f%*1qL77;T=U=Z2!72ZkALCU<6pj8^Qou7I{T(M^tgEfSD> z+2{RF%=c{i!7nsR=08P9_#V|xBs7-WmVfK-Q@ckz}7Xp9a2mH1CQ}`3UQwNIvCQeH*hZpM|M@84mAYZ+~U-Id&{=g6T8=Kbr zf$!u2VgCGmMa8q&8+<|Y?sH-Ozz_IK{zLNzzJYg&=x@=*ylG_C#k6szbC!+(ByS)5 z!Ot7906^4F@B{v~yr#cf2);owl0U^IAoBOY7ySG&^M?d~;0OFA|DgE;-&@9p>!;|i zNX_Z;fiL*^mpV73`*yf~f*n=Pq^r5blq|{DB|v2TI^O zv5(fDXSY!w3VUr<=cS!p%?mp-53>VvCjYpf6;tdQJGcT zbsE@=&Fz7W7JRq*SCm0MV|+eqEhuZ>@GIzhutPoGR?mD_eYbVi{4(dsKmn7}4_9^* z%VjO4|#=f(PCcPvwSN{-fkE;=puyXt$ux!}y_ z&?EHb>G2`0N7NIIcY6Kv3pPIG(>T4}aaO80&jDYO2dDV}q0oz2pKCo{MT?HCM*^kq ziNEAbRlg7!9Z{BbL^>sLKHM(mL_B7SWt;;&<1pOV8KTEg@*U>GwU6h;`m2xE`b(XV zpB_^;l&DA18}x|w4gD6&$NP=PZ4v!jlpeV)@=Xnu%J;{Z#B{C4_q)73l^@z8^aeeG zzAO5jG2{0eXUurG?wv8UPu+Lhyc+%dvU%=~u(KlpY&f zCH-o7e(iDWSBh_sE-{92_t#W(=F`1=YG@tQxy`{$M^?I~7_b?szAt}^-bc2rfZ##x1 z&2k9ZF^MZa*Q;_5$*~tA&?EE)JtoF!Jwm^|kaZ+UfA36bEjrGQ(HdksM%QT6>z2z0 zh903e=rMV)yibd2mysIi8;Ql*G0D#k3_U_`&?B@0{c3yVDzaPJ)jYzF>v_duvIDQA zAKNULSFlXk!Hz*3&-oebn4vH4L&v)kb@DBr@)1sXWhGIJvtyhr^NMVCOpszQJa$R{ z*W+yJTwu3Mp)mI~SVw?;gZ>rzhHh9F{mJ#V{jrk>1O2s%P90QHE*3Y^c|03O4+Y+4(1pMDUJd`sO<_A3ykndP z9rP=a^t!)H{R%r}uhHimt_a5Uqbz^_E6?=0(@})W>rU{^b*Evp?)08BTKzJK#?6l) z&t{S(%hPmB5$jGIhI7xA^@`_Z-AQ~rFVEifTCY0|qjjhEoO{$S z44EwkJ#xF48>z>8y!Ok~;kH1J?}#47$Ma(SL1}JSLk(gr?a_g+MGoT+6&sf7teZ?Gn^mlqZivvYFs>ctLiVg zy^Np-$oCi$GRt`>ZBpjnGAR$kZCmZfk0u~~4u|oL&-O7s;ylj?#y>)_V4U$e#Xam~ zuf41IF>k5z;dyeg-)}4Y6(hWLX$3!ggsc~6ddVlu=VAKELHa#7pG6CnXHQyj;yBzG z3WVdj{QmQD<2hzd8e$ABZ-w!d9(KI7<88Qq`EZZ7W1K@nyPbU!t${FOIRMZe+kROuE&PuVb^2B z@(8cT+WBG6&JV5qV(pi3d$aaSSRU4X3CqLUFJXCD`z0(7Yrll$VeOZ&Jgof^mWSQ1 z5|)SEuM(C=Vf$6={K)n%;re6ym#{o+{}Psm?O($3cz@r|+w~W_{t~W#cKszR54-*n zmWN$`3CqK-zl7ys*I&Z&u%2ABE*%-yenLVc#Ex$_U&8XR>n~w> z*!7pNJnZ^QSRQu$B`goS{t}jlU4IG7!>+%C9@5>F#& z^P_P6v-6{{JnZ}^EDt+B3d_UJkHYe>^P{jl?EEM!4?8~!%frr(!t${5qp&>e{3t9B zJ3k7`<2Yk~OW(fV-0pv|`(MKK*Y1A_%fs$}3CqLoe+kRO?tcl(!|s0x%fs$}3CqK- z?}p`J*LTD6u$_oj z*!A78JnZ^zSRVF!^kI3}@6m_l(YL-wZ}->P^_Ot{9O&yWj`2RS%!rp8@d_heX~efP z;tiRQzV2+qcQN9-8u5=9@!gF02d!`Wz9p=0`@SVCkDeayPc-WP zBqN?M;**W|Nk;r+BR<85pJK#MHR7ik@u^1qbR*tq#HSf?+kb}ZkL^Fh^4Qy`FZ&qr zeU13Xjre{>e19X}-{(uV{evBYv(C|FjXGWyEJ2aXTJ_+q)eP z!t$`=L0BGkJP6C9!RSvW8gV-wgym(&gRndX8}-HRcL>YJ?so{w!|rzo%frqO!t(f_ z*C*}#Agqtxem~jQ=P$N=;rdYMcyHxr<>z>AjPqgN|IpXlL-@P{E3RTtnd7=AH?E@a z5c=^B7xyq7A8N!Ys-L?f&S7|V3jVGUtsLJ^rTBlws$iLubZQlS{k+Uce7bQ4!w;vQ z!k7FRT*VLCL(y-jonE_^(o(Pg>r)K>5o&C~_U1mX(~~DZdp18G*{FT@678>@uYG#t zx$Z%ZvxH(-R1z&BcI6n|kNs8U>mIR-Do^r=|EhRmSM381e&VpL z>iH|{W_})|20< zj4LWCone(jsYZ`hKSl_3G#W~E5b-8+`)}#CLZMvO^uK-H; zv*pL9s`|0z4ejS5?^DU)+qRx&7&kl^ZN8#E;)gr@8JN zMg@ki5%$lD7kr3%jCzZDjByI}TjNyKhwJg-UOm2-03H#-sHb;I9QF8NS8-5geJ9SAdOT*h8n0dW_ewp!PyE)&s(*3C z50oe}n0!{(-{foBf4lVus$I0SY2QQ|`^D&bJTgaqtVdOkrM#%eu9O@3xH~Qljh}&f z+|kt9Ij^l_Awcu;rx%44Ce8JDB_)j}MZnh(h1FxlzwitGH63c-(BuEOG3|IxTWeSI z;;vck9c}H+9bNOA^J<<`3xY;z!v15zKc}t?JVG}d)gJ}GRFL&Ql|rbOBM2!9Z*o_2 zVG1@fLVtn&0sRHQ(2pSASNweJm#A?MHa_#gVMulCIlW_tj=bP-)jm_Tx}P!Sx8>#V zc$i9t`wRDtN^gF}xICtFpU5^~I?#`@9#>Nw@IR&JTwgX_RLA{r$UV91Z5e6T59(e| zJuUsT+x{n0)^X@}4cJZpRp0sT zWfS=M#`M28{ybT=@y`$3{S-y38J}?+g5&285G}@M9Ev-iBDzdV@SQKLyQOaDli%C; zsoL%882^dy53YXvmrK^2{rVy6sSNA?aqRZh!xvmSVO`fjj8{GKogW|h!m6)T5Bbir zbwp#`AAj@Y#y{7cxN-F9ch^xF>RwA+SpCEq7ghh|;%P??`~IsBHE-B|-Tv3?GvUdX zPpp3R`g_K=U$Al0Yk%6a{6dKygfnu{0s7^((<@N%fs0~^7t9a5QpeAykmX(Kd(EU`n9A(sgO~>u`73z(!&hRP$aQjUh@N~;66F3m}$AoYmj#E|p{Bu(# zln43>L^?FW#1OeRpom~evFYfqIu$#f;c}<=3 z=gd0v!z%#Csit<}{d@tb_G9*e66a^#jB^5hy1jK*hubYJHC2zJcE@ei3DDB`8mC6l zlIIK)_%R&yYnN?S^CS+lgWZ`Luh&;#cP94k-|L@jb4J;nSpO_D)<3tL);o5mmU8d^ zg&v_d=rLKP^$7jylI$dXeGjaEI=psD>z|&F`iS*Ur`Og$c`V~$hu1%OEri>PJ3#aZ zy+MzuJ!So~kG9LWc_6PDWm`AW?UC1wbZ^Ih`M3J?fzckJH|Wv*nARioORV-O7pF%Q zGH(4(>yg*7!+K0j9T<9q-k?Xw1NznR?uptTJh6S_uImn7eq;4syIs3(%CEnn#$MpW z`LqtA9gSV3x@>yHd!?xHQLn}5+=Jm{;|HTh_r=pg>x%ivh53tGmuWe`pU3@y2ysot zdt}V_Fit|>P>JilrQW9n^Pd}xNWBjwIU~YRXIm{N+_GW?uhBS+&*3l~eqx=KfR%ui zfR%uifR%uifR%uifR%uifR%uifR%uifR%uifR(_9Q3AYXY1a!sjGDJRTM1YRSP57O zSP57OSP57OSP57OSP57OSP57OSP57OSP2v-fkM_3M@_s#?OpV9())d#Wi={^W0K;7 zPzdJ}k~{1D&)7$s8lmGM;F^mXbX?9SINOYP2s$IOgzoHM$9q7XH?|Tkolk(T&a&yV zF#3KLKA+G%c(nR4LfB>7N6Yy?5kb9V4o*`M>C>YA(Q45xoT%Qk0j`Rnu(#DJGgk6Q_@ zg%tHH>C$ve(UJREwkd6!5dAX_Jwk8LV{&_~N9b2`_Y-M@Uy4CTzc`a=rz2)#j%C_D74_k+VYD0rZA@W;h;vjy$xl&waTRLBBePDBe=BY)~?yUFFgv z(?mHmef^gyA|4DQt-t!L$Hi25oJV+*!ohJA=g%>ILf_^Q_T;0Nh0b>tE|0WwT%IaY zCg}ZM$}_F$;^wN_&Z@}^o6c``<}YY2JcFzlhqSaUY(C_?1&jDkbL-sZjzhA-+lN*H zRsvQ6z69nhI!!3Jj!W0{mZ>`{ z$Fac|aMMn8oGLP0k{kC{cF0f)vmN8UrsIkK(|!O&1*bJmYFgM_HKpzR`E#6u$Bk2S z1rsQbc+h47_+YYIbGhWF^u5GyH*0_7Ozo#G)PC|ceXnuq&)UyMFOTdp(3QPKsVK+x zBDckpb^aL$k^nrw0k5T2$}M~WA8RAP103+&lQbUa0nc$hR>_(K9^intWwFKsJ>V@F z9L572@D{ggJkSH4*Iyx2{_<+^;hx*-o&zSet-iW;6M-Y ziDiC%V<{2sVwlv=k>~4j0v`G|^}=#h_yQ07S7vB@(1o7k@B#1P*9*STMZGokg#2F- zc;Ij8(DVgD%={T=@YH{KVNhf6zt#pkw9_c;Fv>j>ZRFv?J7C z6Cd!vANv`N54w>Z1`Zyjmd}{^JbnnBj#QuOkNY<+>Bi=vwXt5#Yhrb(;5IR_QbCdeJ3t_~KwC=`E z-e+{*8;}Z1_3k`6bGWVe_qeD1F&%n@-k?YD0{yCZ_np*+>eRY7 z$N!tYv;N)m=1m+vkt+C^QvbgwKBuw6I`~B6xqj;Xjd(A0X4#j@PNbg);{|fHM+QVW zJR}M*MZ|+)r1e*y^~jwJ`fGj%=rNjwDeC_Fr89X;e=>YmIhbO(@c4=G660s;f8={R zGCzgg2>S!$D8^OLV}E~{sB(WlY?$Nl+=b<#b{`0mbSQ`J38iCcD3_h6^?>{j#p zE;tgdj%b|SNCJZ(aR^#4<1I3pC5cVdaeY}BlHG6Cic*JgnoN2$Ly5+<*0~F z0NGIscuH?L^XiofAo&4L=!u&VVOD3n&OO_0Md7tBH6t6o;qnD4MT><@dj4n(9Jp;{poK1tV zE934I3iH^E@i#6P8pf@QkN|FCWEpyH>Y$49B|3(5mistLf}CHYFrG0a^ughvlP!l{ z64POO4fYxAwdBj%UV|Nnbsns@VEqc~9w0cGHhS=S$2X}kAEkygggy*;RQcj^E?d+b z!u_RuH%$E0u}>;{Epe3gK{sC?{R-%yUvaODSnsGQYtVFMy<@`;;dqWrkCdTXt#>F~ zTJL}#x!!S9`Dl^`uTYpSN;V7=l;^sjf`x#lrtg=IrflZaKjP3M^ycaD7+o)*-~Pc( zzNOq=-4mWKbbo;J^eB3R9?`y`U&TxJL-kJ0?wftShI{d)l(KpW`c?7nN6B6z z*Zr#Uue#My{pxOAL!^IIPhw%7DGVDy`P-WPvV`KQbImq{|49(kOK(_@fY zFr1k~|4QNZxSZsKarci@{D@&CgDL*LbG#cSHnUomi3hV!tON>_0M}RbK2GdW*r(Xv z0=w0Gp9eb?b}O>wHS$T+f5SfI8e3kO#Xj9lzT+)^;wX6^cW`O;Dd?adO&(DkJ@D1H z{bAI{T-mpaSYYX3*FbO3V`{wCBlN4tb{pr!YPt)WXOvay!l`+N!liiz{K$DmdF5!T z)rHJ6g6$$|o}sDfc}6tl0tMJdz@bOz4SGzB(|Uw{b#d7qJ#n*N^x^gCB{~lKl+T?U zL^dWn*rRwK$MN8E1?2zH_9?}K_UR`omO7(_o(~#5m*(fJNB$i}V?58lO;_f;Nl*Rn zpP0}4YaDun-k`^%^wZF<=8wKx?JNxFaUUJ;zG2fU0>F2ymx(=jrw9dlL>$iznQ1+e zlzlxu;@nWaihc$pMdR#K9;5gdXP*xAc?R?bJwkHOFXDa0kJBUi)i^ztSNgM-V7RaF zWtPKzF8qCA^Eu+&9@Y1S1s>lQ9{I0N@OJ@2?wJd3%t&k7x#If*)J?wp{BO3&Gx78} z>HN|PfellnVpJDV{s58g>X$m6tPLcnVEUO^nQ7^NZ!+OH|I9R-@_tHS(^t-|G4sne zt#tim*8iK1c)_bXlXgA9Xh$pQ84V%<_n-gJ`q+e{s*nEO8|(h>*{`mvc@#t;e9CftC{=PvHk>j_8X`(#5xmye|%zDmUX5Uy>HDGKlKSc|8x&3Ze0lTA<%&x z9Zm1=H2-0Tec!zAG!WO3*uTstJH7Aw4#K7NiEi;D*C!^@E^Yp0x;Q(UAtBGG0ZUWU z>l4wGGxO>XDbOSI20fze(65f)OtSy`Q+Vd^{^4%@K5abdh~>xY4|uNvy#40#3+$4 zl1^>oDhlnQVt#yABmNPBW%_`pprmj}JBDx%J6X}=6cshq1;^;5T&`e_1mnktVhNNJn)Zh)%c(be6t<_AMn6WN<9Z%)T6lk0T2Aea$XyB zfghJY;DJ9=&fS78>Z952P=3Gz|LwCh|DcO{X|_M$10MJ-=V^SJn&b`{Rf~6xtZ++`2!yKsf$E@QVz&14j=HqA1UJt z=%Ulb48VErQbQrP}6`%lQh><>9#*dC`*n9tpH z(^$>+8qe)A&fQ%k>le4lxx3xv+}+N_os;AC%;(lqLI?9+cemo`F+Y9Zv86w-9P|ji zL63=D<=ma<_x zK8NSWfl-)m&)pR!uQ+IKkF#m8aAn+`LSY^|Tp54k@a(y}XvvI_WR{`lvYfkvoHNed zi9S3U5};-CkYoL+ea&K@!Cp&j)AkzHuVCkzb|>t>AklhS$U2=8*3inquC!3uFlbin zxx3`?vcFOM#NIld+P$>*O`wB*C0P|k?^pG@+<4!VY6!)HOhts5Pj>2klltZYy>Ie- ze~$$2_Z~`;7_EL9FZegxdkV<&f*^u~nIgA1e)XZl5kUPg3FAEU20gkT(|Uw{HEUI* z$8AsfY*M`Y%}pF%O23sHAUB(ZMDBL&dH2&G{#>Zh36SGdQ>f-DYB>Hr%9YOZW&HDC zyg)WRaz4MMAGZtBX==P5mdiUa0>Z@TA93grdV?MlduTmEznUA?<2zwJ*6g&Ip9S^! zN)A0XL`yGHj}yzIr5&foyJ#B|x5o>qOwoE|-LbtkF!cz%L66B%T942#>x$lR(f;&S zr8gZXy>6Hvi0f4RMf=11Uyk$p;kDctGMw&LdH$r||CX!slaE)dbncFTkJIBy!slO{ z9`y{ED^f%wr_}!H&>o>T=n>;B^sD*1jM`(>M*7u8#|x1{Hv!Y{vo|LAz6)T^M<9n^ znjYEe8JK#6-k?W_5&G5fZqg$=^1Gc4PIpa>)4lh4r{*Q@6LY#s>hst|jY`qyU@Nzq zSk6C<7yO$|kEH5^85dI*IgY&V{tq}4?fAh zcDU1vI;-sE@4?~}Nh)F}2~lFb@w^H1B+Q%KzsUQ3nNOMTY4G01eDBdr8S*;!Wb*mE z6rTs(qw@c$x)9Z7+{5&~4bMMDp9f9s5H-&U6+JS7alsWj=uZ;67Dq2KlRlR?aL9n_=S4J1O-;{#qbXtdoUb5+tbvZ}+H?!Pj&i5#3;GPGC z-k?YH)6lOej}JN?r$-*6_!qB7wmb&z`wHj{dW4>!U&Q;0AE!t3t8seVqmtXBcEK>x z`kL=#iP+AVr{)e}0z>k_8Ro%XZzn2kh_ zaC5^(krTBdQwju&K<|F{P~ZhX6#Y|{7sz3IXCCOrjC&s*~E`p43f z>JYd72P1*K2Hl;ZzkJi8>s}#zWY%9@FEx!B+yCxZlbU%$6Q2jo$FW_c&V$PQ=T$=D z5hs_+2194WCX^etr90f;$DWIJsn5e@<1g;I?lMJGp9c-Du>|s}sHoU(_YuxeO7i=T za`h(2afqiAZdoyE0-um6JCt&WSc?hJ`cMSzc~E;Ew7;GQ#kvyKnNq*k>r7aG!aiJc zUozI8f(@K)KCCnG`}w&QS=O0$lXU~}liN$YO#4B)`GmcBtl12igVrbD2W&3?{rudD(Ik!08ZGoU&W>gv$TMnvLQ~W06Va4I+4P$T z^a#B{kBLEAkEkb#A-+%5neN$)>iwX^gYR_@kKrEoSH(PMyQjvyPxT0TV|+vey=;19 zaMoq?eN{Y4^NW!1Xw5kN*+Gxc8}yj0)Ov(|6^}}fpWNj0OYeCd9S`d9J|7?yf*xa@ zN9yr@(&POhDmqe{9(mz%VCoTigC0}mT944LmY%OS9j_uijt%Hlr{eGJ5-{J-KMsJ9 zujc}R_w#f2W0i zvkv3*s43EdBDr|#>T$pN13f}-&?A_Dei8Tm?ljHkme5ap|4CfvV9{$-KN*}m%01OI zV?4P0e)Y?w7^lZf$Z(4P)_36bt1F2mSM1;Z|9(FE$fw`WmpcRH{XFJLm^UTm96sh# z*w2jl6W+_?d|a@W^Lvl)P+OZs{Ta`jx(9zvSrJifCb8}x3s(^`{Nxil?mndbAl-bz z=ubcg{YmP35%eoh~*iH-XYv%Y*p5kXgd%H#0XM2eq#&_7;c&?v% zf2%1}a}`Z#g8Q216XmEqc~N!TqeOvcrCkKmIIPD7%7j8IBPp90{UZ)NLT}Jx@=>iv z=vQ;YdJFDr?yq{(^QM8QN9YZDOg*gi2>p6`?Bn~I(R#ACNA7S3o}H%7X^Y;VM~t`7 zua-~tea+l|^1SG?;$u8SI>+&x#zTUlpRunwa8`#JP50o-2`b8E>sL9SI6Z2LdZ3P{ z9+hL?PD8&6y+MyCHQJ?)cT>O0j{UJ49H*tmalSUe={}stww$g;&TmDHchU2vnnCj& z{$;vsdd!6OQ-brRXv#ipe+sc4S5g-P`?vqkn|w`0=|Rt%$_C853G*b(o4_;XQ#kjD z`#&&W!aW{)Gzi{-(7^Cd_e4}`?L?b`TFQjKnMLv z;tT%0bv~_)(x2cw=r_V~A22d>tPH7o-gFb;@_Y$?)V%39x_gv=nJ%tBfjn;}Sw@{d z)6{fbji#KLSAR%>9-%krF(vf^`t1+AO5%v@O>W=dW7DfN0c1;)p2e+b(`@_ zwIw+ZdLXs66qW% z#W$FqfAF|*hnfI+#?5jv+7PjGvyoUX_tVXsc&fT@^_BCrpRCh<;@7g@X+52Ry(5FD3VGf*$Z3=a&^61|Hym zH}-U$ALs#Z$>1;^;DG1Ky_=v1yoNzxJiq}jCHGN+o{y(PUO5F0c!?&hPtXG%&7oA) z0UqFhH#VX1kT38W28Hne2fSB4EA>b61>U>mVLZSAZ_B9~5A=Z7Flb~*4}b%nSAQj6 z;4K*(#seJi00(-AzgzC>VJs!0UEp54nH{>EfQLRZ^hx!b@c|F~mKKcTOh=*O)09=^Z>fAtKF54xzopaZ^u4|w3uoG)$L zl^<8eZOe2~eHFt5`?sikg4BXxlYW1Tvz^MQL`2lNI#ChpXFL_JaZG4JcxaC(E*8_vgM*W!s{m#Fs>-Hgk2kob{y&?Gtw z!@o=y*Y5_~#Vu@18ctKw`?{hj8wmX*%KZ!PKY-q#$JA|FkI=7XhxY&U@$a{@9{INm z{kVTKU77PVaXWDD>ww;%NA%Osua*aWuf6-r%I@*v&S({1MG244kN1gON-|`3r}n)4 zc^nTum&fpw9mJ=5{r!25IG54)+gpSl0NIagk32^4?^e=Qv>v&U4&3`Xpf~6dGJ}2* z?<;2Xp;tye38;l9F`Sq^t|f%kO?e75^K-oD}4jD0^-%fk0{{95koXxRGF zgz?{be%;#JPFFZ5UNXOW=}9L}*yj_Euj8`bIsL3BzO`3#^?@h+E_4UR&;F-vBdwKJ zPb7TCd63{3{s^M=Wx{71jz3Oxnbtvqf7k4W+O|vY-|+TJKdoW>hraRq3C}ItXZ^2M zp1z*Suzt~}mQ;^>;2RT$9Q-(!VZsfMEPP_d;g3~cS^c|pR0dThraSv{ZM9T}+6i0N zPgv76qk7!)CsO{Mk8YcG<+?l1`u7Qc-+4r})15rx<(5&KKmWOF>OVf@?oG*eCp}eh z5j)C z4Bx!3Bd(tUJbbo`)O{T??|GFT@Q9Pk`7kE0egFIIdhg&n^#+d>_P!27A~~Y=z7Bg| zM}NJq1M5myXG**x>rAr#g!emmPlWeIcz+XY8f?F_TSA@U&NN}*ckbLjM)kqT`TF~3 z@DsPnI>L(L)`h%zt%PgFZ| zFO+X-il6$LtYzL@+E@ZQ=!4uydVe>@k`ngfcQY=pb&DT4mK6WapXGTN{Y-~<{@mB; zunEyW<6JL!ts8oS9>ERttK&D5?En50o@pH%u!-Y%SIqs#Y5~BzVe?%vo(q_GdMvqx z-WBU_!3*~YF&1QF zB%-Qk1MhRs5?|Nf2JY#CRXM|1e!~^r?ItyXKFgWE>xX=gAt07B1M+XGl!xKA1>#40 z#3+$4l1^>oDhlnQVt$fjSUc>WO;qj6l|>9M46NIQmi5A*LhRq z-;1yC_hu>h;UmuB@_5gmFrSC%CkG90W%!TKPtlk1?WZXCe^?G`+>>z*;~pR|m-6OL zve(;tXK(D?W4!-RCyaOXLxwocyX9Hl)&FJNr?{N1_=&IT@h-Kjv~dq~&|ju@*LwiK zm-1`M#MZ_;-D)Eqg3gFApX}7!X2S%+rEw2_A-Mx-4 zrNou;phnuA52r(4&i{U^WrVm;uq`RG@p^c-upQWxp7E+gyZ%+F!i zC;uq#DE>EmrX-4H9@%A}aXYV~RFnf}?_QDeWFSZa@BjzAw_BzB!WZz(GoFA0UP|yl z4|tCAG3qkP{4<^c2fXBLoge4{Z^_^=9^ilnc_Uxoy;u>(103)G2YQIVSfN8K7hVfq zE_%e7qU8;`oPY+tznYzMXdcx5l+PN)XLoPr__Or; zr93FF)PY|}9P9Lm$I`>lK-LEQ^pk!#|^uoS5ek>lMFr$}69x zpCAUjI6VR+PLF!MV&I-FfZm`-$OHP-asO_^dGiilKYskn)$`_Ezkd9~gK7Yu&F8kl zt|wYSHa-50@`=;q-^=~ngJE5={`;5PW6J?FQi=Wa2Vx~^PG)kLZO$||1RO<;-yj9u zwpxzevSNkWpU(Ik4$}$3+pv{@m4KCim4KCim4KCim4KCim4KCim4KCim4KCim4KCi zmB5Ed0=&v-*9$*P8n?V!30MhO30MhO30MhO30MhO30MhO30MhO30MhO30MhO36v&* z!qyX4@_yoglYNgPYE&A>Bqab7xED#z$>7`!fN-7$=VNeQ2Ip^Z9tZF*(VlXg8RLEB zd}4G>mUA=izVa;x@sr2vb2F)T-c{#mKsR3>`@2C0`+}37v*%{=m0@3i^YkcsgC0{? z$$1N@C#tL&-*ULHLGyrfG}$?NK+bvTb2E&~=Vru@JU6q3&dl&H)5Y!Y4z`P^yAm}u zeQqY2a%NurAq9Gb-k`_Cd%Asx+hrAK8t0C8>xaXk0w zsE|5u5k5Dw(z&*Re+DE)v+1#&`n4MgChFXbrpD(j_&y7N&wgeO(J6@^p02(b3qKm_ z{ZC~*PNi&7t`XF)M~x@b+u9a%mQXy@7t{~b7mNp}M>-4sX2cb$ejuJ+uj8vI0pE-m zM*3oRr}i?w8G+~W7@n#x<~JjDs9fdzr-1sxVWf7 zi>MA|)8p_8zpV$ueT6Tx9B!(>Z$=0_z8R5R`CV=whPy5!?V~${zZr4T%Tv_%7mEF6 zgfnW^2EMZ&-;7u(-;6js$2TLMxc|tSKFh0#=3Ej=x;{c zbMf8P*Zq3tgwfUdn-Qts-ulFkzkXTum!^lm88P>~#u_R^%@t?dIAP6`dsk0gvO9e< zqUw=N!=G4p)yYRpc;#~b2F8YmJ5QOram4*k*F5^ww>LcduiH2L@uV|r|9SAb4KpwJ z-A`v&}b+>rW7`@NnigBe+h1&Wc%oJQiX2=FK~C{S@Hg z^LVPh86oqlSLs32xD9ytK8(q0-~VRB^UpnZg5%h4MnIRv`u3X<_L~v??VAy}PYU-( z;XWy}0o*f%IQ(giXUw15+*b0HQhvV)%8s*rw!_K-miV;3XJ_?7?aw?{?%kQB{nVfI zJyOZHdiOpllmquBjhw9W_qhp&czmg%B)|c0`816OdceZkAn*VOys>iM5an zz#G}A^8-ELnfFNn4tQHSG#=>rc&J#VzZ+b--4{*Q(9OxmwWU$X4-1os6VE>fDd@!CmMDBpbL4K^#b^S2mY2ijSsqzTU`Et2mUMa zZDY^{z9~oK4|w2@Y}fgNF3KN=4|w2@m3jiY&}*Fh0T2AtB|3l5MLPr?=oRG$Jn&yR zP2+qYMH;^^g@?*|K~ zSN}fPGW2Zj&?EE)JtlUL_j;n=p3OC$n(IAM!=xAM*#LbpBd=L?&*s{j)SyS`4SGcT zhJJgq4DUbff&1-d=nZ;=Y@uJr8L`oEuBs>hDgDfh1EVnijr5Yq>GYB%`d;ru-ANj~ zd<5NuFRx_^BlsbO=Jq(724jr7Qz*=1vn%6ooLFcWw=O~gxB-l1=((wbD$1AW7}8np z<0uJoevQI-#*oklhl7q@`Gp+QVS5es8SJ%`*lV!kV5gaOAMCauQMT6>Qo>*w+FTmw zx(DB?mWiX*hEuckJ=&gstc=HRmA20&C3MiQxPL8j#(?W01W8x^OAEIBS zlkQ6zv~IAkW}6Yut~tyv^!^vdrF}LIe16ZM-!<^}$10s+^xcQpZjaD!KV~ta=mYp(Q_&mrnEbQWBif}F#yw=O z-LuIT4Ba!_y+Or;_Y8N-fQ5U7v0n;tJoo9S5WnNBxvGL&%V3AUE6Dc@Us=XK1CpX~ z?eQ*J`{wp|4Z-j@Wjaj__Y6l-HZl4~9PJT$gC0{d-X=u9bWikV`cYpvnEb)yoADy~ z?LO{Ahkp0{VE?0>shcKgD!E@o#OaaoR}nhvFiMYlC4&@AwPheV=n;B@9>F8@tK*kZ zdsL3bHFn`Rw>xe}+}_w{d9p(~$MM{8s+dN#^Vu5{Q>yL)M$oR(b?zDz*J5(% zF|IuZDF(yHw7>eSNB#yR?BCxKBNWZ){Pm-ppnVDeyp2-~bk`u>Q-x%Cs`Y1?zhFLt z_mi0CU><|@Z>-0g>*-vsX*AXhq5g&U)s~Y#y<*pLE_YO2PMt0D8S&je>G3i3`{L#w zXy>4VcAofS6um2`pJoR;c!m+zPvibFpX~I0^XY`k`_08yHn3?2Pp7@+{L6GvvH=w2 zxp&04tEp)_IGQr3^e-QJgx;XXGX8}jIhIY2oX5Z_uy5{=9Mnot&uA&)I(Bb!cS^ zd3F-FM{bvwNxugBiQ{OmZ989mEdP}KG_`(`rsp_8`^j-WQy$l1;`GRULN+}HDF(yH zzrXt29$k_<#@$~NBNWZ)L`6B_`U*gD-(BQ8+*ER9eS}LE#}CV=oj`$q5($Up6Z~#? zm{QNgOdW7J*e-(|2D=QoVZ9ml7wj?Yk1+Qc%%DDt=kKt~c%Av6%6PjhIYYkND874) ztRwy^%8o**;(d$*po9Ly{atbN;&bXtPX_Qhv*-&)+! zJ?p#|K`+iO<1vbVA6EMsdV?NQGTx$H(t9@2-!{_QHRNAY-?sIY&Gc(XZ8&8!{iJ{O zH5?T}zl!;=f{9;ere64<%6H26mq{&7kLXw9^r)$kqRB;5&djSnq@Z7g-k?X69s1Sr z@Vau`I`hdScb*SDEk-NmNgU^NelByXZ0pR9>pWA=`%R);oF1cKgK7TG#~8vv;C>Wl zJ@Ps;?BL&0+4JGBzppd5%Q|!1{HRzJ)-}>9E75lEy-a;i`6&`6%QLxB$K&YMHP+6Z z>$Ds^Zrq`9Ai2^ND-{t?!{oh9#G`CUJkB}pYkGY3>wh+s!k?9uN0nHwMPh%&FNQx| z1Iij&uQ%)GfMp1TyQ}FOn4FjO(}K^73nLHsaped;*ZKFR{4(2XkY~r~JMu%hGUXH2 zN7Qppv(8GuO2A4Ww*<-@wUfAd9|EnE^_3{Bp_F(s{pQV^M`H$`!$%(#97BNFUA-%U z0spDB4^?OO!5)AbMxR#~72=P14wBLF?s}3z zmY>F8yD>=xgnjTm3e$hD%wfBVWy&to*Q(+C^z$8GzGD56Pd;@5`v;SsxJ#ec($7Ea zfBm`{tfx^@Vg2wvaL8{hDaUUHo*pKQ`TT{Vo0{sOFR!Bd!4A)9Bahg#uwP-%x;JZk z7Iv=2^Z(X0)-7l|&ndy2Ra;*5AD^rW?NEX}oA`ybXZ^av;)JeIw~!W$n`zzD-*Id2 zdB^jCOv)&WE~~X?dtUCAqLqM^Kwp-CvZ%zK^+$5E#M$gwf(_Yw-1o=Xv)D6c(ia*` zNB#1!XH}lPY0vKZz>g;IkL_24nYUu>S=a|r_N;Hy>TxJMcCw_pt4({B&XcY`cvkv* z>~}P!zvtd*?OB$Tb}YxJAD;U@>{-Y!&YpdFRlBNhbQGj|=~Ihh?OE8buxFFkYI_!T zuAj3Gu|2z}!=d%r!_7ze#$(rKcPzo4P5#{4vwj_7RRr6n8Acp%e%-V7tS<{)p@VTV zoGCwzWB*zSSP57Ou@_Wiz^%{1K8UhsAvIr)+|RS~ zebb&b*Jo2X?(@*}J}iIDaT4)N{$bBTesT6}-s`hszrvn%uaxy!S*JBiP}f+$py|@y zS)UzVf<2ph+1j(Z{rPQB>n>QA5C#lG1k>#LY*-$d^2z;NOWppp60j1m638uq(E4ob z_Q;-EpN-8x)@7-@Y|G_i&nla*zpT%~K8Uhsdt`m~cDp|7*Y&VX;g6X<1VXT9U6Eg$ zJ)8IXtk|!fJ^QmytNpZCr$xT-wLM!kt*c{xd#6*>CR$Z}{5k~6bfUKed)ED#wP(Zi z1~x0|7Rwo7JTud*J)0?`pv=s0ru;a?Ix7Jy0V{$0BoMM^V^Mm_p5^mhF;3aB^en4A z`_}14W3OwE*|RGzIaSd@xv+m0dklJFzX9xnD0{X783)#5brQzO-1`ku-?H|sC&rMS z5gapp2!vqILVhLNvtqw`_UuM&&tjbx`NC&=wym|flm*&Ydv>Q1?AgTs>wSAR^F>Wq zSAA#yCu_@Zir1N0o$Bz%$ zgWTW8?Vsf_%(ve{@9SbAh3s+cH^7+G6YI0E4@`Sj86%#WxPNDt>6`Yfc^)je+}g8R zc95SrRzjve6o))}R^%6F&*r^8EA}hw+2l%X&%(|P=g#Z14INFF@cjlofB)>RCD^m> z3%zg8X4>FEo%N;rQ(0@MKPV&8f^jp=?w>^;nkhfXE$C;;k5jC(60j1m639;i{{03{ zC4G`0#^qR_6*~;b@$0jhV;FFpAdNn;-=IhL&x(BzWzYKiFSu`Km+71Kta)Eo;tp%i zYS}@4!Eq8{9|9rRvyfk$J)7@&FxaoKXWg4+eOByT`cPqi3O+MTE>$8x7??X?z{@eYtzGTeMhh;us@@?Z*0#*W60`U@1 z!{Pg8&kl(%2-~yXuP^MMWjmj3S~>qkdlK|`Tg-Vd*auPetiR_#kK;NY7L*^4vS)+e zQ%K%p?ODHSX+1$-zVAaI-795%R@P~eFMP6R7c_NsHMbs8-`3jI z(bU;huu&Up&+cA=J)3&k+Oy$$O7A?w<|HBU3(Mc||?vH0!JctOTqC z@{vHmp50elO;K3c_Zv9DwN*lo-)}HDinvbmY*~utv)@4F*+c8IvHNFx%ASRN&|CIw z%HBU4s9C`=G#CzmND%fcKZ3?H1*Q=Zbz12&$=mV z&-%6&YmMs>d^3!A@VTyc)}GaJ$dq3&9pOy*DUsVhRsvQ6RswlRK-;q?Gi-M}LUwzW z`=g+Xw`aEtf`!9<&7Q@2tgO%ag)L@%7WP4}*|T4_>$ARwFg}{T4}p+p&x-s?wP#_! z!k%^S)%GmxT;vPCu5ohfJX)b`rY#4`FL-&@iCv!^Rf0X6_@T9DtvzeDpHkyo>$A~_ zdXAQ2eJcSg0V{!A5(uo%R`&k-Y)Au0|f;|iQm1@tzeuX{j-Yo00 zvQCS9;gdZ(|NIu(KkFQUq(Xhwfc-OtKKdS68k-N55mI(r>M3i_T6@-PKc&XGu4SVU z^*p0~+5{^BD*-EkNC{Nh`)64$vHNGa|JC-ZFTP^zS=a|r_N-5t`+9bnzFybn^{1Ck3!8z+lx~0GanXrUzfS}KvTtj1K0;q_N-t2OgmOn>8>{I zS-yWZcwblQUTe?#dJK-C!LSd35bRmVFV3FL`~F$6Ut!O>SIYXV*ty6TKH0PL7dtJ8 z7URdR&wi`~dp5CdggVF8oAy{HvteD?`?@kEgcN&U7bIbQD*-D3D*;Hr|DHmY?Xym{ z^I+r0kC$~t(2PG1=G$(7_52`>eC{_;h&^P__WJrP?1LzK){{WQx~zVdY0sMLv&qj{ zdp0b`plyV(4}lQuS;#NWp3QrGR_s^Uv&ly*<$eQg*CJo|WX~>I&^5n(!KFwk*pIbm z_bkDlO?}VWv;JBv>xAnNd^3!A@VTyYy*VGq>`Qf?C~M!D51?F`^6_O183Go*61x3k zC153BC6H4B6?VS?%c$4wS=a}?X3t(@?ODIFV=Od%9|9rIo)!6(YR|%cg+1%utL<5= z(;{E^b&aPkntOh8ubc-PQ-VF~K5p$<-_~Mnab1FMh7k`w*LBa@vswEKdkWjfmx;#<@7D*ur@(g^PHOCD^DZsMu2t2?uS0$gRVCQ7iSJr_)~`D((_ou4!-xaUuXEO(^<|-J zW-xAsGv%jo>|ZMZD*-Ekd?fJx+OtvTR-}|{%Xz;K-kzBarCJ-R*%`=HnC**mR0>sP6`G1G@Y$g^ifex=&8uwP-% zCa;zCSy`t=zVPcBr_JeTYHvPdYIDS1fo`nDHqi|Z78GmLof zxvqnDeOAjMQ+~m8gfr!*L~j3B30MhO3FIY#_ji4k?cx~M*X&s;kG_L8us*BOidmn9 zeb8(6?6<8w>(?uckEZWKAmrJzBEM4YS=g_zXWc7heOA_KkuQ9{r;c z$$zXpdlh8{I~V!FCwq2bb5rkpA8g+e?AgSFy>HKE_60$mIZod>pU>I~wn;OLIN*Ff z?E0)P3ti`faWkALKaFGmS_xPQSPA4Kfe{QO=fUFPv+Xx{{<$giZa(PZ_Z#pSrYz_n zxDE$x%6#@4D6C%GZ$RbI{jFY)^$T3g`Yh~&D0|kY%=xj)^i6v<mZEkVXkw~4MYXAqmtfB( zALxC1)|3U>8|uHcXEP;)6np9k0V@G30Z1UYK8v(WpTB{?du-X3qqABk7-LV+ z7A2^7dp2s^mE4$y_v?G-!K9Sr6=ToBK8UhseVM@K%JfZpmcOSEyx$;|wDzp8A(mg% z&xb$=_AKNVXV2z+zk%4VuxFER$oj0T(;{E^WY4xXUD#ZeXluK$NSjtwAHNQPvg7Pm zf<2qM&)Tzo9b%aV+oTyr9B_W!v+J|IEOgBb#?5f1{4|dJYb9VMU?q@`1k^}aE$6{v z5wfk%`h7jntvOR#6%dwbuWwfhZF3kvt``fTAc zvlvzaRssWE0`LERgUoSE$ynK^BJF(aSq0jQ_N=$Y*Ax2`);vwjU@yNg|>Z?4ap z`wdc8S$kH?4)TKy8}xk$gkaA?esT6}zWWVezrvnPJ}c|9vQCS9;gdbP=)A6ura4_m zDcDyN(x^5ywgh`N@$KHXXEQks>&oso$dnLLtUU`!Sl>#(O2A4$OQ76-AB+VN+W(95 zVA`(iuI>qY7WP4J*|UkSSbNrQ(?Po^81^9$f;|iQm1@tzeuX{jZj|*|S*Jz5@ar0< zH!m*Xas#5H>{$B$mS@%|J&uZC0enHy^VIKk^*t3vdoIRWO{#mhKVb3BT*tv+qCwsPK(ZchJ+&ZhM zD0^1@0a5V}EWw^leY5xNS);fVg`RZ%xAttNgpgwGSxCbARsvQ6RsxWK8VS|@S#2{R zJ?v+*X9+TBi&8Y+o*fn@8;mgx@Atu<2UFhyqw@63_ikYyMA@^xEuzQqV(i(}=dC^K zS0Bbp+Ki3Hq&N6pH_Rgfn`nClNn&))QZ)>eO z)JrZn5S7>AdDydK*SW1+-qa)7FF4}@5m??M2?zNeCvr-DGn~KcjyQ+Q1$aCM{OTJu zJm|RY$&EpHJO_N@J`E2#?qOkgJO@1F0XpvRaCz_?@DKy&xWhI5`dY^sM-4nVXtJZ0=}s)Zc^075n+PsJsrMdXYM2mAXd``NjKcRFE^kmo+%fu(~^T z{NEbsqv{-I8F84?)iWMJ@sAw(IUU1aS{JF&;ppdpV}I?QpFd&6ihuKXq1=r_zfgVT z^>=%5qz@FI+d(%m1HJKgxy12W{_sH;`a-$zJX^VXvOcetc9uLLTt8S}>`*SipPIb1LabMravezmupT_XVzW4RKp(pPzd|tRKQ5*){mYib7Euw7&YF1 z{>cIcTX&bG%0AVl;lfNWsxW@;;gEZq?pNJgw4Yp~=Y#MWIv;t3seW)Vt{-o(zJ;$& z(Kq;uE7`K!Ut068C643E`Lsn{?Tfmm%s;=SkS*P81ypI&xU90K*ORobP!R782jFkN zxkojA$(4FMh7XwH{hO(e;@>d(rRr#gMZf;%Snn%I*N{tx=sg!F(-w8iYnoF;>lVVL z;-h>OP6KJa%yH+;o4Se;xus8Qn{#1vm#&e3E8Zu8^r8#)7lmZwwO8qX&|X0U?HFhPLot?BA0*L(l;;?+$yO7SVQ_*x}drjS@+pGHx?I&&x`#z<7Lshi{sl7rz zXs?h{oHS_FtF0sMVIJwF;v&oFzbNNo zPrqDxv8(s(nhgD-KQr~q&{3|`{*jJcyF})f>!#WMmlcRbrwrvKN4UT%`XBr4$9S-YmScd}$*5?q1Kc#kR+x*t9 z;upS!P*gz=rg_^ISqDOX@KYy8uftVdKdQQ6{Z4`TEilC2wqzpN-L=0MKG|4@^XIS( znfjWx8$|9zuZrw+hN-F1l&Wf+a;(GgdfIa*;J51-^B#GPq5tMyhvU4Ke@WKiZVs=* zK_2G%4RR=8stc5?e&Um|4ktc5-(!hgJeF;e%Myh?Ks@tAsw#Iyxs-*<-Oh|?X)kfcO6ndC(!HbUCU*?>#N#N z$$nn=EO+!J&_Q4JxqiGs1$Kom%%5H1FRtwUWxdP2S?15;$MKVY`oLQ6f_(C|ze`=O zDit4k>hJ4a!Ttg~<>PzpRoW-oD{#?{5r^Mj*1Hm4)$KKTqxMtF!an%P-yc}(U62pj zE94X>kJ46?9H&%#l&iPayT*EUVc#5exxMmw7s{Dyy$kw;ejz{T9dY=ju2+?ck8;pF zlI$LO|IowhU0Z$&yT5Pi7v;>g-i7|m)UPkcaH!Pvs#5V$nh*PW7sl(@^)4BYG5$i& zu&>HRklY{CP#EhW+19&ub9RIHbNZ*yD$rt@C%MPG^(lV;#~ZTpX)&ivtY36qv(w(v z+)?~X%$U5!6)-{pf`u1?_uD+|{Mv6Hgqj_CbJNoWG^x2Oj-& zKXt5k{_PsUX1IC1?w$$k-R>qUH5qMAK`E%tt z{W!}dxmfq({G4CXqQ6Zk&U&9Cj0c?mO3DuL8VYNDbyug;OIYsf-<$K62aatKY|1W(Ti_( za~f0QG%AOR>okoExuGA2+_;YPro;Hj`7fi45U-^$>N88qarRMvb$N7HZ)Ds@jY--L z(&6m{p3?x&VQn_-qtXTo4ATHShmp={fM*}+kQ4e7$Z0geWOJuZqVrHE(vGfAp3&UV z)x0=Vv^0|6jjPhr2E>+zX+W`3Cm4zv82w;}XnKKLCE^TcN$8AjJnjl*yp z=X4Hh+50p&o$(l-e~>Zc%V|uX^Wh)o4_XYvj%ji_V7Oe6E5;$nbre0z<~+G(1IqFY z_8;e;AUMS9D9pUDV>vLV>?6!FLYV6*!aOHOnC&ElPpPH4O^l%ZxX!;EkPCIOqx(CJ4eNwgWE8-G& z$R`)W zl*iIP_rg=uIUISM;i1kmU56TH5FR?pKgpZ2YQr z-F+NC>y4swzQ=w1z^Pr6r`>+jxfi~0&jpulUEQ*H>$lFoc&~3>!jXIJH+8oihF29MlhGq~uIflME~UJ-KC|S8^bbG# z<#g)urOMxY*>&j+i@!+axW+5T!)@K^@3dZ#zGwcWUODcV+qt#r_=!skEyEFe?N@Wj z$upL|c-2km|M}e4(l1?obNadN8`D3&@~g^!=87BAn=ZSSe%GbfU7SpRpXBk~j;l!) z-Dxhz8j;2QBn$pscjnv|@-4%TLxzr6HE+@SSCY4;U;X0O(?3ssJ^lYazdZfZ&##cO z+~k#m<-xLe^5SdMk9G3cMOUdZJV>(8bwiiodMd*~RByBC;(*b6j(hZ?t6q5hx|M1E z{qjq6fp$MU~|zB$F4eB3UfCG5w>@E=~X7GuNj#Uiu|fhVONKKD}0C z!S%o^!-CJGuR8tQ^?Ok}i7UhPXU)6)w_jeB{>_)~O#gPN_PGqdy7qQe7A}XD#qu=D z@GR90mIKPL?h@!j*G-ng8mgCE4*p$o(u_4x`IBtM{pj+gufMtMo9RE^c(=;`_si}| zb9sJ!{T(8YZzvhOa?P#jpHQ9P`tf}CS5%$+5!KHP)He9XbpmpDnCb-D2Is&0(+ggY zlEGKbz2Nr$yXl_v|9)k4`cGf|wkpH#sVrOumdCHJzf+arHIfDEfqz^#Ie)I3PhYVt z{X^0N*9UH+Tqn2;+(sX!_K7;lWmwp7>g_=pd~D~BRIOk9#TPjLZC|@D{TAiVzdzjY zt@IX>1()Fsk^%I7RaK1+|OiN)JE2 zlH?#|U^(c13NpA_w+U&N)V{V(IpoN8$?F(|=awxJTvB zvQTB9ddc;HWx@54b--w@YZq)o6MP!~foka~FLkrTR+|E*1JxBv6j@1)=U`a|h| z-TII!!(UfCKytWG)rGgddXK6PTn5$!+UGAw2V57qf8uhWK5|{)_P})!ef6Uku@3xt zu=wON)?kdF{`Cc&_jl4^^n7s=u8qzf(w z*9WebTn5e`W#D$n{BvF8zLM(#biwuD>C2a?ddT(Q`;8`SXwSXBlwbTo)k&t_v&&=;1fgPokejJzzbs4tR`W9dQ0UF7OY15$k~S z=lVBd=&%vD&+WWj^Z!`-zbhY0|A%z(&Nm)W`ExnY7C3M27yigHpuUOu=YC1o1L_ZG zj6gl)I>2K#=gK zuYcq+aNqbVs*h+3+#hgX!2KfnBIM8Gv&x^!!1f#3AJ+k*^LqNu?@<}PtN7>o_#Y|< zw?o#!-^fQh;5xuE(0#(H^c&aRA!DVkhtL7)Ah!YTi@6SR{$HNi{K9TKj;KQZ?B8&9 z%hL2J%Ae{0|5*O~WBK!sWx#qsJwU(A@<-pyeKL=atOKqCun*W);ywuDzpj;CjgI0qp?t57t9&i>e(^U%+(`;~~o*V+g7-+adGM?SOyKKljT#Myvcu2HXa?PlgQGPJ4lL&p+m$ z+aKq>^sIR=MA;q4H|TR2p1<;|FVMKNmGzIYk@e4QfZL&J1C&4agWUeKt$3@l6IuSx zlTE_ypW58k%TGCL&0f2XjJ5lMc_VxXmEqR&+HWWQZe{s1-`obdKjv}Cw~=%|ti}Nv z3%UPg-k%}fX>PK$rS_zyyAR)~3Umwchf}{WW!#ZdpStYY7uY`Gyx$_dv+TL;zo~2j zmOa&dYI|Jox!+1L0`%^*kSr?dmN@H^?pTlo#?p(B>_;XiYURAyX z#`p9hl~3A+tOTqCtORmOV9O`ASNGMm9IJhWKrcPElQ$Y1Pt%LUT!i*|bICahgM z@&kS32g)di%I|2T7Vp>AJI*#jcMsY_*(A^k; z;3s97hq`NjrtbGl?ke`yXzc?Re$v(H@R4r%r}2gxa$Jvja}?+v#@YGV)7737-gg9kSIa%oI?i{5PMPq{yY7;EM+4=N{JruVBYX+}xyqB; zEweoL-8Qd=isJ3j;xe?{tnYQ^@O9U2U;WxI&(Zs67?7MNAITr(ad(mONI7$rr)6Ab zdG1I)s`BFDg!5;ajeWu|&mAkPCfu-W1i|?GbWk4TkMblBmGVGNQV)n^q)Uv-EYG`Z z&s@*IC=Z9-oBZ-{Ib!uC`FrKrS(gXp%q5>>S!Q`|UAEnm!SX15UFVl4tG*^l}u~oUt!)0(+`{l`|FXRvTsNyp|8> z3(uHwyBGF|P#}&D_XE5S68$OTF#QE>bDH>D0L~K=8ySXqL0=f;1$`m&f-v)fFw=vJ zD+BZ5!?$~3=EYA3mE63bFJ_DoFNc55;|2X_kQel$%nQQI3&MU0;>*Ci`0(vsn0fKj zeM)gMfnLy$XX6EZaF7@D!ORQ7%nQPP39|EY_lb3?FL#{wI)s@Qg!7@pyo?|~i~-=K zqpeHTv02TF7Btf(e5hyW!-M>w4`+T5W_}RPryQ&sAHLlSGe3U1Pbr&jn4cCILvr#X zV?dA}i~-CK!pslCekro+$A@pPLzwwNI3GID&sZ6Qa`A&PB*+i*q4Vd(bU>GudQQYsh@UeIJNfBwbzw7Bi?%9UcSSb z9p^QT9-LZx%EZTLbMt|%&5OI71KT^AFLn-`+tk(M95`=&M<*pMpxfo0#m=S9WzOP- z^IPdp6U8`mDTgjv)YNtM;paMCTv`sOT1f#fJnNEXbxHJr?R@hbb&ThCv@L3PE~cXd z^gGPyy0pF7>0ET4^5-pRI^SufkF%V8B-3iTxOuLlinE}()#;)RmOkp!>a?}fCtTEU z-bLWE+MDK%JNvM6odb!={I2GOvpSmR(Rp9GsDBod^3&N0SfJA~LFdzZ)Wuv7MM8t> zv;#X@nu5Tk3emBswQD|?rnUK! z|IgT)gIRW5d44y60*UVC^q@7I7Dl2ZXr?7qr$(AuKoZo7+N+vXsOri>0c?EwUcWW- zG2h0wW&)p3$af=7#El#4*&|{)qGP9n zY$@4xn`$~-E_x)a$5e8dkKS(8+b(;%#c0`E?KVdR+@H{sE%0XS!+m?Zor-SNV}E%v zUazO4agXn<`|I;+f4QC>>O=dh^WJo^+4kwEO7Y2bHg;!w0B+v6#SjRlrhgI9{Z*fi z)lBJg`0_}p6+OP9AoU*K(iy`B)06G!sYd8XKN@2_he!Rz;dsT`en0U-wrPF~)q}-yBz)nR-f-J44Ly;cl(Dx<6V^dDGFceyh#V^7MFoUd^ee zid!vX*l()7>+y^kdP46kx8vD(UTI)XJoRI0>r3nUOKYvA{r-raWjd);y|me`zqn0d z4o}~VqZb$z_dvC;ukKIQ%elUym()wQDwHtHGd_68T4A0om~=uUoXe^VH?%bOV`U7N z^OZ*3gR-d>%Dk!Kcb`Vm?#*~MQ+tOMp9IE>=Fx=tz7mjy z#!}T$wVUx`!=c25J*vEPtHo~K8_c*vU{f7|A9^5?*>rne8Anj&tiNtbWogj&G>0W! zf2s|fYZk1gC(A9vFk=XnyT|+@><+pF-O_c_dB2+OHjL{2a5i1hN`;q`$!6?17!?m z5+tLe@rosw5%)NQnMhXI7aHAJ3${J0R{hO}rMm`as{U#qW8^0;BNc4XyfIIn*3us2#xGz=_?<%GVdSgV3n`Wt{;=3u-{d4DpE zdH2On!RVl^@v^FM;bu5yc23ry>BTyp)XD=;Z2-rFt+XYnS{}31W-O}d__Q}ZXS``) zwG|37yx<^`t6BeCquO6g=UP#6k#Wxcdfh+oMXJ_93)l?1ij0iv4|Th-&uyq2DLrQH zfZ}HT!FcAiyg%#@kC+-Qj1?oP&u{oXliyX?V71s}xTgq(aCb{2HUM{r`qb3WYg&~M zAUmqTz7#K_9c|uhnZTopb;9WB_uRime2Tg}1w5aeHq?RPLJ0lM>P$c9l|dUmG48Pr zw$sJort%!>d6lW>^}S6maY`S2b20{v(>Fl|6x>865kL+3?NP}NQXekC&MQ`+e;4CJ z5E>oidV=7^{#5(gow_@zc>!qJFPYT*p8NNWUS3hTYOsbd_J~FOA{tBI6<$s@Uodta zOP;JvP!I%l9{3^JtDLiH71fG|>g9#Ca%WVhN7La^Z|Qq4D%hB6P#ssJ@uUZBQ6XHE zATpx|uj!#3YkHpc*M5>|vl2ym6rLNHGY z1^S|JBHErK`RpC-pv0QMG4fueH$kBAucn(5*hQ9g&o7n8W;xkH<^a%@%Gs< z;%%~UR{>f!*v(i`=GY;yOd!zwrcXjJ+@3mz9vl6@=`o%B?)(9ARYg;X#sJm^{|76iSy;bV9OUFLKLZ&eT{-@Z9|lmVpwPha=$fDkkks{lGsi z$1EB~pP`R13}J>~(~uOHH#vo1o%>LF)|6^89y#ie#%i#e#8(F@MEN*C zdI;8nWSn^_)c~rzTu8hrIVNPQBSTMBEegJpFJ{E8E-4!bGM!YSTgb+-5spxMPpgJr zj`#P2z)28BJTU8v__7gS3h%!>9{1LVgTCs_1f128`Sh#~r`Q^~nbUAX&Kp7H%!vYe z9!<4%+N2RFJfn#Q>$q2;`Hhmpe4ilMlUYKE)h`r4i<643?=9gbgb+%O)%48sqW);i zUn-l;&N|aAhl|+nEe2m5$lT_axy|oDR1tA)y=EZ@P#rSt8$noC5GZ#$Tg5?lQ%Dta zLD+()XPv|xtt$1e!>jca;h1`!0PO|eyaHFzXTjr`XA8)1##9NO4I<${VUxB;>oLON zOtL&2nd>F-tri^!{%cD5l(nlY*N$0Jl75;LoP=TN7hk6CVMy^)^kvtUuf z>dnMsp^0Fco=$0=7G^TF3PzJdTV!!}f6(8;Ed^(MG8U>6>gvM=RH+a;x>+6FswOw9 z3BR|I0vX{k6izH1+6AuYT_F+fB-qhzHv3P0jDlgP(V{&)Iji_J&0b5 z&|rK5v&tYy%8L;5eYps#x-Ws;d`@sD5!}p|OUcBDYqMd0wMn8a6$=blMUid@ojf8N z#og4;t!$n|g>0gO%&o<);@3J2sd^?eZhvelolohl zU}ttSDKU1lqfptMA)D%aWp;Y4M9Kq`ovAC z3lqZP*rM@((XECfusiCeo=2sQjnT(6X|pAIk~$6b#5o+naz(>gj)Rl3boMt#biwf{ zimIpt0tdoA+}?7*D^nOJn6>Fb8mhSlMWWN(#qH1ijd<*s`HpLC@6YK)dSfmYTv#Nd zt}p$ditdjx<`?J*R7G@9_J@Nf=3ql*IB|I8-tUtpRvF;a(aZK;a~P zT9WQ2I4pwFhg|^D?R{;q*`G7Zw<~FMcPF4Ckwtn)NXf$1s@g3ySO8V_!oP#Wcqj0+ zuvbjS&Df(IYw4r{fv%^@?|zw7(+YjikAIBNDkYkQOwir_=u`_O3kzhWm#Bz{^K#3| z){NF1yuPo<=5Me6ZM6?(MMkSm;G{pS#v^M?Rd<9?bdq`(fk3y~A{|IfQ%FO{LvsLD zRYm55P7=q?$^;j|xsOk0Co@Vt9P7zy)FUW^LgIHlgOtNVnuYhG8{i#brw}luKHf)F z4J8i^HG?!oUPKEhb3}>J`N2~#5-bU@u4cS~1L(2w&F;kdoksR5fGtcYZx^HyYxk8n zf~fD)Xy)#umsq;!B=^Vjd!gdVC-^CJo0=A9tk@Vx#!3p$&@Py})>J~H(d)0#(bjAM z_zfHiafI0ei$=gyL&~)M;GYysPKq%OCW}C5HjTqktOY}pFL+hN@X&^a3v&&(=m@A7 z%e_uxU{>cI{$Qc>MUL2^)L;=of(Y0wNAagQcC+a`LuE9@TeJo@7S1+Ba+Da1;BJ0&*^4a^G}sAGw9; zz!WmJ2MH6C*9Q|lmPL>`8Avp5mZ)8C-e8rYi*7*Df{|!#7*B|&46Ud?^+5SFMi0!) zS8G_AbO`#*&H0?htj~My>u5sOoeoir+%GXW1nE_akg4rbft2QH!G|AcWGKpN4L4GwY;{Yt*u=qz(<;$W4&H;NGubKi5WThxYhlHiE{ws5>-)ka`(B4o zbC5|aNr;Uo1l;cS=y@zQ$tNIB8774=W+faNj^OxKTR+Kh)Ss@sf)(f;!cob-qK~xz zeapHHfH~LAF~kUAo^u~A52OT1_kPEHkj5D}h+pyB5=@1bdI76rO#);PR9nr3*rdr7 z;*BT#0a{aaW4sM%Wgxc54I;;(xDgRGb&_hDw*x&6Vo9L=-oe%Pnsu&ZCXPF^XmZp8 z(UJi@Tx)pfDaB*aW@HGJ88UXfub@hU2AnTcgahrjfk$mJgTFyoJ(nGxvEbzNHI}YO z7!$xB55iJD&WEoXJmJc7(-Et{f=!wv$052~Ga#Zs?h@0YU?U9E+?RQ5>cyDK*NgGm zW+I!#bVx_0a;T0+=O89SD?*3gAO_>Ai$IgLRX^=+QX6ks(3R-Y1`E?_s}MCbV2ZfG zOrM|`grnC|flu6>y^3I%?wao%kH@R~GJyntW-1cF=PM{?1fNN06=Hd6B%#n9Q;XnY zlHQoyfx{9}7@-I&gFoQ(0;9mLv(PSL!!-->A}l(|YKA7*J2^~R(i_f%LKe%#9P7k{ zMU#Xggp@X1cioS$7)MHw>qO-l30$5LDr30}b$faC46`Y&AwPlT(T<#m;)^y00UfUz z(+HXy`jCQ8sa+b=;fD5?!*L&E??)gS@Bm_;=M}hp%K?SK0iH5T;J;z)#rIjt5p5`jQ;qS_>n&GhE>4#HIb@TSCJxYaVhs|Rj3w;;5{=m@G##OS zy+W=4!ynhcLH}4*ZW)^iSzjoYqGelG+2&kon}?eydk(C(h>Q8wpd6s844h+arBYT^ zB{>O`XNh2E5pQImvA<*6rNc!N9NZ*?9O^InvC;Gs$6LE^48>%bNvZlHtoAiG15@3K z$&OAB*{?v18L^8~)Ibnp&&(lS01Fsa*g3Mcn7fGCt^=1BCF!LOuUB5GCBGU_n7+C* zTeax*-S#$QP#~=~_xcw)p*Mh&paxpj3LrFSd7o)@=jb%i=tZ#_na)&CyaGNB{a>wt z#DuswL6N#|>lLIQ*8rIhm|HEerX|VX%dud}1*0m-zQuvSut`l@&PZGw;#iWbmcA@9 zF%^>!CK!zR`G9|x&P0O4c((0ZasV|3O^MG0v&o7>-4)jGAP@|D=8$n`Q8R!1J6+c9+r|Y0^d4;fD=ruJsg}2nIizw#NIXv zw1WGL(aQwucZ3*pVO76mbE>F3t1`pl@hXe|21L-jRRDvTQ{Q7G>aeDC7RV?1%W{+L zoM9X8VLU=rm!e-;QW|}0o+uP9mC;e1raYCB0lIJfsdh z#Yf9pK|fFg22jHOEwPl6k#_z?^+)*BIwyE)^n)l0Ppi?c8l6-~+N=ZgszQi(@)XY* zTo|N+{K(2ocv1U=NEma6F9$~&XI2Ozp-uZ9M&YE0N|{=tD%I(R>J;3#8thpd{FD-M zU}PG=#YQv2lMv&m#7YB{IfF_I{!|!OlRO}YD`>p4DHO0Ax&8)>iw4V$f9x zC$m}XJsguJXAd7KhyqR``Y!4|Gf=9##ZI0#6}hFK+xn5J2LC0A8?f9`=o%N|fF_i9 z&f<|9U`FhtvmfgK*pimx9_N7kDn?cRglRI@^a<754cS*HU)uS5K|(fFh<`%b*!n=x zsX^Fa?2yU@nmnAM*Q>!;^TcOhf@X$JL{a6fhhazvUMg|7nOaGMF-n7-xp$4uC=i!I zX~Y|WcD-DP-vP}zS1YWbT$21kgqDMb33rPcp$scfVb2wm!Z*2fmZ+3YuAQ|`miXva zv9}Jp!yUq=ZZ-Zate%>5H<{#!`M3gr!%jd2gQ;AvV%@6eTA`Q+cDeEA_-u}xLY~=t z)*WVk(UXCsnK2AR1s>8cT-%swd__~pkR}?kimm@+I#o+ji;ss!$ks(SLgTZZW)p2D zG}s~ZV`|%NQIuwC$fXSn_8Q-(>S(z<#)O~JMPlTA<`(N1^W{FBQYIzBnxwS(LGgcC z4Y7PM1Ep&n4VI(xL}+bD^HtfH#kg9x5)Nk3Sm0>EE_Qi4EqK#5#7$JwKKcR{c(x2p z26aj47D(vHr9smQGKV{{CKxO*9NXzavm8a287R$expep^sNx$q-)bW3Td1%mpc`o# zmz`mI{7iVyDG(c|#X^r2gBayFgc^V%9YlvRkI+Qp8Nqcc zyn#arqiKJDGaC9^v^ais1Itp1*$nVYD`1VGJy$IIM2PxEBPZ+euObxvpytEV(?T3F zdg8{_GhwyI-S?2T+GoHB4~%aszGg1rTOx=V=3C)nLX%^)J4*u&&Bi`G7_`D|7)DF? z0|f@%`{E|0V!j-Jb|Gt_uC`J}BD7F$FeJpRXbGebCGtej6TK+CuAH`fGkxO?^Eb)O zEJhiSokq@NL6^dCZi)pFf!@e3l+|V~LH=cp;46E~7k0I6)AEp15HTncml{bn@Bxll zgFQz^2*P5D3YQrjPc=fdAm8xFKm$?!iKWGtS74qAm~U_|!S}aPz!E=#>O{1qqiH{Sdwm=X%nBMk9m4gr45lKQ=zcPf{S=!L zV^+R)z^qe}zc|^wd2?=)Gj7#UCcWheUgtNs_jBi$aX5_rqTI%`;5wT|v1AYs0`Xr$$Y$wa{+7yqwzS_H@1Z~DWk%G`q=g$CeX2gZB|>U{na)Ud^kwRa1emWaOg4P`4~w}n+QjzKOdhz6drv5TEKbhwDi;H^zQj~ zym>KR9~dW)l=I>AiHr%jFrVa05EI$_hE|0bbvs)1ccbYN9-kzfbw}f+czjKdOzFlD z0b6O>yn)&Vmlqcjvkq*EXNsl;6TLU6$w6v^5Jb^wq?&UwnXkr&5=Ny(S%a=nP51)} z^Lhz~+gKL6JEKSOvO@#m&k%RMQaKxgjM?G8M01&3APh9SF!)R>3^5uY;Su9U_8@t^ znA`7_y)u%4dv8)~JMSUL&*X*z)XbRt&g($$j50T1U%5OhCH#h38Zn7D!D!fN=brY* zps&I8ZF*)pAnw}bQME>o(O>irXXh)tZnTb}gN!K)F{5)s$ql`!sWo$Uo#TnO#~8Jb zi_I|p%M5Ksa8HDeX5_Fx$K&jmt6Py@ftVGdB^k6_+j8T!NWjpAe|V{R4Um~I=KCB4 zy4(lF`HDr^1yiA(LWv-=R|@=_)9ITxGh?>`krTRifu%_1LPB;$IiJ%yn4^hn9PH0- z5Ts<~Ed;Bev#}>Z33feq4jI=6Gh?`-0Fu*SwADh*2z0ejjiHL3sanxL5o3S0fheBq zNAB*j9LYL})H{u_k#bYaN~m~*LoJ~ERM?ws4=#POmi7gQK%q`URQ!!EaQ{kmTxes& z*GuR-!!%i)>nGprS#ZzT8 z9_+w^S`A*&4765O7)K4URg@;?4_^85ZVG8x?Lj9r-2PEr03$kfw5_&37Vg0|KE(YFUzn+m8=V35S1d@@zsui zMeHPlu8t|m*1leuX|>{79T@w0vLae3E<y#<9utQ&C$;T@ zq3|LJZJm2gT0>`)n8-W;r^r522%;G0)wrk-`a3)tP!Xzu(FcfeHe}&c;ELhEir&f; zFMEsT$;T{obMobtyjUol6%l>L0iSbTL?7n&H7(@0fY=$?6Ojs0!6rlC6xuA(rw-rM z+(a8vm^<0Y7FEQ9t8&fu*U}y%J!{?B+JI|UV=2;7?gUtt@zKr)sTlG=J?Bs$k0SL@v#h&sYZyhq{mAqoU`Rczb^foZbSN@@qFikb@( z)0;wdXo<22O-?p4Ju0Si+Nh^p-i<}LU z&Rd5AyW*mx!X*#e27^T-5ZC@`K6fXnFqiXo!U$1Oy0(c&rY0V`p@^poLV&{A#BK$~ zA2PC6sns_n3O7X7JpNwLY&ITXSo-K5hIg(RTc~fk-5T1n?q@<#Wn@a429euW* z*Xbv>ERr@*flI+fw2&6wxEjCiXpzRzaeuPFl<>#G7`9Sv=}M;9cm;Zmx+Dgeb-_jK>;ZdWno|BnTE?LR>77;Farb z^X}6aubA*)Bish9UlMDB%vfe(Yi`7HGM{}*eM`s0WRR&oBo#pV2y{* z{IL-nZrw0O9#M_hk5y{t6<9+Q?1}KNOu2%5U70WTo0Re{R4DN51t5$x+#~oB4N-lDuk+- zWxj!n6(W{2-vOd7Vl!zF>2~IV$;Dx7<@fsi)Cel1Z%E^HIHlCdM9h%PSMrnqG~5kr zes8h!Kr!u<5~M-IiiJoDHv_AvI48ughuCaqs!?Q} zOM)GG6$3=BTTuzhfo@yCQqfBwkFa;s3b(Y9W6Fd(TML{srIdHLZBG)%k{fdvgRM`U z7=Y-tha8_C^>{=Z#+CCna5^^Iqp2Cui^r z*nx$RS7v^}RL5cIw@xZMe?i zh*CRyVe*3&4D?d%gC^y5D{O2<1iQXAtZmsNB>!1p4UG$2Mt#+0otltB}>kNr~Y0k87tOw_Z?o?J?w11?ocY_8d zpFBk8$b*Sx!WJxb0_L>hPeLLaP=xJ&s;>xFLAD|JAP2Ry4zW{8Sg|%J6%ymteMBcn zf{_-4iG>Hqg}CS}W*>MYZ}AqZU%2Xq4z&!<(brbHrBX;PbJhdYUK+?!t+2C!_3J~4 zT(O^d_aAB17?)zOi+v+d8dw&thX@BsP$wOrKG0XYvBpgS7Lz$raw^uO={*z-u(-n| z%dS<_a*##m>vde`lyGp8vMOkS2lHBZDGS0*$9+jbvlC`jqP)<>cnr5kjOs?d>BVUc z4IU2dt@>|~wB&>a%dXX2N5fXzL_uEiOIIGbT&pqeiQ?r!%h(>MUv>rLj#grCz!h@X zQOmG*``5$fkFFUYmm-#^9E1WLVIPeEB_PB+!e^HaK!8m{Plfr-xY<648#6DJy3FD` zhDA!W(Vx>?{z#k)^gTr+htfr*P}3H)3hT8=P?>tA$@=j})dAFzoRNC1MK{L_d;~ zAT%ghnj|Ptjm@LPzUZL~k zn4IaG`7#q6v@JhXkC7PGeHlk(cWuG!2Gs_5Nwxu?TWPQL$@<$`b#nQ`VJo!eIcR0L zL^I<+B56eGSQh2MY(ZNmkL4Py-w3WVmvjvcjUASCSpQh<;;lq)RSy}NB~jtS0`)-@ zg6H&Ox-i`!>L+ZA2t_=4yD3};yB0VDM^EQaK${~%odUT&W8N>$E4Fkv5>K6(VJj$m zS_VZ~rl4HQ^<)_HbSGEOHSVdBI9&zbf?mh_$Kg2hltx)uafR2f&T!&k>0D@DG!msd zC`3OV6LCN~+z>b#NU?(BGv{Ys!lvn0iv^j2IU}ET=SSdy0ch+IjK)g+metqz%it!Z9&u9BI+!y`@+Q5We58;<7- z-Gvk?$>PeJA%l2WlvQRfTo!VDY{AJ!cH%)V zZ*%|yA<2Va=t1fL=&?@HX@T1FV_D7W`QDLyJpvDE4XZf#L4G;iA%9C7GSgqCXqk(fx?1#4ZOC1sN)n3$DyDyE7SSn`mD9*2Gmy`@i~Y&aq>0~Y9yL(uJF1bn_ekj5NEnF=Hg(?K1@Cye_q+L zoM{s}N}F_6pP7J1?z ziCI&L6BSz%*gq0eo$8~fDe+Kbi$mJYg?7T}*a1WynZN*XJ{ME3N= zzO2f1iT0G&dh@n2G{7Ku`@5^b#+CYEMb^0$H^u^^+Jj!aF1Bc$2uj~VZov3DoD)$T zj)Q^1sd5Skgl_uc)@l|n@hw77#6de6f?vtuj_Gdmw$?VtlZC91v^hdSH-)GYZ@3Z> zHuAv3aCof4L-HBqfyNZIBXF%4N^SYSBTaV!>;u?RtF&DdwoKcDgbiB5Fqln*sdj8= zq?ZYyx1?pK&^tW~w8M zk}4CK9E?4fNgc(6db52U5J7Z=59uj}y*x{rXHE3TSc^B}qFiPj(xVm6L1f3NOYt`x zD`fD+tv{eV{x8>v-XVsxO4#awnFiQrd4jw^HVJ8MlqHe-! zish*OaPUMiIeX*zfYYnxM__yisEh5ns?UOC7#Qp@F0K?&9&#mRmrva zUf;YjgOss87&}&_gRRpDv*Mc6;?mGLs-!ckCkULEz?%{X;o@kjY!Pl%F(nVXQ)aCL zB!U=b%jP|w$`;8|WkoG54An8FR_>mvK_T~Ch*~XM!{8iz?nw>MIEW!G9|;zyj;2<= zQAlWN%L~Q}!|ZMiNtTh4+-1=Zu|QysWVRux#1m7rqA)zlAh98s!V=>&0BcE0ysR$e zY)z3sWXcqw=9?v+-)u$mo0`RX(X+X(YQ*xTI+Cv}P^uh;%9vHGm&a$Yo}qTsf(vp;?SrbKq$5KeTenUd>te~Y@1w!If zsOdcu7=avUcXKflD^a4__^b&D<01C60Ji+K;;X>#Sb|S42adt5{SbsyMrJOLKD6mVQ z)pj=hAde}lzS7l8Vt4b%P@`wK@uNIucK4cHAYV9@_=z{c1^(~Z!j)}KSVj#PTaS$I(AZG%u;sEl$g_)3m0+=Qomu&8G!jP6&Q9G{NTneKcjL8Xr;=I z-HKYl(T>Y~z;?f=au%WwUnp0g3uU`{-SV9aNy8JZRhZorTHj&ov=|K7FI9#Zd_)(3 zvs&pLhSD0j_K+5srZ-2Hsgwd5Fj$w`4z+;>y>%?*#qH8YMAm-nBYSrUE%7>STV%0p6d=ULbY{?E)}LwldghtPN;L zeJP{)U@mg2V3PEpW7lDd%Yj65i@R*512KVZ!ic(HcZe(k=y@b2SmL&B!b&B?QIB{? zrg6!Mg;dl1fO^~_=#$kVXoFV-J!BCyDbhTV4i53#C90o(B8f`!Ad?PzebZA7EM+V@ zr-7y0M+YYIjxtr9)?_qU^VeH23#{FINS!hRb%CKY5sYatb;%CcUMDCzO{H2(qCm|H zd54G~u`5g=^LjoFNHJ{N#?t6tn|qP3;SCaiVIg&YLWr5Vt+5iC$ZLPC(Q&q0A;mom(-qF0GgdNh@0H0)-k$ z6VMxITfe*BQC|Y1uQ*mW|3a=BUS|)!gBizJ47?5DX)l^OdLgYw2dPf0MW_--qoq@D zw6a{O&;~O;28)_s0oIqV0LM!lqJ7mnJ2w8S%x_6<##-)Qmg)5cBAj8C%S{GCH@==m zx`f2)CgrW64ZoiAuK{RoG5s0*$PNOW#74~}x$(nV0|Oz%8X zA?Oi$9QoLndw3`AJ^VF92Q0KAaz05jt_u+JXox0w$ye5$awyzRv-3pBn4T1=Nx)ko zmr0SmTL7QnqxYnvj6DY3ST_SFwJfq>ej(%COiHN>aW-L!8saS!-B|bi^ngg!1l1~l zZnn7GDMUzrr2x9wICe>{V>MB7)&yQ`PJ&AV!WJqrgKk)t)u`5OVBj16syq7(08I@xU;Wb5J=w=gv1dSFFyV4i$}@5k+2C@-9b@iee+r zPgHBQ=MMs-qtGEF-lkWL2Ao@inwL!r8kf_fF-PaYZKZ~cGN$1gW2H4JWBbMXaSK|w zi~bZp_K_}dV>e7dG!3lVExK{kx@X!|CC2LGJ@#&Pqqp23 zO*hRCuN5z0RY}ToJ>8)MzzPyceMnA;u=d!!szH(Ic{BoF*q>Xf6|cp7Pj!&u{XAZf zX-2DDlNw7os7E1Ag^=Mvovz;XV!{W>9+|KqoKSdVS6(^!icJ~s7}ya<0M-DmC&9u; zQth{ z1yj4{f!1Qk6oTy(n&@?tXHQ581LtAMUsfSDG}SIyuDP{Nh#~$U&%BByY?bmB#t52@ zZR#@YjfY(LgL;|WQ|Va6=1htEp5B&WYX{_>PD+J1_{rD=x!E}CuUq?Ne3$Ms+Sk*H z7hW=Kgt48Cu4jxfjjM#8YRRNl*3OZv?F~EWmQ*Tzouxix?_jUbT-Vz6zN31&GU&SH zDZ%w!Jvtec51EPa(I^J6Uti;P5zGu3y^$Y~GkJY458=FZd06oa4m4y7JjC^gGjtAT z)-ea+V?5KHnuA4ZHKL5VZsyNqymG(T`V z?5x9Kbh%OByO?O~^T3ZbnSxN(%hJFWE~8-;pGe%t##tM>oswj&7{3kE0vm)ixSP3g(skCeh|l_!lW#9cN1!S7y+K7(A`^ zwEvkzFbBl0IRFD9W9eAyAlx&|iO4RCwH^LL33`IJSd&Ku)Z=h?J8NYY>WKXz)G~D* zP0fmJtuRdT$d+cMR#W0k6K>VvsX&xTpy9`8&Y%mQhRZm!2d@sM{R!>hC}xh(-{%bd zba23w%^9*DIcxH3Zc_EGBhJMS7G63WYcId-=9R=9fI7$>46@pixAx5S7^q74V^#M2 z5fc|@Wo9GXELsn#F1MIt%U(4WL&F+BMZhhb{)nUJAQ8dE8~Te0Y=X|c5r z&g=3+8x-tGhRtER&N?YNW&^7W93nIVVl~#vI@u{-oU1oEiObqQxy|2)c-CN%l$FMl z${pN})Sh5r9LnV+N5^D)#oDsb*vDDLM0Fq(e+)cZc!it9^Y)HZsb&EhaSG4^2y z5=EAEovjm5qr$|SOT!675=-{srMxuP424dt!!KQ(krP@Kh7Z^6SDc4#b2R)=$>fd! zbIA)x7+t~OW#u=}3bqooe>z(bLGOS77!I6frHvFc6Yz!fIB+%RjyWpy2*xp5W}WNU zD1+tj_Qg%ex2KmSDxDfxl&1=&-~$`P#)b;W5m!TNv2aO@PoWRKN~ejKXn_>;tObEB zz8C(M5nt3T0Ne2K*kpPn6&G5VO#F`IfP&tKX^96)1bk0S+&QwZ+uGzzn42lR69SA6 zmG$7!V}#-FZHhI%IRk?CW>rz$i0oU1ULKO%YRh&lD_%ry_>)6c6E`QMm z^<3-R39j1Yq+Wa3_9Cp)+ug4bW!n1W_ZBFOCYjwHaCl5`G>M9m_kjc-$-EfDgdT(# zbR-CKpF3|DF&Ih=O8R99#FeJ_;AK?TpK>ZY$8D%h`jmT~87a!nX@Y>1qCN@HbYg|E zc?bbSv3W3Pq9*j1a;hGjUL*0j;=mAec#HvA0*xr3m2peg*0OTqph6DXjy=;g&(K7B zCl<-$kU_ASYLC+*t;Um%AJki1o*X3GEExP{x%0iV!&X`uP(8Iii@;VyXTJ;rxPhS- zx5fEjafOI=*IL`^g@EOD#c&|vZSyHX$lNVPh4zUT1=V{x)I?9);Ts-jx&r;>j(9c-zBWK0;sl7QnwaWarZH&ARxtiOoM1GR#|EbPjorJEUn42;T-UHg7hu z%_QDKaqYYV-9}6~%wyJ=(CSF4GVN;1POSSh`w5vw_BlGabRzy99Y7Y;&QlkmThmvrDy^v4n2S*#1Jq1-zfsnOBV}P&CIuzvGZBuxk+5BpiTpv8Fg+F@BtB$% z%v8y9Nv0gLy2!Avr({>A_+N7-akJM-HFx!6dMX;zbKh->OZtMs2L|9n=>2m0irWr} zN23&*1O#9R@XQUt0=DF5PbK ziAYDpvw!Ai7CM1>lP{C-9_LQ_ZYF|$O2etPNSesOyNCQZRK%M}y44L6F`}cah2HDH z_h|$ys}=sV-*pTj&l9K#N*We=e`0foHmR7gk)||G;w_pCo&3brFJ%4VHWnPbO~Q|u zRULsbCL>+<%XFx$ll3MNY3g@Qh}4&U#G80AMgxewx^nAx&M1g<$;F`avBVW} z;@Kl+5`?-P)W<2KZ~)q1iTUqy{ro~d|G=dfpY!(%{{BHy-&5&?q)MgiIt;>J_aDB2 zgXBgC@dqMW_Vwq67x)A%0~9|?!=zUZrJ6sF$5gu=y3RpxZ7!A>BO<)gL;}^Q__9wr z!nsL$S)i0mCH3u+SD$X;9&g8$llhdX@m`75@A|Wyw`)kg--%Ot9jD+s>`SiRXizx4 zQ+njMaOuEDRndFHW9M)v7L<7DauiIO^1Tlu4zogD$W`o(5DOGxQrGJ%iIN-^gN6nf zbYcprfcQloLQt7ite^R6YUd|pK9kzt&>Ml2bfzC@(VVvev|JuuWfObETc1zX%FI^+eNYjPv0wM z8A*pS=}m8+mpDwz zV~fm3ob43QAAU}mTro0eJowZ z23bIx2)%J`Gd<)8HM89CiwXznKH{cMaZ_mG>l=2&0>k>|NCqB(zfZ$fIiFV$&6yH6 z$7KeAyQjH%dV8B&lwtHtwz%$qlWHZVTVk4_w&j>vE;b5Yr#ib?o!zR=Z&v5GWOL~e z_akD!6(enM>?w+xUkP;m`@cSIuYK`YS+3WFrGTUxZ;#((dv4O0LE_E>|z>hgPKbRt(DC)uce@k zv#>`~JIO%I;tVl$n9fYT1SmJI`m`pHKLi}yi^zl$MqEYX#$+X$nF4csy$i_$&TO~S zisT&=vZZ;3V>!xDg>-)rsx^oW zY%<8kwDxEu9Ekyo9m&+$}9M#)Hqrl*lFDVc$cgewZ&TkB~_IVmepBO7=bJ6NT9=#bPjf_lMf?e54(^Dobu^P11^S=1sjt1Cyw;obFbB9~X)-Vh)ZyD`$FkJ;nlk8Y2%UuE31j6M zQ!fsdjCtm9tWlNKF1M`27atdC(7#_prd&Io?LfqgXeo-;0rJxtK^ZBp=dJw-V2R(aJ{!iWm^*T(EP!WaI%o#K{-2wUK^pUsf8sri$x|PFyZ+^y(2-4*^-9 zRcqwtBf2auEgn5x?F1=6ET&&H3-tzknx}&rr-Pc+$9R%;f!Uq*y&8!ETF-!JL@-t_ zZ}=40Jc2loagfet+(dRo7fZ9tH$8J|pl6gp<+3NZOz3Bc#S+NRdH1mV&g1Kj#;8Y+ z8W75i7Qh@}D;_2mZA31c-u+&QBXmm*23g#+B0&S!Vbd5Ty(0MSAj)5%F2Pje%FZ@Q< zES1nMtsJdg7KBfd@3iki4!Ru-w#_{r^|{%X8!fs%jWL7^(*gA{v&&ePRT_lMQ?v`> zOSqr-78R}L$j4eiIo=>~b532B%^)u@m>9`6k}XgN)3dE{Ds>{vAz`&y|$UC`Iq#z-b=8FFBfi*+b_Brh|DuqX*v1vsow z))eM0JhvFS5832JVTMZNi2x9bmkVDBa{(;BJ`<8| z5^p5E8vO=)ubA~ORO=LShw4Y^63UZVtl4unL;WXfAKapGwsg2*hvT6#PxUj|d1u=s zSI!8oD3+59Q_=un+0k7Um{OK7olr*Gh{b%4!Uq;H_6Tj1FC+1-7TJk9)rB-k9lhgn z4VmsUurY;;461Y8D3wT_8LCyU^dbgCa*5$Z6gPVHa4=YK47Z znfs_zaj;vS<2xeh+f}pOOO_&<$HR9-ih+`fNcX-;YiH=NFuXDJZ=@J&U>5z2y*+x6YOWX0v!M(Fo%cj$#PUB}PUEcrCf& z4;gL1ZhdWURX~)99lWxK+3!+M1JabDI6gK`7pGK5V$?NL^ac&dAcl{yjsjNLW&0)D33RIs3XB~IKJv#|6iS^hE;iN( z(b2O}fw>s&%u0G2+`(c?8K-_aos-xeC~nr+_?I^$e}jzO!b%^}G!BM5Vbl$+nvVYOwGXP z9+%^egpk4>`TvC6YP0AUH(6>YhipqYtk$R>)3=Q)M-uCTvaF&~ne@Nz>P>+7F1Zr6 zKt9b+V+c*M^8z~EYGUcQ$k~&zLC{Vf;Ql)w#)-YL~b|5wlW6`r}jN9S1H-Ez2qfoW_uG|2) zoCw;4adJ>xv@^uFWMkzjDB~W)OSbCd+#N_ydjmRIteGVSg zdc<^j76G^x5fo`}&~hn;Vn_^#1Yt=D;lQX^c&2E>uoS1x$5R$M^V!~s>nkuV?F6oC z#H~;{1O>9#A3?+g7!p+pWWuEmXitpK(C*f;a+Q!xbLMg;mU@7B$- z)XPC$#iHKC!>%694rZIM)p}qsR<())68NL(i|8ITUyBP~e5oV7ZFW3dO`;*J7_=*! z$RdVc3nc_h?NJTRoOW|5oZHQE54sR#SxyM=ba^g{=81^=fD{}pp!;kk&S5D&#!^ap z5NbuXs*E};6dxrUH|4_J`WFhoK9EC~WdAntU>Q}CqLPDRY%RUPaUvJ(Z=i^Fx6gpE zwX{igL#vbBQ5Rv1HHB{Jw)$PSw#4iqS0ZNSM1_Xj>J*0!?ze&^TdLZM)LDqRuh3&7 z!Bn23LZ{J(b;o(wGAa7q!o{0(PKcX@`Fe~DfoNqgRn8i3F}5DsZtoW@VKW?ubud1v zUMqG#)eoMnT+5yrIJ*?{op~+OH-CWvg?COJrEpJ(eD%wo^_g4)tD}uCRyL0%(T~}K z+=JVVbv-u0YP^xPi)@fF3BDQhN{Cs({=F4;I5-g9gGaq$gRBoh2r}Gk7GVbpC)RxhJYko51Ma?<7-ZhC7ZG7FnJJmREsEo^rQORBWKFLfa+}DgeY0gtJsJ(yESP@8D^2OR z-wv(}k7!J7bb{^?(4~MQ?7rRN8|K(oSy+m0=Ke?l4bEge{6a zeQE>>Uk=u_zCs1IqWgx@cIkI~5D6v&jc55E?Du!~|+y8JEE&(9vP;jN>ge zpb^sbO%9<|k?R+@JT_#B7M|)L)7$`eJm+eq5gsgP!e`;DqL2hVm@O2-a{>38DBjrU z73y(|cA4-*P%V~=k$WfCiO31Nlkjg^;ZHwY!uX@UVvX@0w?u^x5uZJW)M{6d*JbQh zMpaV-QtuWWmk~Tzh{J5dgIPR912tiChA}I_-VVdJoRAS&nl)b#X9L%PDcK}pa%T73 z#-eSca8Di*_|!dKdB6;cj%P-jxzhDEC@msr`xvfA z?1BJmaR(1hDmv6HbkY`})G^70P}qr2=@Z=$l{6rGfEhG}V8=Cc47Z||o5IJACh8`4 zNDym)AX^boMA1PpX~XAC(-0XwWKXe;4mm%gN_9;CBPflxe37l70LAE72~K==TR+69 zRy|}UQ{@EagB*gZkLg}i2?|Cnd?+r9seFiRI8aM203FaNDl~jT zTtUTJ?NgM`vztbDqOEaV1+l7jbP>W$=|Mb)Buy~&Wisyd03UtL`ystKK$D@z|pN56-K`!yC z@O1!mQl9A^tfB2)QOu zNhphT@+B>k4Osjt{*MUp=3I*hH}jO6cpy2&23oWrEgO*R0!>s7^ISdvG1SZkiE--^ zceJip=3NjBcmdUo&8qv=KwYT&>CkwyhDeVYc`PG2$o(+d904^`oqw!4*TouN*0GT8 zH;YjX+4gtFgQtiS7r{(`tH9`kW%{}z_f=qWqustDa~FDi@lzbR)cwdYqEEY=;~r9V zt6VrODIvQMbdhbyid|@Uu(OV{yW5HBS$suGS?is$;6XSf7aM|x9i6-S{wM1Dg6G0! zU1DCxB_a)KG9UuUjn)XLq=#2t?zpi;E`FDfgm4)UDUaLnRbO@mbBmY3UUOGu*95u( zT;>O~*y3z_F}{$M%oNW+$Q3v?A*z80SOY1gb)kUeZkD3mxDPwY=r!?Rf( z{gIeGqZgy)2(r%7;1JadzHd;i9{+W-j@V+wWv+7p;@(JqfP~KDQUpo){`liv}CJi z+-=pXtWUGcSqX?xsgDp)se{ecbwsBI!1cf}G85AoMqaUyffm45Sp*;pk1V`t1B|@+ zE*yH1@jUawI)bdaCIcFSj0Kmh?(oWypdc1Ohg>+M6{4&XsL#0V%6pkoF49zVzEFrp zP=XT^u zY1vnX(-kMVQ|j@}>iAZ5e4D}`^(qQT6TT%iidTZS>hu!~Nzv#oD$l+>%jp^~c z6*xMhU$$IC5s44gfVesxF)6+NAU^C~>og3zvUJojG3c^ZgsR6jl%mWm+rTPwI@r5-o*A2NkG0GVALwBD+gtHe^V9Wmz?d+Nk9; zFPx8+ZsP@V6y6lAxd_9`9_y$Ph*^=~NHY?%7x9JJ*a*mww-oEioFTzbRJY(w3~Uqx z1wV#52QCs!dVjjuV9P4=_F9|5SX)T@acqDBysw8RVIDZcLpM=H46BuJ2Sbl6n+QTC z(RP=JRh2R`9O_l?^nyfrQjR2$oKc`A=5-%QZQ*OuSi*(q4HH;i3ENrOGE|eptgz-S zw{E-%m9%vvt=i^k5$H$U=`>9>j0hD(!3YL{o??U5aaK0Gq+pFMQbKI@DOd_ck>r_} z_fkbCX$(IjUApM4g<*bbWrgu8TUc<~G8@v$DI5+y(+NvSQ5~_yw^kgE5JE#XjM5cv zB+jt!gf-OlOuiG_IEtFiiycr}^P^ytMmpGAPW1$&%%~MZ$-e~GoDuCVu+3T|#*rw- zL9<&HUNS}Q7n#f6+7<#6fp|M7>_xo~4T1 zAhm5*&ZInN2D;hx?SA(p0Y9_i4FlJQ< z&JJzD8@t-{b|`%K3kn}9-o*9v0f!2CDHy5a?qXwPI`=q1Y#r)wnHZ%O_fvW#T|DVk z7O)7HQt)epCimj*@%{Z+XXForp}vPn(d>{acz`}xWy`h09Ce%wYAtbUPg*|KjBPB; zA})JZle7*T?j9S&sAD}PvAWk`!8hn6i(j#J|8)V7J*~r}xYV1yG|K_|4p~5AqFReB z{dBy(O@6iOxQCmD1^zZ9v;qx;y;m3VoTuiX$@At?o;RI5s(JTY0cn3rknXnvaxqW$ zTgp?um2GU?U3F-X)-bcWc*E9yt__HrurKCv?XCu2QGn}Ea;FE04y}q(3!r|n0P61+ zK<-bvku_q^0Ke$ zS5y8^;h%tHe(y@u6q9uX7e!d$7Dzkfnf5xy8o5;%U-$(#GoPp~H`Mc?oSvz5&3Vy@ zKAAXGAEV=Suejyp9u68wfi7B(Php)7gcwF7*tOVYZMg@T4lE~FhLDknc@I!XaOwew zXk0e#v_>#iS}VqT@p-{Zh?5hz$1=9_75b6#{&Kj5{>U9A4ijUiG<=Fq3@PqX+WL{1 z@%Get!y$wgtx(2d%XCRAg#?GBnI|H~hH9U_-`nLAv#L(x2VOTpYP|FnD8fHqlGbsk zz2Y!0P<3dakY?>xukOk^s@+AGp}fo#xhzL^?Nhp$;luu_1O>j;fJ;!|+YOkYcq=$s z2zoesi;*?{g5wbO^+u?h?2f1CD3l|7Ivipnbd!HLl+SzD=Y|>esnP9Y`9|1O71JnZ}PusZz{{f#?Y+ENrKscl*9{c zArvD3k4-~$N(H(=%dwpP47p)c5q_05Ps``9DWu=6h|rOtAE*9l_7?BG#2S(xU=VdGjm*ww zo_CxDD~-awF@$*m8X7!7lqnmLt}dt@7VH?3(EMzGB*ZQ_d#j$E@I0_zLKI@V#j{CR z(UTPEx~e4In9SP+2S{pfw+4o>9Y+d<*)YG6Eo>l7WmHJ;cNdEb;ScqNZUuvz^=Ufo z@3|wI^m7$pSDfguCgqJ%T8%6{SQ?F52C~x1U{oI_2%l?UWfNss_vm4gI?)yq&U|N0 zY6T8=1Eb~WZm=A*0FF-qjTTL2@l5=5PX;6FvGGx6ye*d)7o#^bXe~JVmKZ_@g+yFv z5j*n>D6s6l;642gdA&1Oxz1z#J08zFmnPaZrVSy}h693fko_3IW1|PVM|VnJrQIkCr|AjWeS*+fZs&YYq_|d~U6==GwPrxts#zGVmo- zSK>*2pn3?!aW!7RUyMhJRxd&=`o#CUKYH=$tCZWHtd{rfc~L#LvY`TS&!c+xg~Y4s z?)c3sglmc*rTY2!{4Ue$Sb^#eb_#z-h0&+uN`Z>f?>~Qdz_Cp*51Z;0Xv!Y$9>gY3*BKjKDaD-ZY^U_rC zfSsyU1YdGc8;)5n;1#`BmtRA&VzM|?_rzg5>n~6VjU2W=R^8NS`R&oeusl7ZkJL}q zy}Rx^zi}6PaOM8KpN^y+gJ-$Y2A@E}UO{g4Vgcr7V=X?FGJwPv5?9@lXMK7ci{hgf z-UWJK&bt7s*AG#v$FFPg3~T-aorQW=eCJ(mTo&Xmq^$acQ(dZiti30?Y>97dl^k>A z0q!tPSHML+d&114$D8*rVF2kfqrB@X2_YZs?!o7Qsye!q+>cBf3bk>OL z#L9`99#{scp1r`EEXt4HPhZEAeyN^4loQOeSAN9rv)9rMJbQqeqk4AVYOsJP|6Slw z8yY6xw)m7b3k%afdnlYz7H;+I$)XiGkXHY&NsO{T8~fyNfAs)qoNprXY^q0K0uRzN z`EWlhER?HN$D;Nx#}bIDXITrks|iP*jCUf@#7uc=Z`NnY{zkK6R%tjpaK=FTsSfNZ zBpTac2NqMaUE*CGc;3mQ)^z}P*}F4 zLUoc2xzF*z(?z`s4PzKjyO0NX(96JCLp4e>CQqlpPhayf>AySB`<$f-6V%OmU{zr! z-#tdL1NaOvumiXwn>6;r4)8HNZU=C%oN&J6sN0M8w7r+J&6Y4$Nm^xqb4iI=NrKvB=Hlfc?nh`FA8$CFc{2AyS=o@;*3Q&A>AqSXpZw zpr;OgI9CGe-elm9yM;9ftn(&=0yVv&T?f)rLRY)Fbqn5fasdbzecH+Pe0Z@>4XOMa zWBeIs6m;GR6!rR%QD1u*mV{^1vnyXmcgBk}&2gPw8`TuADa`RVIR z{io^dH#-^Yk6$supB3J?^Tua*Ep`A}gs+fVHrVDn37?3Cm|Yq#Rpem&tIKsU#O$9# zFRgN@4yf5(q;3aH19RP8QRDmQ+iuQOwkpt22s)@lvv!nA>stn zQaxRcf*J*xLoj)H`b@(2P`SXF0aP{(& zM^zyd(G8pG#Vc3pf%c}C$TRq-0*9Tb4(_UVVfD&Kdn|D-B7(^pohNrq=)js{avyLa ziHzY!IPvQ1VxakKxyCW@&dKy1IBRG${qW(l)%dW9-_g}>PBH%*tn5?id>TH3R!LLS z%FuAc!JxVTI;Uo7m7(s|c0!QK^E&M=PUn6*zT65HQvC`kR_(Pcram6(v7fMHiP?B$rr>bHrXJ z@I^23^d8AK4mqd9>u&*qH_W>)#=RQ;DOX@j$HDm3K=8>S%}rTl$WD*q z=$I<75LoAVvx6#zNb+(NN8M4@dhx5&M~|iAtCvyCThF)liu2V$@qWr&53Iy=nIIjh zRs%t2oh?nA&NYM9`C-Y`DC06#&!sHY9cw{ffLr%l#of9e!ilq3s&VM9G>Z;B*Wn9y zpFY93k75YFjxH9xsv>%zeOoo`g|z+|d!>teKYdd9=?X+p zbl6#{fs-H~+9y{H)ZTk21HsbOK$U%lcDD&Vf#2-Lde0$nC8LrdVHe-%f&RESM~s}; zS@jt=@qW@TH!fCs9r9EUO^Lh0?$iR&XkUiH$Tk<@W!7rIuxdlyotIu}6-$SSjDg9$ zr9l_=)6?Q2PvHxZX?I{<8OH+lN*e9m6pDflK0x>AdEIJ8q3DE2lTs`mB{vYbyelyY zIa4jAdg%N0X9ue7OHGtWhza8|fjshEUwXmalWnYD&bV`*&L8Q=qPgYs`0Ds1mTyM)$0wN3TWGq09zFS;hsyk_SvVr$>E)lF5n!`a6 z$W7G()bNvDI4C*wR5b|RHQ#1d6iu0K#Hc4DDz^_oEHJ5=(uovpay5AJb0d5LTrI{O za>3pP3+*)@fV$iktYoe~Ae_q}z${$>ss>H+E4$fIBVTRZz^lQ}QRkwRm=4=O>1Gz= zCqHk)N>{aEOxxk{D=rKx)u;w9byWl8>zJw2K%dG|aS3{WH}J)r2mSf@HDu`Cic(TpNXz(L-_EqYuN_gM~Qj~_>c**JPpFoRGjs#TVXD=vB(jBC(U`gZ_!2Y~T$kXlt!TUie);Hb z)~59}bppJ!TKf?GEg3D@ZcLqwtP=$zGJbmY)a6*9Wi0vF)JM71l5+ZVaS$yg<%e0X7B4veMKM5#K|d?RK($@*jTg>9C4o& z3a*?6_(XQ6-*;0VtVwaa?k$*STMb?M`0U{L-06Uu-X?PIXw1}Imo$hZoKn8?V8K=Q z%Y_W$1%z1P^Oj5e6YX0mk=ILdFQ+B?ERVV`agnIG418ZhUdTh+`80;> zu(~w$i@|JH40T`Wt7ukgqu}nE*FJqMZm)^gvjusnfohx{9JBz@ygScd2W+>v%l)S@ ztM0AOR~UuluHZzBbRHIq?R+eBIVDzEA%PMdPQt;IrfVVH>ICPmwyEFnql9q|AZNUEPqj|!}QJsE@o zzrTY62-QfP#f2Tbcx(r4yt0R5L?sW=l6xRd$+NRW>Xqo>Cub( z!{B>Ywo#)jyuWdREhVQ{Newg2pedq6&nIjayc~t4#PO<5T#1c2+}_b%bWK{zA?Uvyh>JA%{I6GNn++TsGi?s}jP3dR59RIrm+e zfHLJOxF_FL3D1$?@hiG2$@aag63m@nm5@VzRZ^G+uS$_V_A0oIDws*GN_*}zq^?Q{ z4bxRAg{!?P&4g?U6WHqESu7S~Mi z3f=)obnF0LOgWUW19^#6uvG&L?K)?n3&L%u`;ur&C(BEpkCghT&x18MoqxX03Gaq} zj$PF!C)Er8bpDx-ND6m}H8Od`34jLR4yB#`lrxfo_dA_G+Z^77wxdQYnzD^ec~;Dy z&L8+VAB2275w!vHbpGVo1Ap5Ef}5?gG5Q>@#rPHYeSE^5RsfuM07KELhZR+guFVYO z!?GBwhk?Ka=sl`8x;y%Cx3F!Z1-|#J2eC()zY6N-d<9ZDc274tZz3G>ScJ{kCp@?T zXUDCC4g4cqknosVC!UT7QobBEB!;GM4=SSdi*pG|J+P^+KYbV!1@&Yw+yqRBtDhf` z1)~YWArcSMEta$RFjY@RcShS+s6|D{O-x?u02X0)1s*N?vpe>&DiFt*yJ0`BGlq2W zG%Qk0UT>^A@PQHq_|T3NiZLhnrs+&r_k*(6Hj-y|K6!NqO_5K(XJF0)Qjt6K6v>!hV-qytmeqpy}-%i;z~_L32!CFm#xD*U<}RiejFtjTg@c1ysA{ zNU^O&(FNGKss@1)SPF^E{DRVJ%uz#((F?eDN8XDpMjeDwXSpM9_3zGN@~(Yj-<_@d zF^NCYS;{RVXW+Md^Sg3h{`7a{{`MN=jQjslgNr%8y9O6?cWWR7ZnvA>?fc)><5%ao z{MD=Tb-x$L`&$}$A@TATE@bEyeNVPNbOYK9^DQ5^GRx)9T$!!=u~#XDeanX~C13u? zr7YbKpfZiCeZJ)jm(wqQ;&Prxxv8UGCa4sp&pd2;>{1q-UpbsED1BX-OSulFi=7k- z(8Qs=+rOhk)V4b3<-%xQD)I%U&FR9qZnh@pcjd#x`R!G_n>&rbFVb70kNEbyucj-! zZ-lnLBa^%@4|U*J*~I+q<;hx3H!H*DJ4%G%9m^A9`yF|1-4AB|JF`Hb;!d#d$d&R! zIT*06{N;&suMg{D7Z@C#@sJ%%hk;hiCEXbe?{H!SEZAdQcv*dqL>y#sHh~GFw)s&> zuMssP>13vT@j+=`u|eJj^B}s#6ju0_h8Ly`S>j+(webbd@S00*Li+a0+#y4MV4#5= z8VF02U~o8&%Q>|bbr3y?IcqNh-kqS-3*J$xhFBU3!D@R~fWlP*w78rXWyo$+h5o*7 zM@03_RE-8877}KCYx~2OBqQtvc>3Uzy=t#|i+|Vh-@BFmRn>dd`}|!a`~#lvRU`hY z>VxXn)eZmqn%M8i_+Irfp|7eh-&s|EQ$4PJU;VavckigWUi~w<->+8HpWeAqee-AU z68mFybGb@k0#AMB~TUsr#;#vggkDCg_yy5IO0TKmuZeO3J<|GpxZ zzWhP;FV#1DUtd%1-&3bQRbRdPC;IpU>Uw>T{-!so>T~{GC;#>8_r$8|Ka=|dYWvUC zKkvOqAN_`w(!;?1iBz>X{`*1o(cW))_X9@gBj@^>wtw)>$M1bi9aQf>y!$z6U-QmC zQU2$yo9d_0d_g&1Q_u$rzpYX~sJ?pp=i0alJYt|JQq8 zR_|YXpPso+S?^W9efx}hT>tp<>ixHm`0h8f>FYgO{ahvcj^z^YPFsD`WE9%kNnHtzwEIa zlo9-jJnEG%`Qpcv{67B(6MN5Zf64E!>6vS44<-B~?|r1+;_ctkqW7uK3;%nU-_L<6 z_CCFEL3|e}i^cl=UZmoBy`929q8xu>0{Q`d;C7$dzZgz?@v5GC-l4O{k@|-%~#E* z-|^-j`1JKx&1!%{bBWw z)#q>D*wbkKkyc*&!f=19OJta!{y!QWG5+$SH? z9{s18_Ah+#-Mz2r{wfY9s&NIe7K*-v0~i2i05r>A6?^tLkn3D*pDW zzX1N1{QU*6zhJKYYr=0;@9_6M-hZq5KH={Z{wvZ;`1^$Z3-XGxT&wzT8Ie!&@qJckh$!}Hv4UqqqzyFTE56Ji5^Y`z0<9qy7 z{Hfgkz&HPf{L1@3P}28!`+o%XKe-I`^^dB*C3TVM_$Kk;{iznlEs;_o)~|B!Nj%sW5f-JkN_pAr6uTK$ao|2O{rcVPbqHT*xx|9?^5 z|IOdW)K-0ZhrheT@9}q^@B>=#kmr9Qt~x)W1>ZBheye%{?C<#dA%A~O+RtgxzvWLY z+oM%aY1K1&_7nb|lc(f=LHO_ad&%Dce}B#2E5<0s?ppOJ;n)0q#^2|B?-%sS_uYoS ztp0(rz99eqSJ_#Bx3O!DRx@_&#&(>fkTR!j%FK)@Gc!ZVDKj%OGcz+YGcz+Yw{Ptk zJMB5=o_qh-ek;-lwzgn2l13VfteQ~E&?!0fn;ctm(zUDFMm*6=sXBzI3sUR#pgv_n z!=yGj$w;_{z@cN}OBtn9jgip=KY4;Q!wjQtnv;GD{9EGI%8+HYkJiL(gIil_#%9z` zDx;3WiPsKysrB~MnA4E0_`Zr$b;P|BbVhC$;&g>>Tz7{awBnxB3tgt_1-%Vj(->{L z4{7y`k<| z1-Ic2clur2@4-R14-W|ckn|qGV|sxn^a6#6_Y|JNb9h0Uc!}#P!oP+$@D|>|d-wn! zLE7FY_zYj*D{k?K{|&yw5BLec@Kc&yX`q7ztRQ`Y52g+55D9$20Zwp%AGpCE0w53~ zLllS#(I5z-Lkx%su^=|Yfw&M4;zI&R2#Fvu1Va)?3dtY@l0ynesTJX#|Eu1qaIa1I z)i83dtJIo~U8yuQptK-kfb_)6pcSQuHv7Gd#K{Di2|LkKK3TA5)vSzirJSuQJJ&fN zC-HM>PL-SRc_1%t`5-?Oz^@=~g`hAL;W|_cW!zs(71IjSPQCq`T@}Z_1eAnQP#VfW zSttkPjktDI0eeNL1eKu*RE26#9cn;Ls0Fprp$^oAdQcx4KtpH*(q`p3(U`DJpeZzi zF!I?PS`elsw1U>Km3+&y_V@ivUrYG5=oYR?-&I_-)1+UPzD)8dZJz!X|Ft}csfCKo|srVF(O`VaVsmvl>nrWW;`fw7q@h2>eIFC>V|Z z7#IuVU_4v7uv8(|Y^hhlDq zEwB}~!FJdIJCWNHc46O*xrgv!j6IY8QSMTwbx7~eGM9UBFYJT;FaiGqm9Cn<+xuvYW7Cuv7^bsV=7a1u_zX*fe1sgJYBIEVebT*F1UM7dnn-0BKtaaAkF zGg0n08E?rbN5)zS)HSUFqn8S}R^S;>QC&x-ckC2~TyxAzU&ntLyWSw3n{W$m!yUK_ zvGBWxd0(r0jPx$#|yP)mz-&!F$L|yz*%N0rMk#g3s^;zQQ-ce1{+Klj~n_gfXDxMd@BSI`MSE zoASaKAv zNC~MRHKc*GkdFBByh)Fl0WyM&IWy@sRAyvl(aWi&{NeZ=85xtfQHRM}WQN1>EraGz^{>AmWs)P})o+^o1iZG># zAFBDPGP#K@-162wC%1{MDkdrZ@{QoTLhN`OGNLAAttLl0aRYPy8 zYU<5YEj>)t)|;z3dJ9!o=bI8b=S6frm8I8F4fQsvk=|A{h9=Mynn4&kl_M9;F$st4t=p4mbq z#_c@`(+l_Bgpq!#4{>`Ue;4)J7u}^D_QS2e$A18M@xgTx`RKO~QXNvZi28)w>{% zu^jG`Nh^$5j-)H)aD>^;Z+kOBdGpMig1FN?d7MF@AYzLCy{KPFHx=pqPP@A&Pd)JC z-r(N+{T%3tUk}Q@mr-^zkvq$fM-a?VW>%uxY{R_|^Ugj9?n9hjYL4Dh&DDFWdH6}2 zk@nJ@IAQbx^QmJu{})hy^Kn}Ui(oM$&#v}Rw|fnDUF{?N z{rcdDcn3(U0qIKnmiy*-(>p|(hv5huMaD5WZpgB$6UaFUr$FY!r*S`{4@H8R zud_yaJZlYiyE>1I3;HnHmiXJ%MI+vD`XclC64#fBBV}*}^D10}>u>{Z!Y#NBcZh!% z?!kTZf1r<05A~6rJdIM1bk%Jr$g->F*k6#2lyL~N zYpHvq{P7Q^-d`d2HN1hh@D6vgZl&JFd%|;XBI5&oAK??>K4U(>{9?$KGPkO)`UKj! zSq9$>e@T})mOhcb$dfLm`i*#*xlekb(*sZZsLbwzj4=O}uVD2fLZ-AAp6|%{1miNqNb*S5|zS$Z;7m zWjt(EeiokVT>BF)I;#+xA$uBQB}rqt3b4#jftHynvSpTvVsWdemf0#A;e#MLabsY{ zw9HYlEOS}!kvJ_>Yzr$p5EtS>d`MuKrxFrA5hNx~FeD+(q_CD%i)0pAwfG%B#8O8k zw=Bfns!~8o%c6*TL-G{|cPB6rGC;;W>Owu ztc;{bZU*9H#FTz1lVyp@jI1nNXN7E#7XR#!19Czx$Zc_|JeFn5Yee7WDlfX`gZzXq z00p6tC!J7MTDEwyKhbJYAK- zzchZYS-COWzfxtmE(_96mcuL$os9C4K1=#8X(OviRrFn>Dp=Nf?svHdWj$q`GV427 zRkQ@FO2n^BI#o!sD)wqno$DHyHK7)S@>DkSAFOI~E#d1xUF4(+7KB=ttcCFaTW#!XOw7LqO7$^{}DnJB%><(WMvp8IE5VD`;l_`tSY22=e~Qdmtz_B5`i zb3FrdCbVZY&(uk&SzOO1&K#JFeICq*BO23?$*LBR&O*{$^gs1rZcLaZ zMjm8cYnP`F-mR8e_NZmZk@e>=wb%2Umgl0m#=0E+x`CuG?N8dy3S_Oc?2G8fS9$zb zV}`P#X!h^YuB>Veve&{o%K>6a8?meP#NR;JaHH(K{r^GQJ@akCZo*%}jHjNzd)nD% z!f$~PRw&JTig`F;xBb_!Qnv@ycFSQ;U%}ccGIml9^6cM5Tv9Dx9H;80#7aH=sVA_|K=Tte&_pF*}i(tcmR+?J(-)fqv0bCSo_+tMqNMCh$tT#5$&T{ed@lv?~j| z@|5Xo%s23swBNyd_y8YC=M(yUhA;3HzQK3+VM(ffTDncKW7uv=MkCX6rkuIS;g%9>3a<+Sn++*(KZS#K&g_+t;S-co^9T}6g>l&!okGT$df zQ&Ft9RaD$%eIuIa0?{D`a%1vdI~Tgd!i)`ZAg)#Ne@Df$-cj+bcU1zbytBHg5?b%6 zMArK%G1tLXw@Ly@AsK`~a!3IwkuBrGR915hC^grz9+8Ip$XquqrmX#>!%Ppd29yC) z)|@h8W`fL+1+qdmkg-^H%p8ytdAZ1sw1eE3c_1(3gZxl{d=-R3*b74u2!*0h4EN$t z0{a~5q$KuI$Se(IaFeydvY6#?m$kOuw5RgeD?mluD`8e9k5!;5RD#f=E(QAJW%rf)p+LP-V&^|`&O8(p$)W!aKg5O_Rs-3 zLMP}9U7#y;gYM7+dO|Pg4Sk?5^n?B|00zP!7z{&TC=7$)Fv2SDXP&8%*hf)c@=PC% zeGH5x|Ka3o9HxxN$Ky5udl+wbY`poI$n_+cjLwp$DTdz*?y(n?&kM@O9E(jg+T~07 zRvG2KG~B0=)^w2mF^HaS24QAW*0Zd$SR?Zsk!tTY~$Mt?VAhOV93uzp} zJdEEFIEpUExIPZ8(CGwWPr@lUjsF=q3+LcGT!4#k2`*dTt1E=N3fHV})OE_w%*!Kn z!}?s^#QhfBhC9f=YyDv4`yTB-6K_fH4gUBCUdmO z5X(oNujYDYY_8)#T!;tpAp!pKY)I%MeeUn~B4NfmRCy;LYxAvNC~NY%x7n6pUx5r>f)gOV#yJnjbRj;g^&2 z;_@tz7XtN-eA-n5^l6A+BWMgw;0Vvsraqch-iY@}HKPu~2q*J$^ZsOh#&rv539X>D zAy4OiVJ#fJ+Ts?DTRUhEvR={ww~o*WIzt!SD?$b63X;ceq~9IC9w2LBJ-O}$y?vsn zK0eH>3D=Knd1uw1bOyjc7)1ENm_zU%3d3MHjKFOqj3Ulx!i>RfEcVLWW8-ii4->ea z2$NtkOd-rvm==T`+Mc5x;F2-EqQ%o)O z(OJotm0n$zeAP1Cm%|EJi7d&>D$G#LqS<(JPX!Te!LB&yODcs6W5z@mwRFh=2r5s%}C3tF{9V~*%h!I znevRcQ!EVC#0*I z6WONRL%1J?BR&r8C_3#y#xbafPREgT0#3pyAK8NtSQEwBWtXKBgeewWM9cypD5_T zXc@Pwa1CUxCH=XqH(W;tdCznMJ#P9$)#Sa!9(0uSZ{c^F@OP+-yKoO#_kE%f)4cCR z&R^g86;lsM1#FuxkGEaDf{jnjJXAkzL*q>oO$9#eL5?*0{jroQ&-@-e1 z4WP zKuc&v_}0j218pIM-96#91e)xX6nX9NZx0;^+Yvf}oClCL)dhQ3{JKGR=)rYQ=mou@ z5A=n8&>sd6W*`j0J{X2zABs5)hQkOL38P>%jDfK*4#vX-!cWAUggF_ez*Jj8Z5m8R zry2Oogju#kJW=Sw4LuW67UsBO4%bF|w`OBEoOg^QeeOp!*OpDqv$@rLTd=mkChumG zXbX`q&;Lcl3q_*DmAw#`G#Tq8)fUsPmk@TTEv~lAM%Sc$QOl9B0#@1*Ypd{E4QpU6 ztb_HoWZDMgZAA7a!fmFGx7cL9D*FrLXj=)h&2~d=haEQWv%P@YX$#SI*<}9ZR=aJF z)E--MZLckc+DG{P)a?Q4>>&9*WJ{r?A@7GNha+$lj=^y_0Vm-UoQ5;Dis~%k&)HIH z=WVI93%1nSMOzx}5`LHA3S5P2a2;+yfBLAKn780I$ey)3hKLjV%cwrH|!357+PDJ$xYEN5V9K%wM4FtZRS{7O;X3*uV~vz!x0g1Q+;$+sRSP&cHKwO9i@fqqTu&3i$Tq?G|3GeRc#gl@9`B{T7}Kvu{G*&zqyByKLqP1rnkX}_#% z;GPfi<5mC)LLs~KCuz09DmOY1~j8Gm<1p3FJB zP?lY>S3zzVJ1N(*gVOo0G9;eaFNU$(vXfoPB99jRU&D*PcTbL4Ufqb_9eQ}u?}@pG zHrWfax1ocyt-M+v>^sO?Uwb~yhh3cgxEBBZFaQSH3uto6U=a4fFa(AocNpey7=gU4 zyj>WHISNL@7#K^Karn#LcbWT3IgiKf0Qb)X+}|-5YRmORn8fvDdqFMgzvfTwXP!&w zGSyy4n`U>b=`h1ySet22qs@ZZ_9EIGWX*+nFdr7!L$w6|Dl@%WXqP>lMJa<~+9Km# z<9$j*Ir7ZKz66${pUmHvVJ?S)+>T<6UB(Q>)lK|wq4#Y}*=KqOo$i91Hn@jde!||jm(v~?^;{lF<=l`-{#&2M^O>+O;HACNe`%Xi zj=%HzioCvtp7u=Yja|kwm1(om*33Re?(4VszXLfxA^S6=tlx9}fp8xwhfky_`}3v$ zNr3xeqit3pl`51&70Q9gPo%1v4eSt!FusI$fD>Hc2X63(00@N09Px_B&Tct%6a{-!hz3Cr9b!OC zP~3yD2p=1{aU#{wZikOi_rHpmV+ASdL4+>i&|@*+DQ>E=h@0@w>eAt($* zAO~ZyQ2dKxF9yY-1eAnQP#VfWSttkPp#oHdN>CZ9Kvk%Qoa&KkYj??;gD_|gEubZ|g4U60YHcvvLO8U8_Rs-3LMP}9 zU7#y;gYM7+dO|Pg4Sk?5^n?B|00zP!7z{&TC=7$)Fak!xC>RZ6U@VM-@h|}X2L9(4Rc^F%!B!`02aa`SPV;GDJ+BKumV=XDp(C`U@feJ^{@dp!Y0@Z zTVN|}gYB>bcET>$4SQfO?1TMq01m<-I1ESNC>(?1Z~{)kDL4&h;4GYj^Kbz!!X>y2 zSKumKgX?euZo)0N4R_!!+=Kh@03O04cnnYADLjMc@B&`KD|k(NdxQBF-obnL03YEK ze1F|5F6q^T!;tpApsgLUzu~&N1yt*)i>Nq5l-|`)&`8_ z8{po+a4(HJj|)}C*R9I>HY83%{2Cg5UmSuI_8+Ju7>rFqPg+ll7UJ*lR&;r~`GOp08PcNjSB` zQ%Kb(uCg{a#^(*NOJ0(2l1JW6G$gHYbbO64Ct%8)%sWJM&KLtdf+Z+b9-X;!oN3W zALtAHpg#;C+(7&W!C)8yLtz+sAC5T!Mq(cYqhSo!W8opiHjemZnU9V~#srwi^(5+V zGE9M~Fb$@|444VC2s4|ob1>yx;#_3T<9a?UfQ7IKzs0^ywI%p1#V+Hk%!~n+;kO)C zz)DyJt6>fLtc7)iUr+cAxNU?@uoxDOBDAv_}NW01AICzwy+8F8P(3;bVl{mQqQ_S(0(_Qtn`_Lg?`&NqxP zcWdr>8FRbUd&>R;e1t@HS+_UmNS}!RnfPDeD}00R@B@CrFJQW*7?`n+&_>f8X*7$2 z^XrbbnvX-)vBJ5p!ns$&xmU3}cthbZWp~uuG2Uy3UpvDuk|SL6#jg)*7joZ(u?lDT zi*6mLY0kKlRv0@pf`ArI+)kR)(V6)ceKCH14)(-?KLkJ^M20BHj*5S%CvVY24~ULm z40Mj^aI0934wOq5((QumE=Kcb%5^u=qR%EyT#$P*9%g(<015F+gqawE z9r9kXyOzY!gFZ)2$@MVg$zJoMgbQO9=VQ#|dK&J@(1p!8oLTHe`n|Of!XyV}?Zce6 z5ApjDj<-QZIfZK}FjG2OYpEQOIZJM?C#6P48Zg&Rcpeb{J9Q{y^K{rlIdSK7^wZLl z*9?$|IYUOwOpadUyT368&5XP(=${p`IodEcYD3y>NL%JYkyUp5b3~-U8BFZCAU8-l z0hodCjdb&1=7oHY0i^3z`Jn(5ghEglB>f_efm$eSr>J9)Rt!ChQy(Q9{n2SKb05Z@ z$S4J+p$z^ZwB53fY^t1Nh*qAkrI<5SK;Mcqj!O7fhR^n)l;cozABygBet8(z^!bFV z3e_C4gW8>`)7AoP;G zx(zTJLL>YdLwn0;-ldF2htcR@exINTa+*Rj$1p97F!daxwAj|+T65BFL7FWc1=L+m zGUfzElf-kXR^(B>bzr_fG}pQ1Ea)Ec-5ObKB4o0TfxR2^{cy~7=-nPVKu7fI1f6m3 z0$m-WwQdf1$8Wx;>+X=boUFOX8Ef|25T~bOjMfW!Lm%p?uVbv%4_*4>SCIAF0hj|} z5N?B!IRu8nFx-Y?%9-k*C#+AN2! zn(dfKU9%?^y*^V;j!E=Qb~PWj1+Wkn5oa+haZEOz`AczI2Fqautiht^^R#qm<_mZbWAtqg_|7Z)MmnLfvvc2gY6*m zsV$oj+jXq%{1DQ?ERLo(k6BxYd2-H$B|##>zGA5 zpQ)|YM{4_sx8E@v8FF1r9dOL~!#>v&&U?0Z9&#nz{0R4hM&1?}>B^qkL&%f8qK7e$ zfSi>+ig}FmqjLJ%#wpq3j)mF@I0>iVG@NlP!fz3}FCvZKYj9_&hl-r2jRZW^|AU`- zwsQ+7Zc_uZpMQjr(cK>A-{;WdJX`=9?f)V=wW4ffpVuYaF2fbL3fJH|$e!`GuU*~7eh1y|!acYT58xp@g2(VgAnm7^4>6w+_c^>EZVu~G?WIHZjV)tsb{Tpv z`wx2m`?--<$PnFpz-Gx#+cL}SHEBs7`^F(_Qp>3)^BW#-9da^ZEbG_uJeM=Ja=zB7 z-Z>Je_r^VL-mf2!EBEY2_#|miZn9SP*~qt(-M&ush5J$7Wqoz5K)&n@Sb^@rswk(K zEx>G(>qly@?Gbx`%<;oF?$21p^PYB3`}pqBQ{Lmq+SCvHe>&t{))4KNW0jVY6Vc3A zkt6-VDlHdbNSe0HI~*tPaKHjqr`g8YX2AM9Wr3R=B7rZ+dn$RK?!dm6dUj&Ezz^Ku z4*?Jeks%61g=i22(IE!JgwK?veD@+2X~l*(&Q)4mi052QA0+vdZ=twVd}lH(fm5D2 zZj})KM9wu@V!}TsM~Z$;#stY2kIMNTc^?o=9`@pv#3?)Xcuz{d5Tx#t-L+a$@#8uf zW(YbY$4mi1oDNUvT(6~q)TEWhd5>Y1tRM1T$B7B)Ap_}#AUh**GC^j@;@rT!7FA`% zp3S*Y%MLjpr!%U`<=mv@HBOpu@|3;o8vX8b5 zW?3i)<)H#pgi25u{{QN86Lq(Vx^t>3&dpj??A0ij^*mYSc~YG^tbtoi@+*C4C}mj7 zqr05_txbA$$X{K`upZP$P6KF2JtQIzjhtI3KXKnmpDrfv;gQt@nqqGTVc44+@_7%> zbxUXkt(|hJe;ci38#3Ig4f3VUv?cxMd@&;2d52{P^S)|_4%MMOtY9rz!gU~gN5aax zm`H@v+0R+s`ABubtuuaIpsPn_H|KWRoOe#WgLeFHYr!(Mll|4XwC?EG!_fQpdxp+D zk9s1%pthYd+)27S(SIlUZ_|1?Weg#EuSGxedGy717k$<)!t5f9xsOo%dOLSBp4*N4 zZro)LxA$4V=oa~XLH4vt8|jB0{SDdlS=a|U_plDThq!x)8>|Kq)YXA*1xp$v%eD&Ks1y_?zu5jIVpx z{&Fp0rTiuQEsYb=>Mru{S-fX`ZllX>w7N~7@VhTLLmcy)KAbh6e9jWTC|?T+0#=xf zux5T3H{yTZ=x3zunrCtM=;k=`0&*_GCAbW7j!NEXUBP~pd|ZR;^eG|MyYx+TlK7kV zgxR;7P8PkX>Ix{XrhbD}Z<*a&a>~SD2{_)T=KJmmr-bOq z*%2)f`AH1HgiC^%6p}#*B!?8P7f7=jd*bD+{v66@uJfhGjdmB&o?aQx@kpl0D`TZ!__x+l@q(#qk5Xx6< zWG*ak%4JR`^SkuO&S2!@BYq#WjIIw_Cddp~@XHF>T%YK#WKPZAe6DjqPRQl@Y{(No zkvjmSJ;X)-Fusr@ZvWX&zI4OBe$vh3`o`Sp_j*!s)&=uY=J}v5?$>FT`AJL8@E5== zi2Hb2Kq2BDcCeqHGB4u#&fJyvnfMn43-v7fe2Zc4NqqT^Kykt!`GSu;OLG|e=w)rg%GnlUJFP11`k~5@p7*>CZ(yEauYt3oy6RfifbiyjF%HL=%%+E54ox=;`5Lj#w5 zNyl5JY*0tf=*p&(jB3cAt(Xm{f=0xZa%@bTzJzar*%YKt8$cMw?#OYfX0**PUW&`w zo9yG4enHAa-sOud+5g`hxh;s>5?Xw)o3;Nx~_ssC=o2 z;#2L=sXcTcUMOEKl6V~nC*Q^BgxMK)c?SPphjX!onQsCSrz`&5pgZ({o-RN1XBx&c zg1u4L+3P}E*yxb29+gnNTqRX+SAg!rSCHm2p6)}OzR(Z)y8=mD++`mQ=fcr(Ant=O zWxZ#xp+{uOR>DMykTC@Rp@frlkzuZ===OIR5?}HVO}BW`m6356UsCd34>x2-kB~VW znIpi$*lDCIMnt$#xD9v3j0iWH>oG7E#<^legp;!-VSIVXYzJ{DXR}WmPq+z2T5+jc z(UY+%*E0W^L>@BJ!aFgxk@55@o&n}}_v9OUd`FtNQzWbw&m%X!VNP`=pq(+cCH{1n zLAaTivtTyN;d(C2gZZ!k7P=A|^)wZEi(HA&o7ErebumNu0u-+@l=v+H8B0vWLB_6= zY{BH+Jj*HW@-@n(gjwcFf(*{SAzQu`u)>wpsQVX&J(;2B3Rj3>u5=|g{A4VXlRM5V zqm;NYHbmdmu2iJK_z?SAWUoW!ddx7sG$n0o1NOD-{E_~`?8D6db)zdS<-i#e#;)=n zQO;X!LdIs;0$W{a^lh*mc5uBDcDahNhGNc5YGCdrtvx(X_QF2o?I+9uI0%Q}Fm3(_ z>4)$ILy0GI$D_C(gX3@lPQodWzC}hM<~L~Qx#)jSyNVinT(=YVjFc7O&S9R%yZ{%m zU&6eMc?I*TtC)5T^E%vsn{W%i+eUf(WIXD9wlb>1?JoXu-t4f0^AYqR_g%k~lqFwpo~lR4cnnYADLjMciYY7j z_?1;QKbuz17&oNT?YM1o6;hG>3MpUQ9exWnr=Ps%3RNz@47wlj-F}QsAOLyEc)z}u z@_3Jqy^OMweYm`nBE86jiGmpwqCqlRR1j{_AqL195jijN7&9jRu^=|?aWLbOPCSSY zy%{$qz&#=MM35MQAqgafWDo+$AqAv_f|PSA;-rQ&kQUNGdXRRKfiyB=&x9#wbuwem zf;}r{Hpq@W2jqlYkQ?&go)_{#ekg!jLCiu>7<&=SP|TuG42p9tXJSfVFA1f%enp*> z#w-J6p&XQl3V!MJihdc5`%=n9#zlc}mwXJ>Gf{GxxHU6zFPr6dNsmM+n`$YTLZtEPz&nQUS)jF*GP?i z}IugKTyWSr8PblX5%zud&+ z{SWqb(4H_Ipd)lL!sp>Rmxt8ykO!F;ncq`2=R>AGo&Dr1czFqv*Aqt0f|>oCtYTJF z`Sd7!LGP59%T9Xa%Fq$lTkB~LcmQFpFIj~;)dFK3~88uh_F!961B_V&xK z%l8#CQ=feZ+Zy^pKj;qwU?2>F!7v1d!Z2jZ`p0nOG-qBBq6+BaBF6Xy^%2M&=_hA{ zA8|(53|o=(WDQ5=@1uw(Z&*fS7Ng%DgE5VddfSFbFdcsI^?cL*CdqVpYJ<15OyPMg3Yi6w!${p4m)5cNWJXB+zoqRFT|xj z_91&ea;I}g9l(AN_n(Z14`CjLBXAVr5cU|RoVWL$y+6+N3F4e2zU=cn#r0`8183nJ zoQDf=5iY@HxB^$<8eE4Pa1(C9ZRFj7yY$8P;682-;30kTBR~1x0p~U`pZEouX#USYl_ek=5TgZUQTA@{wXQ+@DT&KaWR-1WZl*^leB;N6=qXuFyFI~wLa-P%iy2X5R#BF^C$SvOuS+1hHXQ;yTX!5R9&hp2=FD6L8Qj%+F6S26K z?`_7$j015Y9>j+PkPs4qd1ZzP#B6pC=?}J zG18N>y2ZIJ0VSanZteMEdm!afn(H!97Ro_+r~nnAlA#;tQjk-HIAOdGn@IXqu@|De zsu8BTyQtBRd*``!(%MG-_Tnp{7QPZ%0~s}q^old=@Qx)Vlsum(Q{>2Z_GHXb3wgDn z4%8)1s8&p0&sRoe?W3N%Bq2-ESC=%NN%aXM@01&$n~ae|cvEEqcJJcvR+VV)^&|XB z6PGz6aT=jdV`u{DwK95B+R1%a8J;y|h+BrZ#d%hi(3|0(jlLiZ8O;qje9MH9U<>qW ziJ!)Eq80MIa0~)e6A+x7P zRxflh%Zj-M^(Xns?UMV|u6iRYCT`K758?VkKivAGi|lV0K-hsW2=~D-1V8!i#!&Ic z9;#J9&q_SkoNBmxy*2_yy36RJ#1CCZ6L$=D+2=NvIOC8}7dl`cZ^SREPjK6`iSEk! zBzF~KypWtVwB7U_8*E$;g`mQ(+o%r$euQ=*^tXE#JkF zZ}jSnVR)92ej(c4EQv$@XA@@*I%Kd{B2QIWx2Q^aR;4`6@zU?_cCt5)wB;SnYQ387 zPZ;@T++5_%gZaqI&3Iedw0v)F0eUTjMX(sZ**v3{VD4wH!cup2Bc0#zy>aCmsuIpz zuMFW!xX%BQ&NAdRhb5#Rri!WM?wXAAyn31GN#85;_v5_pki2--b_ieVZ?e5Q@H|G| zN|1BqB0F3Yooh1(^veEEdHypF&vLiKsiOxNab)dTzWHgEkF0;Jrd-y*T3Bc3Z$2+% zeatLV=B7s8>e7dJ(-j>hZPAOqmo^gu8?f*7jNxQXxRLVPQ zU$EI-pK+t4@n=|Xd6;Do##e^}{|F=Bul#%6oAqF}QTYa5SMn*}6*9jm#N5du-^*+G zw{rBRBkjhV)0%C?eD6=SQCE%rsMmj%y}Xl_?>Zef^3(Wl_4?O5*Fm5u-&?Ou&{Fcw zDgo(7!vCO>a$ufK9_PE8{;jQh_4SsE`7GN)9mpD_w6(3+x50L}s5kS(|Fdj=>m>Jt zv{8BQDEEzwlOJ)PN2NV4b%&}Q?lA6^Fz&Rl|8P%Af8)I;n;Xxh4wO6J@6~s@Tj;xB zx4R|z3)AIr|G@aFf}8J808B-hfP4#5Af@!NaK!;CLwFJ&(4+@}m(I(W+C@ACdE zkB;=&{7wn!oPo1omQ@<KcAak|IV8d>QNZ<=B=~AAoR%6PZr%=8e?(!c%4Glon0A%&k{gCbU zA4uOg5VwK2$@u}s$o`TJ`66y0_Q)PtQTzw#Q6U<71^Ev~)?nfdj?k;5Pj~9QJ8s=Q zZroSoCx$=oiVPXDr@k^yTk(#=ZZvw0(uH1uZ{3jA-B1JOMDF5ihNlTb?AnioTrz&4wk3>DE$1MZpozZ`? zo{6-o;`fyP%FO>1^rat-(07;~mGb6P7jk7!%OrjULEh^oQ>FNVdltgUce$!cJLB0Y z?XESIo0T-=`JW9lF=jf-TE=MENiT>0R6Q17erF{cnQoQSU*0dfRW9m1gnLNlf>M{c z3H$f=&3}E*%>0J3$V(dL`DLfd=g~VcHO-Tjlqc%~=opy-Dc~tP9MceYe ztDTMP*~p%aY;XOs=I=9yJkQoE`kQN5m55gvSyiBob1wBOe*AVC?p6Kg(I&iM&HZWB zaF=RwT$O^j9zY)k9Y-3E>t10O; z0hte;V|`TKD>UW08H7P|XaOyu6|{yn&=$hcp`HICbdt5iQT&$JNY&ndF>7Lr^$xU~ zj{fW$gC~BERA>K0sta_5ZqOZiKu_oey`c~Eg?^NIf8-AEH@~|#khBKDU>E{JVHo+5 z@#}ExBOnL;m3%jIB=%AGjfOEW7RJGN+$X?z%3>ndlVCE}Qz%23>rCZ(8cc^ydc*1|ei4;x5xBjGk--way_vlVk2Y=<3O?}S~j8{~KW_7HwA_I7e z1dif&431+z0Vm-UoW|`8oJGbtI1d+azlhr<%*&Wp(CI4f>3RNMfSYg&Zo?fI z!F_y}aQEOoJb;Jri1?2&pCC`xex71JgXbXYJ1>a)5?&GJHN1hh@DBg?@Bu#hhw}9T z_7&s)8NR?*_(r_%m_IOo67Cmve$rBD0kaefRmu{;niu%sW&?Y`Xr7_cPURc&@_qKg zDpCOBY{I1EzHnf7f(!fzFZ)g1*rng~#|(f#hzwDPGnuuNsMwj59DR^y9{Nb64si0qj8`trX-<`X)I~ zBKwe22FUr7<&51cYx2#v?U0HvsUZ!d4Ope83s|V7hYZNhh;Er6GkzPH^~bWV(zD>6 zHDC?>rnqt6bKQxP1#WcDPS_k$7r5udJs0H0Ee~$89+#Kve2^asKtU)(nlhFtjJ*hi zLQx|<(VaAn-&>C_~RT9~z=4%CHuP#+rLH;yq=L+mX~;Od4DX#~tT_%H#=``dh&)-VPRdwf686d9-P1oM zfN%B_cN%WfL7q=DDCe0l3ueO{m<#h@J}iKRum~2z5?Bh$kYn~?yph1Z0#*j>W{=e_ zwTis0Mwd0<-9!Aye-Gs(e$tPcI<7?yAIef1=pfI{b-2khZawCPfKas&HX&y-j3&QZ zBwfO7#eW-YhaCa=*<0o9bKGhtdD{iM!MhL2oD0avg?$aUOCKfU`@P89hm8GjAYiv~ zPaVV_Ko1whXxK+3Q-=r>M!duL9|_o{xAWUe8`?`f?WLZ~zVj$yjv@DWz#i77CCncE zM8H1M+ef&4gd>m0I7t}!{-)@&meoZ!@O19mjHV$H_~I6-Nm(A&9f3z^ z2S~oGXInN zG+89sS$&lCyigT6P*w!RE_*w~U)tRX`k_DF7)u~KD(QzAztH53%eVsfpukiA2yePe z_|wci{wmA#`%~r_@+x6uU$yAum3!96hw1Ls&+8|$&v8$9*3@lBaO=~HAkrQBg*{QA`2%kj7W6;GaNJDxaF zH~aOgp87E5#U_n7f!F>a$8?u8uK!0JO@FVB@@rS#I=lHF>igC|@BTxE*=EG=9{q--&v?*iL-!-DvvF^RsRt2d5ug0Zn#~E8%EHvc2zPOl zxlsb6y+87_3&xY4@Uk{3ep2q?+T;J>|0E*)M20?3Bd!x0*UuuZP5qunTyHk+5Bc>Y z$&Xnc!RU~Pw39$mNEY}qLT~TdhWQ@!m8ZS3XM=VWLcHW4dva4?ri4^R`mZC>J7=_` zHxcs8dSQNKl+R&3b;L91pU*XU?~@vtY0ydf*R+`FK=e%y-Zqi}dq&(d(Jy6&ERYql zL3YT2UrxwHxZGUJo)Pcw#fH$==D{s5@TAg8cqVNz#^a+BD{8De2>bNVhjx4&#^7-Z8H$ z75H8+9T?1*J(#CfFl&#DRnSAimBlOv<&jrVGuO1`o2M0U^ZY)CR+0WXjq?NLEIWNa zpw~ye5^l*EvsVuM#F+0B?w@df#M#f}S{0A%XrwJ?nW{#}k`va9fysY*Z3y2$9D+VW zjI~hnoT2pLoLLL}!uU(R!zyt<=`{!+%34WLo^>TKi~E%2+w3t^&A@M}R^Vs-AmO4| zgSFa$$+S9w$r*bjM{aVDT=rfW`p7zhY9#B3G#%d_?`XG=}pWsK4q8EvTV zwh#_h=DqW%`}xq0>-NwAjdlx7bu7{ z?njd5C>RZ)Dj{Pe^ZpxyY*~LDi#ZO)!vvT}xJk&E3{zk#`c6Zq>46rD^c!a1VUGFC z-`gEwoQIu3UCt!_ZkNT<&~CA8$CSB*-0QNBc2-~%H9N2i>!0#{QW^X4eXYP?H5Z-d z!F Activity: + agreement = await self._get_agreement() + activity = await agreement.create_activity() + logger.info(f"Activity `{activity}` created") + work_context = WorkContext(activity) + if self._on_activity_start: + await self._on_activity_start(work_context) + return activity + + async def _release_activity(self, activity: Activity) -> None: + if self._on_activity_stop: + work_context = WorkContext(activity) + await self._on_activity_stop(work_context) + + if activity.destroyed: + logger.info(f"Activity `{activity}` destroyed") + else: + logger.warning( + "ActivityPoolManager expects that activity will be terminated" + " before activity is released. Looks like you forgot calling" + " `context.terminate()` in custom `on_activity_end` callback." + ) + + event = AgreementReleased(activity.parent) + await self._event_bus.emit(event) + + @asynccontextmanager + async def _get_activity_from_pool(self): + activity = await self._pool.get() + logger.info(f"Activity `{activity}` taken from pool") + yield activity + self._pool.put_nowait(activity) + logger.info(f"Activity `{activity}` back in pool") + + async def do_work(self, work: Work) -> WorkResult: + async with self._get_activity_from_pool() as activity: + work_context = WorkContext(activity) + try: + work_result = await work(work_context) + except Exception as e: + work_result = WorkResult(exception=e) + else: + if not isinstance(work_result, WorkResult): + work_result = WorkResult(result=work_result) + return work_result diff --git a/golem/managers/base.py b/golem/managers/base.py index f0e6bb9a..ff894345 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -27,6 +27,12 @@ def deploy(self, deploy_args: Optional[commands.ArgsDict] = None): def start(self): self._script.add_command(commands.Start()) + def send_file(self, src_path: str, dst_path: str): + self._script.add_command(commands.SendFile(src_path, dst_path)) + + def download_file(self, src_path: str, dst_path: str): + self._script.add_command(commands.DownloadFile(src_path, dst_path)) + def run( self, command: Union[str, List[str]], diff --git a/golem/managers/work/plugins.py b/golem/managers/work/plugins.py index ab266d34..634f035a 100644 --- a/golem/managers/work/plugins.py +++ b/golem/managers/work/plugins.py @@ -1,8 +1,11 @@ import asyncio +import logging from functools import wraps from golem.managers.base import WORK_PLUGIN_FIELD_NAME, DoWorkCallable, Work, WorkPlugin, WorkResult +logger = logging.getLogger(__name__) + def work_plugin(plugin: WorkPlugin): def _work_plugin(work: Work): @@ -33,6 +36,10 @@ async def wrapper(work: Work) -> WorkResult: count += 1 errors.append(work_result.exception) + logger.info( + f"Got an exception {work_result.exception} on {count} attempt {tries-count} attempts left" + ) + work_result.extras["retry"] = { "tries": count, "errors": errors, diff --git a/golem/utils/logging.py b/golem/utils/logging.py index a36559e1..03b95d51 100644 --- a/golem/utils/logging.py +++ b/golem/utils/logging.py @@ -37,7 +37,7 @@ "level": "INFO", }, "golem.managers.work": { - "level": "DEBUG", + "level": "INFO", }, }, } From 5d5702f244b28b4e18ef37c3f83020646734fcdf Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 29 Jun 2023 12:49:46 +0200 Subject: [PATCH 066/123] Add AsynchronousWorkManager --- examples/managers/blender/blender.py | 6 ++- golem/managers/activity/pool.py | 8 ++-- golem/managers/work/asynchronous.py | 58 ++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 golem/managers/work/asynchronous.py diff --git a/examples/managers/blender/blender.py b/examples/managers/blender/blender.py index b6f6efcc..9a266713 100644 --- a/examples/managers/blender/blender.py +++ b/examples/managers/blender/blender.py @@ -11,8 +11,8 @@ from golem.managers.negotiation.plugins import AddChosenPaymentPlatform from golem.managers.payment.pay_all import PayAllPaymentManager from golem.managers.proposal import StackProposalManager +from golem.managers.work.asynchronous import AsynchronousWorkManager from golem.managers.work.plugins import retry -from golem.managers.work.sequential import SequentialWorkManager from golem.node import GolemNode from golem.payload import RepositoryVmPayload from golem.utils.logging import DEFAULT_LOGGING @@ -70,7 +70,9 @@ async def main(): activity_manager = ActivityPoolManager( golem, agreement_manager.get_agreement, size=3, on_activity_start=load_blend_file ) - work_manager = SequentialWorkManager(golem, activity_manager.do_work, plugins=[retry(tries=3)]) + work_manager = AsynchronousWorkManager( + golem, activity_manager.do_work, plugins=[retry(tries=3)] + ) async with golem: async with payment_manager, negotiation_manager, proposal_manager, activity_manager: diff --git a/golem/managers/activity/pool.py b/golem/managers/activity/pool.py index 78bbffc6..94d7035e 100644 --- a/golem/managers/activity/pool.py +++ b/golem/managers/activity/pool.py @@ -38,7 +38,9 @@ async def start(self): asyncio.create_task(self._prepare_activity_and_put_in_pool()) async def _prepare_activity_and_put_in_pool(self): - await self._pool.put(await self._prepare_activity()) + activity = await self._prepare_activity() + await self._pool.put(activity) + logger.info(f"Activity `{activity}` added to the pool") async def stop(self): stop_tasks = [] @@ -77,10 +79,10 @@ async def _release_activity(self, activity: Activity) -> None: @asynccontextmanager async def _get_activity_from_pool(self): activity = await self._pool.get() - logger.info(f"Activity `{activity}` taken from pool") + logger.info(f"Activity `{activity}` taken from the pool") yield activity self._pool.put_nowait(activity) - logger.info(f"Activity `{activity}` back in pool") + logger.info(f"Activity `{activity}` back in the pool") async def do_work(self, work: Work) -> WorkResult: async with self._get_activity_from_pool() as activity: diff --git a/golem/managers/work/asynchronous.py b/golem/managers/work/asynchronous.py new file mode 100644 index 00000000..88281e90 --- /dev/null +++ b/golem/managers/work/asynchronous.py @@ -0,0 +1,58 @@ +import asyncio +import logging +from typing import List + +from golem.managers.base import ( + WORK_PLUGIN_FIELD_NAME, + DoWorkCallable, + ManagerPluginsMixin, + Work, + WorkManager, + WorkPlugin, + WorkResult, +) +from golem.node import GolemNode + +logger = logging.getLogger(__name__) + + +class AsynchronousWorkManager(ManagerPluginsMixin[WorkPlugin], WorkManager): + def __init__(self, golem: GolemNode, do_work: DoWorkCallable, *args, **kwargs): + self._do_work = do_work + + super().__init__(*args, **kwargs) + + def _apply_plugins_from_manager(self, do_work: DoWorkCallable) -> DoWorkCallable: + do_work_with_plugins = do_work + + for plugin in self._plugins: + do_work_with_plugins = plugin(do_work_with_plugins) + + return do_work_with_plugins + + def _apply_plugins_from_work(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: + work_plugins = getattr(work, WORK_PLUGIN_FIELD_NAME, []) + + if not work_plugins: + return do_work + + do_work_with_plugins = do_work + + for plugin in work_plugins: + do_work_with_plugins = plugin(do_work_with_plugins) + + return do_work_with_plugins + + async def do_work(self, work: Work) -> WorkResult: + do_work_with_plugins = self._apply_plugins_from_manager(self._do_work) + do_work_with_plugins = self._apply_plugins_from_work(do_work_with_plugins, work) + + result = await do_work_with_plugins(work) + + logger.info(f"Work `{work}` completed") + + return result + + async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: + results = await asyncio.gather(*[self.do_work(work) for work in work_list]) + return results From 6b04b74d61ad4c9bd530a5231f0e7b990c6e5782 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 4 Jul 2023 10:50:26 +0200 Subject: [PATCH 067/123] Added WorkManagerPluginsMixin --- examples/managers/blender/blender.py | 7 ++--- golem/managers/activity/pool.py | 16 ++++++----- golem/managers/base.py | 33 +++++++++++++++++++++ golem/managers/work/asynchronous.py | 32 ++------------------- golem/managers/work/sequential.py | 43 ++-------------------------- 5 files changed, 51 insertions(+), 80 deletions(-) diff --git a/examples/managers/blender/blender.py b/examples/managers/blender/blender.py index 9a266713..adb35a71 100644 --- a/examples/managers/blender/blender.py +++ b/examples/managers/blender/blender.py @@ -74,10 +74,9 @@ async def main(): golem, activity_manager.do_work, plugins=[retry(tries=3)] ) - async with golem: - async with payment_manager, negotiation_manager, proposal_manager, activity_manager: - results: List[WorkResult] = await work_manager.do_work_list(work_list) - print(f"\nWORK MANAGER RESULTS:{[result.result for result in results]}\n", flush=True) + async with golem, payment_manager, negotiation_manager, proposal_manager, activity_manager: + results: List[WorkResult] = await work_manager.do_work_list(work_list) + print(f"\nWORK MANAGER RESULTS:{[result.result for result in results]}\n", flush=True) if __name__ == "__main__": diff --git a/golem/managers/activity/pool.py b/golem/managers/activity/pool.py index 94d7035e..f33de0e3 100644 --- a/golem/managers/activity/pool.py +++ b/golem/managers/activity/pool.py @@ -8,11 +8,12 @@ from golem.managers.base import ActivityManager, Work, WorkContext, WorkResult from golem.node import GolemNode from golem.resources import Activity, Agreement +from golem.utils.asyncio import create_task_with_logging logger = logging.getLogger(__name__) -class ActivityPoolManager(ActivityManager): +class ActivityPoolManager(ActivityManager): # TODO ActivityPrepeareReleaseMixin def __init__( self, golem: GolemNode, @@ -35,7 +36,7 @@ def __init__( async def start(self): for _ in range(self._pool_size): - asyncio.create_task(self._prepare_activity_and_put_in_pool()) + create_task_with_logging(self._prepare_activity_and_put_in_pool()) async def _prepare_activity_and_put_in_pool(self): activity = await self._prepare_activity() @@ -43,11 +44,12 @@ async def _prepare_activity_and_put_in_pool(self): logger.info(f"Activity `{activity}` added to the pool") async def stop(self): - stop_tasks = [] - for _ in range(self._pool_size): - stop_tasks.append(asyncio.create_task(self._release_activity(await self._pool.get()))) - for task in stop_tasks: - await task + await asyncio.gather( + *[ + create_task_with_logging(self._release_activity(await self._pool.get())) + for _ in range(self._pool_size) + ] + ) assert self._pool.empty() async def _prepare_activity(self) -> Activity: diff --git a/golem/managers/base.py b/golem/managers/base.py index ff894345..70356df5 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -141,6 +141,39 @@ def unregister_plugin(self, plugin: TPlugin): self._plugins.remove(plugin) +class WorkManagerPluginsMixin(ManagerPluginsMixin[WorkPlugin]): + def __init__(self, plugins: Optional[Sequence[TPlugin]] = None, *args, **kwargs) -> None: + self._plugins: List[TPlugin] = list(plugins) if plugins is not None else [] + + super().__init__(*args, **kwargs) + + def _apply_plugins_from_manager(self, do_work: DoWorkCallable) -> DoWorkCallable: + do_work_with_plugins = do_work + + for plugin in self._plugins: + do_work_with_plugins = plugin(do_work_with_plugins) + + return do_work_with_plugins + + def _apply_plugins_from_work(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: + work_plugins = getattr(work, WORK_PLUGIN_FIELD_NAME, []) + + if not work_plugins: + return do_work + + do_work_with_plugins = do_work + + for plugin in work_plugins: + do_work_with_plugins = plugin(do_work_with_plugins) + + return do_work_with_plugins + + async def _do_work_with_plugins(self, do_work: DoWorkCallable, work: Work) -> WorkResult: + do_work_with_plugins = self._apply_plugins_from_manager(do_work) + do_work_with_plugins = self._apply_plugins_from_work(do_work_with_plugins, work) + return await do_work_with_plugins(work) + + class NetworkManager(Manager, ABC): ... diff --git a/golem/managers/work/asynchronous.py b/golem/managers/work/asynchronous.py index 88281e90..e2b5219c 100644 --- a/golem/managers/work/asynchronous.py +++ b/golem/managers/work/asynchronous.py @@ -3,12 +3,10 @@ from typing import List from golem.managers.base import ( - WORK_PLUGIN_FIELD_NAME, DoWorkCallable, - ManagerPluginsMixin, Work, WorkManager, - WorkPlugin, + WorkManagerPluginsMixin, WorkResult, ) from golem.node import GolemNode @@ -16,38 +14,14 @@ logger = logging.getLogger(__name__) -class AsynchronousWorkManager(ManagerPluginsMixin[WorkPlugin], WorkManager): +class AsynchronousWorkManager(WorkManagerPluginsMixin, WorkManager): def __init__(self, golem: GolemNode, do_work: DoWorkCallable, *args, **kwargs): self._do_work = do_work super().__init__(*args, **kwargs) - def _apply_plugins_from_manager(self, do_work: DoWorkCallable) -> DoWorkCallable: - do_work_with_plugins = do_work - - for plugin in self._plugins: - do_work_with_plugins = plugin(do_work_with_plugins) - - return do_work_with_plugins - - def _apply_plugins_from_work(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: - work_plugins = getattr(work, WORK_PLUGIN_FIELD_NAME, []) - - if not work_plugins: - return do_work - - do_work_with_plugins = do_work - - for plugin in work_plugins: - do_work_with_plugins = plugin(do_work_with_plugins) - - return do_work_with_plugins - async def do_work(self, work: Work) -> WorkResult: - do_work_with_plugins = self._apply_plugins_from_manager(self._do_work) - do_work_with_plugins = self._apply_plugins_from_work(do_work_with_plugins, work) - - result = await do_work_with_plugins(work) + result = await self._do_work_with_plugins(self._do_work, work) logger.info(f"Work `{work}` completed") diff --git a/golem/managers/work/sequential.py b/golem/managers/work/sequential.py index b5720cba..31caf34a 100644 --- a/golem/managers/work/sequential.py +++ b/golem/managers/work/sequential.py @@ -2,12 +2,10 @@ from typing import List from golem.managers.base import ( - WORK_PLUGIN_FIELD_NAME, DoWorkCallable, - ManagerPluginsMixin, Work, WorkManager, - WorkPlugin, + WorkManagerPluginsMixin, WorkResult, ) from golem.node import GolemNode @@ -15,50 +13,15 @@ logger = logging.getLogger(__name__) -class SequentialWorkManager(ManagerPluginsMixin[WorkPlugin], WorkManager): +class SequentialWorkManager(WorkManagerPluginsMixin, WorkManager): def __init__(self, golem: GolemNode, do_work: DoWorkCallable, *args, **kwargs): self._do_work = do_work super().__init__(*args, **kwargs) - def _apply_plugins_from_manager(self, do_work: DoWorkCallable) -> DoWorkCallable: - logger.debug("Applying plugins from manager...") - - do_work_with_plugins = do_work - - for plugin in self._plugins: - do_work_with_plugins = plugin(do_work_with_plugins) - - logger.debug("Applying plugins from manager done") - - return do_work_with_plugins - - def _apply_plugins_from_work(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: - logger.debug(f"Applying plugins from `{work}`...") - - work_plugins = getattr(work, WORK_PLUGIN_FIELD_NAME, []) - - if not work_plugins: - return do_work - - do_work_with_plugins = do_work - - for plugin in work_plugins: - do_work_with_plugins = plugin(do_work_with_plugins) - - logger.debug(f"Applying plugins from `{work}` done") - - return do_work_with_plugins - async def do_work(self, work: Work) -> WorkResult: - logger.debug(f"Running work `{work}`...") - - do_work_with_plugins = self._apply_plugins_from_manager(self._do_work) - do_work_with_plugins = self._apply_plugins_from_work(do_work_with_plugins, work) - - result = await do_work_with_plugins(work) + result = await self._do_work_with_plugins(self._do_work, work) - logger.debug(f"Running work `{work}` done") logger.info(f"Work `{work}` completed") return result From f0e9cf050f239e6bbf5ce6c01a5bbe60903d6843 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 4 Jul 2023 11:20:49 +0200 Subject: [PATCH 068/123] Add ActivityPrepareReleaseMixin --- golem/managers/activity/mixins.py | 52 +++++++++++++++++++++++ golem/managers/activity/pool.py | 49 ++++------------------ golem/managers/activity/single_use.py | 60 +++++---------------------- golem/managers/base.py | 33 --------------- golem/managers/work/asynchronous.py | 9 +--- golem/managers/work/mixins.py | 47 +++++++++++++++++++++ golem/managers/work/sequential.py | 9 +--- 7 files changed, 123 insertions(+), 136 deletions(-) create mode 100644 golem/managers/activity/mixins.py create mode 100644 golem/managers/work/mixins.py diff --git a/golem/managers/activity/mixins.py b/golem/managers/activity/mixins.py new file mode 100644 index 00000000..fabfb1a2 --- /dev/null +++ b/golem/managers/activity/mixins.py @@ -0,0 +1,52 @@ +import logging +from typing import Awaitable, Callable, Optional + +from golem.managers.activity.defaults import default_on_activity_start, default_on_activity_stop +from golem.managers.agreement.events import AgreementReleased +from golem.managers.base import WorkContext +from golem.resources import Activity + +logger = logging.getLogger(__name__) + + +class ActivityPrepareReleaseMixin: + def __init__( + self, + on_activity_start: Optional[ + Callable[[WorkContext], Awaitable[None]] + ] = default_on_activity_start, + on_activity_stop: Optional[ + Callable[[WorkContext], Awaitable[None]] + ] = default_on_activity_stop, + *args, + **kwargs, + ) -> None: + self._on_activity_start = on_activity_start + self._on_activity_stop = on_activity_stop + + super().__init__(*args, **kwargs) + + async def _prepare_activity(self, agreement) -> Activity: + activity = await agreement.create_activity() + logger.info(f"Activity `{activity}` created") + work_context = WorkContext(activity) + if self._on_activity_start: + await self._on_activity_start(work_context) + return activity + + async def _release_activity(self, activity: Activity) -> None: + if self._on_activity_stop: + work_context = WorkContext(activity) + await self._on_activity_stop(work_context) + + if activity.destroyed: + logger.info(f"Activity `{activity}` destroyed") + else: + logger.warning( + "ActivityManager expects that activity will be terminated" + " before activity is released. Looks like you forgot calling" + " `context.terminate()` in custom `on_activity_end` callback." + ) + + event = AgreementReleased(activity.parent) + await self._event_bus.emit(event) diff --git a/golem/managers/activity/pool.py b/golem/managers/activity/pool.py index f33de0e3..72411fdc 100644 --- a/golem/managers/activity/pool.py +++ b/golem/managers/activity/pool.py @@ -1,45 +1,40 @@ import asyncio import logging from contextlib import asynccontextmanager -from typing import Awaitable, Callable, Optional +from typing import Awaitable, Callable -from golem.managers.activity.defaults import default_on_activity_start, default_on_activity_stop -from golem.managers.agreement import AgreementReleased +from golem.managers.activity.mixins import ActivityPrepareReleaseMixin from golem.managers.base import ActivityManager, Work, WorkContext, WorkResult from golem.node import GolemNode -from golem.resources import Activity, Agreement +from golem.resources import Agreement from golem.utils.asyncio import create_task_with_logging logger = logging.getLogger(__name__) -class ActivityPoolManager(ActivityManager): # TODO ActivityPrepeareReleaseMixin +class ActivityPoolManager(ActivityPrepareReleaseMixin, ActivityManager): def __init__( self, golem: GolemNode, get_agreement: Callable[[], Awaitable[Agreement]], size: int, - on_activity_start: Optional[ - Callable[[WorkContext], Awaitable[None]] - ] = default_on_activity_start, - on_activity_stop: Optional[ - Callable[[WorkContext], Awaitable[None]] - ] = default_on_activity_stop, + *args, + **kwargs, ): self._get_agreement = get_agreement self._event_bus = golem.event_bus - self._on_activity_start = on_activity_start - self._on_activity_stop = on_activity_stop self._pool_size = size self._pool = asyncio.Queue(maxsize=self._pool_size) + super().__init__(*args, **kwargs) async def start(self): for _ in range(self._pool_size): create_task_with_logging(self._prepare_activity_and_put_in_pool()) async def _prepare_activity_and_put_in_pool(self): - activity = await self._prepare_activity() + agreement = await self._get_agreement() + activity = await self._prepare_activity(agreement) await self._pool.put(activity) logger.info(f"Activity `{activity}` added to the pool") @@ -52,32 +47,6 @@ async def stop(self): ) assert self._pool.empty() - async def _prepare_activity(self) -> Activity: - agreement = await self._get_agreement() - activity = await agreement.create_activity() - logger.info(f"Activity `{activity}` created") - work_context = WorkContext(activity) - if self._on_activity_start: - await self._on_activity_start(work_context) - return activity - - async def _release_activity(self, activity: Activity) -> None: - if self._on_activity_stop: - work_context = WorkContext(activity) - await self._on_activity_stop(work_context) - - if activity.destroyed: - logger.info(f"Activity `{activity}` destroyed") - else: - logger.warning( - "ActivityPoolManager expects that activity will be terminated" - " before activity is released. Looks like you forgot calling" - " `context.terminate()` in custom `on_activity_end` callback." - ) - - event = AgreementReleased(activity.parent) - await self._event_bus.emit(event) - @asynccontextmanager async def _get_activity_from_pool(self): activity = await self._pool.get() diff --git a/golem/managers/activity/single_use.py b/golem/managers/activity/single_use.py index de77bbce..a21170e6 100644 --- a/golem/managers/activity/single_use.py +++ b/golem/managers/activity/single_use.py @@ -1,9 +1,8 @@ import logging from contextlib import asynccontextmanager -from typing import Awaitable, Callable, Optional +from typing import Awaitable, Callable -from golem.managers.activity.defaults import default_on_activity_start, default_on_activity_stop -from golem.managers.agreement import AgreementReleased +from golem.managers.activity.mixins import ActivityPrepareReleaseMixin from golem.managers.base import ActivityManager, Work, WorkContext, WorkResult from golem.node import GolemNode from golem.resources import Activity, Agreement @@ -11,25 +10,17 @@ logger = logging.getLogger(__name__) -class SingleUseActivityManager(ActivityManager): +class SingleUseActivityManager(ActivityPrepareReleaseMixin, ActivityManager): def __init__( - self, - golem: GolemNode, - get_agreement: Callable[[], Awaitable[Agreement]], - on_activity_start: Optional[ - Callable[[WorkContext], Awaitable[None]] - ] = default_on_activity_start, - on_activity_stop: Optional[ - Callable[[WorkContext], Awaitable[None]] - ] = default_on_activity_stop, + self, golem: GolemNode, get_agreement: Callable[[], Awaitable[Agreement]], *args, **kwargs ): self._get_agreement = get_agreement self._event_bus = golem.event_bus - self._on_activity_start = on_activity_start - self._on_activity_stop = on_activity_stop + + super().__init__(*args, **kwargs) @asynccontextmanager - async def _prepare_activity(self) -> Activity: + async def _prepare_single_use_activity(self) -> Activity: while True: logger.debug("Getting agreement...") @@ -40,7 +31,7 @@ async def _prepare_activity(self) -> Activity: try: logger.debug("Creating activity...") - activity = await agreement.create_activity() + activity = await self._prepare_activity(agreement) logger.debug(f"Creating activity done with `{activity}`") logger.info(f"Activity `{activity}` created") @@ -49,33 +40,20 @@ async def _prepare_activity(self) -> Activity: yield activity + await self._release_activity(activity) + logger.debug("Yielding activity done") break except Exception: logger.exception("Creating activity failed, but will be retried with new agreement") - finally: - event = AgreementReleased(agreement) - - logger.debug(f"Releasing agreement by emitting `{event}`...") - - await self._event_bus.emit(event) - - logger.debug(f"Releasing agreement by emitting `{event}` done") async def do_work(self, work: Work) -> WorkResult: logger.debug(f"Doing work `{work}`...") - async with self._prepare_activity() as activity: + async with self._prepare_single_use_activity() as activity: work_context = WorkContext(activity) - if self._on_activity_start: - logger.debug("Calling `on_activity_start`...") - - await self._on_activity_start(work_context) - - logger.debug("Calling `on_activity_start` done") - try: logger.debug(f"Calling `{work}`...") work_result = await work(work_context) @@ -90,22 +68,6 @@ async def do_work(self, work: Work) -> WorkResult: work_result = WorkResult(result=work_result) - if self._on_activity_stop: - logger.debug(f"Calling `on_activity_stop` on activity `{activity}`...") - - await self._on_activity_stop(work_context) - - logger.debug(f"Calling `on_activity_stop` on activity `{activity}` done") - - if activity.destroyed: - logger.info(f"Activity `{activity}` destroyed") - else: - logger.warning( - "SingleUseActivityManager expects that activity will be terminated" - " after its work is finished. Looks like you forgot calling" - " `context.terminate()` in custom `on_activity_end` callback." - ) - logger.debug(f"Doing work `{work}` done") return work_result diff --git a/golem/managers/base.py b/golem/managers/base.py index 70356df5..ff894345 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -141,39 +141,6 @@ def unregister_plugin(self, plugin: TPlugin): self._plugins.remove(plugin) -class WorkManagerPluginsMixin(ManagerPluginsMixin[WorkPlugin]): - def __init__(self, plugins: Optional[Sequence[TPlugin]] = None, *args, **kwargs) -> None: - self._plugins: List[TPlugin] = list(plugins) if plugins is not None else [] - - super().__init__(*args, **kwargs) - - def _apply_plugins_from_manager(self, do_work: DoWorkCallable) -> DoWorkCallable: - do_work_with_plugins = do_work - - for plugin in self._plugins: - do_work_with_plugins = plugin(do_work_with_plugins) - - return do_work_with_plugins - - def _apply_plugins_from_work(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: - work_plugins = getattr(work, WORK_PLUGIN_FIELD_NAME, []) - - if not work_plugins: - return do_work - - do_work_with_plugins = do_work - - for plugin in work_plugins: - do_work_with_plugins = plugin(do_work_with_plugins) - - return do_work_with_plugins - - async def _do_work_with_plugins(self, do_work: DoWorkCallable, work: Work) -> WorkResult: - do_work_with_plugins = self._apply_plugins_from_manager(do_work) - do_work_with_plugins = self._apply_plugins_from_work(do_work_with_plugins, work) - return await do_work_with_plugins(work) - - class NetworkManager(Manager, ABC): ... diff --git a/golem/managers/work/asynchronous.py b/golem/managers/work/asynchronous.py index e2b5219c..b7476e60 100644 --- a/golem/managers/work/asynchronous.py +++ b/golem/managers/work/asynchronous.py @@ -2,13 +2,8 @@ import logging from typing import List -from golem.managers.base import ( - DoWorkCallable, - Work, - WorkManager, - WorkManagerPluginsMixin, - WorkResult, -) +from golem.managers.base import DoWorkCallable, Work, WorkManager, WorkResult +from golem.managers.work.mixins import WorkManagerPluginsMixin from golem.node import GolemNode logger = logging.getLogger(__name__) diff --git a/golem/managers/work/mixins.py b/golem/managers/work/mixins.py new file mode 100644 index 00000000..01560f01 --- /dev/null +++ b/golem/managers/work/mixins.py @@ -0,0 +1,47 @@ +import logging +from typing import List, Optional, Sequence + +from golem.managers.base import ( + WORK_PLUGIN_FIELD_NAME, + DoWorkCallable, + ManagerPluginsMixin, + TPlugin, + Work, + WorkPlugin, + WorkResult, +) + +logger = logging.getLogger(__name__) + + +class WorkManagerPluginsMixin(ManagerPluginsMixin[WorkPlugin]): + def __init__(self, plugins: Optional[Sequence[TPlugin]] = None, *args, **kwargs) -> None: + self._plugins: List[TPlugin] = list(plugins) if plugins is not None else [] + + super().__init__(*args, **kwargs) + + def _apply_plugins_from_manager(self, do_work: DoWorkCallable) -> DoWorkCallable: + do_work_with_plugins = do_work + + for plugin in self._plugins: + do_work_with_plugins = plugin(do_work_with_plugins) + + return do_work_with_plugins + + def _apply_plugins_from_work(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: + work_plugins = getattr(work, WORK_PLUGIN_FIELD_NAME, []) + + if not work_plugins: + return do_work + + do_work_with_plugins = do_work + + for plugin in work_plugins: + do_work_with_plugins = plugin(do_work_with_plugins) + + return do_work_with_plugins + + async def _do_work_with_plugins(self, do_work: DoWorkCallable, work: Work) -> WorkResult: + do_work_with_plugins = self._apply_plugins_from_manager(do_work) + do_work_with_plugins = self._apply_plugins_from_work(do_work_with_plugins, work) + return await do_work_with_plugins(work) diff --git a/golem/managers/work/sequential.py b/golem/managers/work/sequential.py index 31caf34a..d549556b 100644 --- a/golem/managers/work/sequential.py +++ b/golem/managers/work/sequential.py @@ -1,13 +1,8 @@ import logging from typing import List -from golem.managers.base import ( - DoWorkCallable, - Work, - WorkManager, - WorkManagerPluginsMixin, - WorkResult, -) +from golem.managers.base import DoWorkCallable, Work, WorkManager, WorkResult +from golem.managers.work.mixins import WorkManagerPluginsMixin from golem.node import GolemNode logger = logging.getLogger(__name__) From d9d41da715c308e850bcde3d1b7bc81e77606bec Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 5 Jul 2023 10:09:13 +0200 Subject: [PATCH 069/123] Rename blender example func names --- examples/managers/blender/blender.py | 14 +++++++------- golem/utils/logging.py | 3 +++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/examples/managers/blender/blender.py b/examples/managers/blender/blender.py index adb35a71..a593090f 100644 --- a/examples/managers/blender/blender.py +++ b/examples/managers/blender/blender.py @@ -18,7 +18,7 @@ from golem.utils.logging import DEFAULT_LOGGING FRAME_CONFIG_TEMPLATE = json.loads(Path(__file__).with_name("frame_params.json").read_text()) -FRAMES = list(range(0, 60, 10)) +FRAMES = list(range(0, 60, 1)) async def load_blend_file(context: WorkContext): @@ -29,12 +29,12 @@ async def load_blend_file(context: WorkContext): await batch() -def blender_frame_work(frame_ix: int): - async def _blender_frame_work(context: WorkContext) -> str: +def render_blender_frame(frame_ix: int): + async def _render_blender_frame(context: WorkContext) -> str: frame_config = FRAME_CONFIG_TEMPLATE.copy() frame_config["frames"] = [frame_ix] fname = f"out{frame_ix:04d}.png" - fname_path = str(Path(__file__).parent / fname) + fname_path = str(Path(__file__).parent / "frames" / fname) print(f"BLENDER: Running {fname_path}") @@ -45,14 +45,14 @@ async def _blender_frame_work(context: WorkContext) -> str: await batch() return fname_path - return _blender_frame_work + return _render_blender_frame async def main(): logging.config.dictConfig(DEFAULT_LOGGING) payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") - work_list = [blender_frame_work(frame_ix) for frame_ix in FRAMES] + work_list = [render_blender_frame(frame_ix) for frame_ix in FRAMES] golem = GolemNode() @@ -68,7 +68,7 @@ async def main(): proposal_manager = StackProposalManager(golem, negotiation_manager.get_proposal) agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) activity_manager = ActivityPoolManager( - golem, agreement_manager.get_agreement, size=3, on_activity_start=load_blend_file + golem, agreement_manager.get_agreement, size=10, on_activity_start=load_blend_file ) work_manager = AsynchronousWorkManager( golem, activity_manager.do_work, plugins=[retry(tries=3)] diff --git a/golem/utils/logging.py b/golem/utils/logging.py index 03b95d51..bcc9f42a 100644 --- a/golem/utils/logging.py +++ b/golem/utils/logging.py @@ -39,6 +39,9 @@ "golem.managers.work": { "level": "INFO", }, + "golem.managers.agreement": { + "level": "INFO", + }, }, } From 41f4301d985774cd8a10c49931aa3b716a6a2fe5 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Fri, 7 Jul 2023 11:55:20 +0200 Subject: [PATCH 070/123] Add better pool implementation, add simple high level blender apiu --- examples/managers/blender/blender.py | 55 +++++++++------------------- golem/managers/activity/pool.py | 47 +++++++++++++++++------- golem/utils/blender_api.py | 46 +++++++++++++++++++++++ 3 files changed, 98 insertions(+), 50 deletions(-) create mode 100644 golem/utils/blender_api.py diff --git a/examples/managers/blender/blender.py b/examples/managers/blender/blender.py index a593090f..62b9e628 100644 --- a/examples/managers/blender/blender.py +++ b/examples/managers/blender/blender.py @@ -2,23 +2,15 @@ import json import logging.config from pathlib import Path -from typing import List - -from golem.managers.activity.pool import ActivityPoolManager -from golem.managers.agreement.single_use import SingleUseAgreementManager -from golem.managers.base import WorkContext, WorkResult -from golem.managers.negotiation import SequentialNegotiationManager -from golem.managers.negotiation.plugins import AddChosenPaymentPlatform -from golem.managers.payment.pay_all import PayAllPaymentManager -from golem.managers.proposal import StackProposalManager -from golem.managers.work.asynchronous import AsynchronousWorkManager -from golem.managers.work.plugins import retry -from golem.node import GolemNode + +from golem.managers.base import WorkContext from golem.payload import RepositoryVmPayload +from golem.utils.blender_api import run_on_golem from golem.utils.logging import DEFAULT_LOGGING +BLENDER_IMAGE_HASH = "9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae" FRAME_CONFIG_TEMPLATE = json.loads(Path(__file__).with_name("frame_params.json").read_text()) -FRAMES = list(range(0, 60, 1)) +FRAMES = list(range(0, 60, 3)) async def load_blend_file(context: WorkContext): @@ -36,13 +28,16 @@ async def _render_blender_frame(context: WorkContext) -> str: fname = f"out{frame_ix:04d}.png" fname_path = str(Path(__file__).parent / "frames" / fname) - print(f"BLENDER: Running {fname_path}") + print(f"BLENDER: Generating frame {fname_path}") batch = await context.create_batch() batch.run(f"echo '{json.dumps(frame_config)}' > /golem/work/params.json") batch.run("/golem/entrypoints/run-blender.sh") batch.download_file(f"/golem/output/{fname}", fname_path) await batch() + + print(f"BLENDER: Frame {fname_path} done") + return fname_path return _render_blender_frame @@ -50,33 +45,19 @@ async def _render_blender_frame(context: WorkContext) -> str: async def main(): logging.config.dictConfig(DEFAULT_LOGGING) - payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") + payload = RepositoryVmPayload(BLENDER_IMAGE_HASH) - work_list = [render_blender_frame(frame_ix) for frame_ix in FRAMES] + task_list = [render_blender_frame(frame_ix) for frame_ix in FRAMES] - golem = GolemNode() - - payment_manager = PayAllPaymentManager(golem, budget=1.0) - negotiation_manager = SequentialNegotiationManager( - golem, - payment_manager.get_allocation, - payload, - plugins=[ - AddChosenPaymentPlatform(), - ], - ) - proposal_manager = StackProposalManager(golem, negotiation_manager.get_proposal) - agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) - activity_manager = ActivityPoolManager( - golem, agreement_manager.get_agreement, size=10, on_activity_start=load_blend_file - ) - work_manager = AsynchronousWorkManager( - golem, activity_manager.do_work, plugins=[retry(tries=3)] + results = await run_on_golem( + payload=payload, + task_list=task_list, + init_func=load_blend_file, + threads=6, + budget=1.0, ) - async with golem, payment_manager, negotiation_manager, proposal_manager, activity_manager: - results: List[WorkResult] = await work_manager.do_work_list(work_list) - print(f"\nWORK MANAGER RESULTS:{[result.result for result in results]}\n", flush=True) + print(f"\nBLENDER: all frames:{[result.result for result in results]}\n", flush=True) if __name__ == "__main__": diff --git a/golem/managers/activity/pool.py b/golem/managers/activity/pool.py index 72411fdc..a85de911 100644 --- a/golem/managers/activity/pool.py +++ b/golem/managers/activity/pool.py @@ -24,13 +24,43 @@ def __init__( self._get_agreement = get_agreement self._event_bus = golem.event_bus - self._pool_size = size - self._pool = asyncio.Queue(maxsize=self._pool_size) + self._pool_target_size = size + self._pool = asyncio.Queue() super().__init__(*args, **kwargs) async def start(self): - for _ in range(self._pool_size): - create_task_with_logging(self._prepare_activity_and_put_in_pool()) + self._manage_pool_task = create_task_with_logging(self._manage_pool()) + + async def stop(self): + self._pool_target_size = 0 + # TODO cancel prepare_activity tasks + await self._manage_pool_task + assert self._pool.empty() + + async def _manage_pool(self): + pool_current_size = 0 + release_tasks = [] + prepare_tasks = [] + while self._pool_target_size > 0 or pool_current_size > 0: + # TODO observe tasks status and add fallback + if pool_current_size > self._pool_target_size: + pool_current_size -= 1 + logger.debug(f"Releasing activity from the pool, new size: {pool_current_size}") + release_tasks.append( + create_task_with_logging(self._release_activity_and_pop_from_pool()) + ) + elif pool_current_size < self._pool_target_size: + pool_current_size += 1 + logger.debug(f"Adding activity to the pool, new size: {pool_current_size}") + prepare_tasks.append( + create_task_with_logging(self._prepare_activity_and_put_in_pool()) + ) + await asyncio.sleep(0.01) + + async def _release_activity_and_pop_from_pool(self): + activity = await self._pool.get() + await self._release_activity(activity) + logger.info(f"Activity `{activity}` removed from the pool") async def _prepare_activity_and_put_in_pool(self): agreement = await self._get_agreement() @@ -38,15 +68,6 @@ async def _prepare_activity_and_put_in_pool(self): await self._pool.put(activity) logger.info(f"Activity `{activity}` added to the pool") - async def stop(self): - await asyncio.gather( - *[ - create_task_with_logging(self._release_activity(await self._pool.get())) - for _ in range(self._pool_size) - ] - ) - assert self._pool.empty() - @asynccontextmanager async def _get_activity_from_pool(self): activity = await self._pool.get() diff --git a/golem/utils/blender_api.py b/golem/utils/blender_api.py new file mode 100644 index 00000000..b1f67523 --- /dev/null +++ b/golem/utils/blender_api.py @@ -0,0 +1,46 @@ +from typing import List + +from golem.managers.activity.pool import ActivityPoolManager +from golem.managers.agreement.single_use import SingleUseAgreementManager +from golem.managers.base import WorkResult +from golem.managers.negotiation import SequentialNegotiationManager +from golem.managers.negotiation.plugins import AddChosenPaymentPlatform +from golem.managers.payment.pay_all import PayAllPaymentManager +from golem.managers.proposal import StackProposalManager +from golem.managers.work.asynchronous import AsynchronousWorkManager +from golem.managers.work.plugins import retry +from golem.node import GolemNode + + +async def run_on_golem( + task_list, + payload, + init_func, + threads=6, + budget=1.0, + market_plugins=[ + AddChosenPaymentPlatform(), + ], + execution_plugins=[retry(tries=3)], +): + golem = GolemNode() + + payment_manager = PayAllPaymentManager(golem, budget=budget) + negotiation_manager = SequentialNegotiationManager( + golem, + payment_manager.get_allocation, + payload, + plugins=market_plugins, + ) + proposal_manager = StackProposalManager(golem, negotiation_manager.get_proposal) + agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) + activity_manager = ActivityPoolManager( + golem, agreement_manager.get_agreement, size=threads, on_activity_start=init_func + ) + work_manager = AsynchronousWorkManager( + golem, activity_manager.do_work, plugins=execution_plugins + ) + + async with golem, payment_manager, negotiation_manager, proposal_manager, activity_manager: + results: List[WorkResult] = await work_manager.do_work_list(task_list) + return results From a52fcdd86b705ba8cc3ef23a840cc6d78de2a88f Mon Sep 17 00:00:00 2001 From: approxit Date: Fri, 7 Jul 2023 12:00:52 +0200 Subject: [PATCH 071/123] proposal plugins --- examples/managers/basic_composition.py | 45 ++++- examples/managers/ssh.py | 4 +- golem/managers/base.py | 53 +++++- golem/managers/negotiation/plugins.py | 24 ++- golem/managers/negotiation/sequential.py | 25 +-- golem/managers/proposal/__init__.py | 4 +- golem/managers/proposal/plugins.py | 125 +++++++++++++ golem/managers/proposal/pricings.py | 53 ++++++ golem/managers/proposal/scored_aot.py | 188 ++++++++++++++++++++ golem/managers/proposal/stack.py | 64 ------- golem/managers/work/mixins.py | 11 +- golem/managers/work/plugins.py | 10 +- golem/payload/__init__.py | 11 +- golem/resources/__init__.py | 2 + golem/resources/proposal/__init__.py | 3 +- golem/resources/proposal/proposal.py | 5 +- golem/utils/logging.py | 9 +- tests/unit/test_manager_proposal_plugins.py | 49 +++++ 18 files changed, 565 insertions(+), 120 deletions(-) create mode 100644 golem/managers/proposal/plugins.py create mode 100644 golem/managers/proposal/pricings.py create mode 100644 golem/managers/proposal/scored_aot.py delete mode 100644 golem/managers/proposal/stack.py create mode 100644 tests/unit/test_manager_proposal_plugins.py diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index ed52be31..1288cf1c 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -1,19 +1,27 @@ import asyncio import logging.config -from random import randint -from typing import List, Optional +from datetime import timedelta +from random import randint, random +from typing import List from golem.managers.activity.single_use import SingleUseActivityManager from golem.managers.agreement.single_use import SingleUseAgreementManager from golem.managers.base import RejectProposal, WorkContext, WorkResult from golem.managers.negotiation import SequentialNegotiationManager -from golem.managers.negotiation.plugins import AddChosenPaymentPlatform, BlacklistProviderId +from golem.managers.negotiation.plugins import ( + AddChosenPaymentPlatform, + BlacklistProviderId, + RejectIfCostsExceeds, +) from golem.managers.payment.pay_all import PayAllPaymentManager -from golem.managers.proposal import StackProposalManager +from golem.managers.proposal import ScoredAheadOfTimeProposalManager +from golem.managers.proposal.plugins import MapScore, PropertyValueLerpScore, RandomScore +from golem.managers.proposal.pricings import LinearAverageCostPricing from golem.managers.work.plugins import redundancy_cancel_others_on_first_done, retry, work_plugin from golem.managers.work.sequential import SequentialWorkManager from golem.node import GolemNode from golem.payload import RepositoryVmPayload +from golem.payload.defaults import INF_MEM from golem.resources.demand.demand import DemandData from golem.resources.proposal.proposal import ProposalData from golem.utils.logging import DEFAULT_LOGGING @@ -25,9 +33,7 @@ ] -async def blacklist_func( - demand_data: DemandData, proposal_data: ProposalData -) -> Optional[RejectProposal]: +async def blacklist_func(demand_data: DemandData, proposal_data: ProposalData) -> None: provider_id = proposal_data.issuer_id if provider_id in BLACKLISTED_PROVIDERS: raise RejectProposal(f"Provider ID `{provider_id}` is blacklisted by the requestor") @@ -67,6 +73,10 @@ async def main(): golem = GolemNode() + linear_average_cost = LinearAverageCostPricing( + average_cpu_load=0.2, average_duration=timedelta(seconds=5) + ) + payment_manager = PayAllPaymentManager(golem, budget=1.0) negotiation_manager = SequentialNegotiationManager( golem, @@ -76,6 +86,7 @@ async def main(): AddChosenPaymentPlatform(), # class based plugin BlacklistProviderId(BLACKLISTED_PROVIDERS), + RejectIfCostsExceeds(1, linear_average_cost), # func plugin blacklist_func, # lambda plugin @@ -86,10 +97,26 @@ async def main(): else None, ], ) - proposal_manager = StackProposalManager(golem, negotiation_manager.get_proposal) + proposal_manager = ScoredAheadOfTimeProposalManager( + golem, + negotiation_manager.get_proposal, + plugins=[ + MapScore(linear_average_cost, normalize=True, normalize_flip=True), + [0.5, PropertyValueLerpScore(INF_MEM, zero_at=1, one_at=8)], + [0.1, RandomScore()], + [0.0, lambda proposals_data: [random() for _ in range(len(proposals_data))]], + [0.0, MapScore(lambda proposal_data: random())], + ], + ) agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) activity_manager = SingleUseActivityManager(golem, agreement_manager.get_agreement) - work_manager = SequentialWorkManager(golem, activity_manager.do_work, plugins=[retry(tries=5)]) + work_manager = SequentialWorkManager( + golem, + activity_manager.do_work, + plugins=[ + retry(tries=5), + ], + ) async with golem: async with payment_manager, negotiation_manager, proposal_manager: diff --git a/examples/managers/ssh.py b/examples/managers/ssh.py index 7ebb37c5..ca2d093b 100644 --- a/examples/managers/ssh.py +++ b/examples/managers/ssh.py @@ -10,7 +10,7 @@ from golem.managers.negotiation import SequentialNegotiationManager from golem.managers.network.single import SingleNetworkManager from golem.managers.payment.pay_all import PayAllPaymentManager -from golem.managers.proposal import StackProposalManager +from golem.managers.proposal import ScoredAheadOfTimeProposalManager from golem.managers.work.sequential import SequentialWorkManager from golem.node import GolemNode from golem.payload import RepositoryVmPayload @@ -74,7 +74,7 @@ async def main(): negotiation_manager = SequentialNegotiationManager( golem, payment_manager.get_allocation, payload ) - proposal_manager = StackProposalManager(golem, negotiation_manager.get_proposal) + proposal_manager = ScoredAheadOfTimeProposalManager(golem, negotiation_manager.get_proposal) agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) activity_manager = SingleUseActivityManager( golem, diff --git a/golem/managers/base.py b/golem/managers/base.py index ff894345..909f1daf 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -1,6 +1,18 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import Any, Awaitable, Callable, Dict, Generic, List, Optional, Sequence, TypeVar, Union +from typing import ( + Any, + Awaitable, + Callable, + Dict, + Generic, + List, + Optional, + Sequence, + Tuple, + TypeVar, + Union, +) from golem.exceptions import GolemException from golem.resources import ( @@ -88,7 +100,7 @@ class WorkResult: class Work(ABC): - _work_plugins: Optional[List["WorkPlugin"]] + _work_plugins: Optional[List["WorkManagerPlugin"]] def __call__(self, context: WorkContext) -> Awaitable[Optional[WorkResult]]: ... @@ -97,11 +109,6 @@ def __call__(self, context: WorkContext) -> Awaitable[Optional[WorkResult]]: DoWorkCallable = Callable[[Work], Awaitable[WorkResult]] -class WorkPlugin(ABC): - def __call__(self, do_work: DoWorkCallable) -> DoWorkCallable: - ... - - class ManagerEvent(ResourceEvent, ABC): pass @@ -110,6 +117,10 @@ class ManagerException(GolemException): pass +class ManagerPluginException(ManagerException): + pass + + class Manager(ABC): async def __aenter__(self): await self.start() @@ -179,13 +190,37 @@ class WorkManager(Manager, ABC): ... -class RejectProposal(Exception): +class RejectProposal(ManagerPluginException): pass -class NegotiationPlugin(ABC): +class NegotiationManagerPlugin(ABC): @abstractmethod def __call__( self, demand_data: DemandData, proposal_data: ProposalData ) -> Union[Awaitable[Optional[RejectProposal]], Optional[RejectProposal]]: ... + + +ProposalPluginResult = Sequence[Optional[float]] + + +class ProposalManagerPlugin(ABC): + @abstractmethod + def __call__( + self, proposals_data: Sequence[ProposalData] + ) -> Union[Awaitable[ProposalPluginResult], ProposalPluginResult]: + ... + + +ProposalManagerPluginWithOptionalWeight = Union[ + ProposalManagerPlugin, Tuple[float, ProposalManagerPlugin] +] + + +class WorkManagerPlugin(ABC): + def __call__(self, do_work: DoWorkCallable) -> DoWorkCallable: + ... + + +PricingCallable = Callable[[ProposalData], Optional[float]] diff --git a/golem/managers/negotiation/plugins.py b/golem/managers/negotiation/plugins.py index 0280c059..0ebe7071 100644 --- a/golem/managers/negotiation/plugins.py +++ b/golem/managers/negotiation/plugins.py @@ -1,15 +1,14 @@ import logging from typing import Sequence, Set -from golem.managers.base import NegotiationPlugin -from golem.managers.negotiation.sequential import RejectProposal +from golem.managers.base import NegotiationManagerPlugin, PricingCallable, RejectProposal from golem.payload import Properties from golem.resources import DemandData, ProposalData logger = logging.getLogger(__name__) -class BlacklistProviderId(NegotiationPlugin): +class BlacklistProviderId(NegotiationManagerPlugin): def __init__(self, blacklist: Sequence[str]) -> None: self._blacklist = blacklist @@ -29,7 +28,7 @@ async def __call__(self, demand_data: DemandData, proposal_data: ProposalData) - ) -class AddChosenPaymentPlatform(NegotiationPlugin): +class AddChosenPaymentPlatform(NegotiationManagerPlugin): async def __call__(self, demand_data: DemandData, proposal_data: ProposalData) -> None: logger.debug("Calling chosen payment platform plugin...") @@ -60,6 +59,19 @@ def _get_payment_platform_from_properties(self, properties: Properties) -> Set[s } -class MidAgreementPayment(NegotiationPlugin): +class RejectIfCostsExceeds(NegotiationManagerPlugin): + def __init__( + self, cost: float, pricing_callable: PricingCallable, reject_on_unpricable=True + ) -> None: + self._cost = cost + self._pricing_callable = pricing_callable + self.reject_on_unpricable = reject_on_unpricable + async def __call__(self, demand_data: DemandData, proposal_data: ProposalData) -> None: - ... + cost = self._pricing_callable(proposal_data) + + if cost is None and self.reject_on_unpricable: + raise RejectProposal("Can't estimate costs!") + + if self._cost <= cost: + raise RejectProposal(f"Exceeds estimated costs of `{self._cost}`!") diff --git a/golem/managers/negotiation/sequential.py b/golem/managers/negotiation/sequential.py index 3d4304c1..5cc6c3e4 100644 --- a/golem/managers/negotiation/sequential.py +++ b/golem/managers/negotiation/sequential.py @@ -2,14 +2,15 @@ import logging from copy import deepcopy from datetime import datetime -from typing import AsyncIterator, Awaitable, Callable, List, Optional, Sequence, cast +from typing import AsyncIterator, Awaitable, Callable, Optional, cast from ya_market import ApiException from golem.managers.base import ( ManagerException, + ManagerPluginsMixin, NegotiationManager, - NegotiationPlugin, + NegotiationManagerPlugin, RejectProposal, ) from golem.node import GolemNode @@ -21,28 +22,26 @@ logger = logging.getLogger(__name__) -class SequentialNegotiationManager(NegotiationManager): +class SequentialNegotiationManager( + ManagerPluginsMixin[NegotiationManagerPlugin], NegotiationManager +): def __init__( self, golem: GolemNode, get_allocation: Callable[[], Awaitable[Allocation]], payload: Payload, - plugins: Optional[Sequence[NegotiationPlugin]] = None, + *args, + **kwargs, ) -> None: self._golem = golem self._get_allocation = get_allocation self._payload = payload self._negotiation_loop_task: Optional[asyncio.Task] = None - self._plugins: List[NegotiationPlugin] = list(plugins) if plugins is not None else [] self._eligible_proposals: asyncio.Queue[Proposal] = asyncio.Queue() self._demand_offer_parser = TextXPayloadSyntaxParser() - def register_plugin(self, plugin: NegotiationPlugin): - self._plugins.append(plugin) - - def unregister_plugin(self, plugin: NegotiationPlugin): - self._plugins.remove(plugin) + super().__init__(*args, **kwargs) async def get_proposal(self) -> Proposal: logger.debug("Getting proposal...") @@ -140,10 +139,14 @@ async def _negotiate_proposal( for plugin in self._plugins: plugin_result = plugin(demand_data_after_plugins, proposal_data) + if asyncio.iscoroutine(plugin_result): plugin_result = await plugin_result + if isinstance(plugin_result, RejectProposal): raise plugin_result + + # Note: Explicit identity to False desired here, not "falsy" check if plugin_result is False: raise RejectProposal() @@ -180,7 +183,7 @@ async def _negotiate_proposal( try: new_offer_proposal = await demand_proposal.responses().__anext__() except StopAsyncIteration: - logger.debug("Waiting for response failed with rejection") + logger.debug("Waiting for response failed with provider rejection") return None logger.debug(f"Waiting for response done with `{new_offer_proposal}`") diff --git a/golem/managers/proposal/__init__.py b/golem/managers/proposal/__init__.py index b42aaf11..82e9a6aa 100644 --- a/golem/managers/proposal/__init__.py +++ b/golem/managers/proposal/__init__.py @@ -1,3 +1,3 @@ -from golem.managers.proposal.stack import StackProposalManager +from golem.managers.proposal.scored_aot import ScoredAheadOfTimeProposalManager -__all__ = ("StackProposalManager",) +__all__ = ("ScoredAheadOfTimeProposalManager",) diff --git a/golem/managers/proposal/plugins.py b/golem/managers/proposal/plugins.py new file mode 100644 index 00000000..9592794c --- /dev/null +++ b/golem/managers/proposal/plugins.py @@ -0,0 +1,125 @@ +from random import random +from typing import Callable, Optional, Sequence, Tuple, Type, Union + +from golem.managers.base import ManagerPluginException, ProposalManagerPlugin, ProposalPluginResult +from golem.payload.constraints import PropertyName +from golem.resources import ProposalData + +PropertyValueNumeric: Type[Union[int, float]] = Union[int, float] +BoundaryValues = Tuple[Tuple[float, PropertyValueNumeric], Tuple[float, PropertyValueNumeric]] + + +class PropertyValueLerpScore(ProposalManagerPlugin): + def __init__( + self, + property_name: PropertyName, + *, + minus_one_at: Optional[PropertyValueNumeric] = None, + zero_at: Optional[PropertyValueNumeric] = None, + one_at: Optional[PropertyValueNumeric] = None, + raise_on_missing=False, + raise_on_bad_value=False, + ) -> None: + self._property_name = property_name + self._boundary_values = self._get_boundary_values(minus_one_at, zero_at, one_at) + self._raise_on_missing = raise_on_missing + self._raise_on_bad_value = raise_on_bad_value + + def __call__(self, proposals_data: Sequence[ProposalData]) -> ProposalPluginResult: + return [self._calculate_linear_score(proposal_data) for proposal_data in proposals_data] + + def _calculate_linear_score(self, proposal_data: ProposalData) -> Optional[float]: + property_value = self._get_property_value(proposal_data) + + if property_value is None: + return None + + bounds_min = min([self._boundary_values[0][1], self._boundary_values[1][1]]) + bounds_max = max([self._boundary_values[0][1], self._boundary_values[1][1]]) + + x1, y1 = self._boundary_values[0] + x2, y2 = self._boundary_values[1] + y3 = min(bounds_max, max(property_value, bounds_min)) + + # formula taken from https://www.johndcook.com/interpolatorhelp.html + return (((y2 - y3) * x1) + ((y3 - y1) * x2)) / (y2 - y1) + + def _get_property_value(self, proposal_data: ProposalData) -> Optional[PropertyValueNumeric]: + try: + property_value = proposal_data.properties[self._property_name] + except KeyError: + if self._raise_on_missing: + raise ManagerPluginException(f"Property `{self._property_name}` is not found!") + + return None + + if not isinstance(property_value, PropertyValueNumeric): + if self._raise_on_bad_value: + raise ManagerPluginException( + f"Field `{self._property_name}` value type must be an `int` or `float`!" + ) + + return None + + return property_value + + def _get_boundary_values( + self, + minus_one_at: Optional[PropertyValueNumeric] = None, + zero_at: Optional[PropertyValueNumeric] = None, + one_at: Optional[PropertyValueNumeric] = None, + ) -> BoundaryValues: + if minus_one_at is not None: + bounds_min = (-1, minus_one_at) + elif zero_at is not None: + bounds_min = (0, zero_at) + else: + raise ManagerPluginException( + "One of boundary arguments `minus_one_at`, `zero_at` must be provided!" + ) + + if one_at is not None: + bounds_max = (1, one_at) + elif zero_at is not None: + bounds_max = (0, zero_at) + else: + raise ManagerPluginException( + "One of boundary arguments `zero_at`, `one_at` must be provided!" + ) + + return bounds_min, bounds_max + + +class RandomScore(ProposalManagerPlugin): + def __call__(self, proposals_data: Sequence[ProposalData]) -> ProposalPluginResult: + return [random() for _ in range(len(proposals_data))] + + +class MapScore(ProposalManagerPlugin): + def __init__( + self, + callback: Callable[[ProposalData], Optional[float]], + normalize=False, + normalize_flip=False, + ) -> None: + self._callback = callback + self._normalize = normalize + self._normalize_flip = normalize_flip + + def __call__(self, proposals_data: Sequence[ProposalData]) -> ProposalPluginResult: + result = [self._callback(proposal_data) for proposal_data in proposals_data] + + if self._normalize and result: + result_max = max(result) + result_min = min(result) + result_div = result_max - result_min + + if result_div == 0: + return result + + result = [(v - result_min) / result_div for v in result] + + if self._normalize_flip: + result = [1 - v for v in result] + + return result diff --git a/golem/managers/proposal/pricings.py b/golem/managers/proposal/pricings.py new file mode 100644 index 00000000..7cbcb288 --- /dev/null +++ b/golem/managers/proposal/pricings.py @@ -0,0 +1,53 @@ +import logging +from datetime import timedelta +from typing import Optional, Tuple + +from golem.managers.base import PricingCallable +from golem.resources import ProposalData + + +class LinearAverageCostPricing(PricingCallable): + def __init__(self, average_cpu_load: float, average_duration: timedelta) -> None: + self._average_cpu_load = average_cpu_load + self._average_duration = average_duration + + def __call__(self, proposal_data: ProposalData) -> Optional[float]: + coeffs = self._get_linear_coeffs(proposal_data) + + if coeffs is None: + return None + + return self._calculate_cost(*coeffs) + + def _get_linear_coeffs( + self, proposal_data: ProposalData + ) -> Optional[Tuple[float, float, float]]: + pricing_model = proposal_data.properties.get("golem.com.pricing.model") + + if pricing_model != "linear": + logging.debug( + f"Proposal `{proposal_data.proposal_id}` is not in `linear` pricing model, ignoring" + ) + return None + + coeffs = proposal_data.properties.get("golem.com.pricing.model.linear.coeffs") + + if not (isinstance(coeffs, (list, tuple)) and len(coeffs) == 3): + logging.debug( + f"Proposal `{proposal_data.proposal_id}` linear pricing coeffs must be a 3 element sequence, ignoring" + ) + + return None + + return coeffs + + def _calculate_cost( + self, price_duration_sec: float, price_cpu_sec: float, price_initial: float + ) -> float: + average_duration_sec = self._average_duration.total_seconds() + + average_duration_cost = price_duration_sec * average_duration_sec + average_cpu_cost = price_cpu_sec * self._average_cpu_load * average_duration_sec + average_initial_price = price_initial / average_duration_sec + + return average_duration_cost + average_cpu_cost + average_initial_price diff --git a/golem/managers/proposal/scored_aot.py b/golem/managers/proposal/scored_aot.py new file mode 100644 index 00000000..80136402 --- /dev/null +++ b/golem/managers/proposal/scored_aot.py @@ -0,0 +1,188 @@ +import asyncio +import inspect +import logging +from datetime import datetime +from typing import Awaitable, Callable, List, Optional, Sequence, Tuple, cast + +from golem.managers.base import ( + ManagerException, + ManagerPluginsMixin, + ProposalManager, + ProposalManagerPluginWithOptionalWeight, +) +from golem.node import GolemNode +from golem.payload import Properties +from golem.payload.parsers.textx import TextXPayloadSyntaxParser +from golem.resources import Proposal, ProposalData +from golem.utils.asyncio import create_task_with_logging + +logger = logging.getLogger(__name__) + + +class IgnoreProposal(Exception): + pass + + +class ScoredAheadOfTimeProposalManager( + ManagerPluginsMixin[ProposalManagerPluginWithOptionalWeight], ProposalManager +): + def __init__( + self, golem: GolemNode, get_proposal: Callable[[], Awaitable[Proposal]], *args, **kwargs + ) -> None: + self._get_proposal = get_proposal + self._consume_proposals_task: Optional[asyncio.Task] = None + self._demand_offer_parser = TextXPayloadSyntaxParser() + + self._scored_proposals: List[Tuple[float, Proposal]] = [] + self._scored_proposals_condition = asyncio.Condition() + + super().__init__(*args, **kwargs) + + async def start(self) -> None: + logger.debug("Starting...") + + if self.is_started(): + message = "Already started!" + logger.debug(f"Starting failed with `{message}`") + raise ManagerException(message) + + self._consume_proposals_task = create_task_with_logging(self._consume_proposals()) + + logger.debug("Starting done") + + async def stop(self) -> None: + logger.debug("Stopping...") + + if not self.is_started(): + message = "Already stopped!" + logger.debug(f"Stopping failed with `{message}`") + raise ManagerException(message) + + self._consume_proposals_task.cancel() + self._consume_proposals_task = None + + logger.debug("Stopping done") + + def is_started(self) -> bool: + return self._consume_proposals_task is not None and not self._consume_proposals_task.done() + + async def _consume_proposals(self) -> None: + while True: + proposal = await self._get_proposal() + + logger.debug(f"Adding proposal `{proposal}` on the scoring...") + + async with self._scored_proposals_condition: + all_proposals = list(sp[1] for sp in self._scored_proposals) + all_proposals.append(proposal) + + self._scored_proposals = await self._do_scoring(all_proposals) + + self._scored_proposals_condition.notify_all() + + logger.debug(f"Adding proposal `{proposal}` on the scoring done") + + async def get_proposal(self) -> Proposal: + logger.debug("Getting proposal...") + + async with self._scored_proposals_condition: + await self._scored_proposals_condition.wait_for(lambda: 0 < len(self._scored_proposals)) + + score, proposal = self._scored_proposals.pop(0) + + logger.debug(f"Getting proposal done with `{proposal}` with score `{score}`") + + logger.info(f"Proposal `{proposal}` picked") + + return proposal + + async def _do_scoring(self, proposals: Sequence[Proposal]): + proposals_data = await self._get_proposals_data_from_proposals(proposals) + proposal_scores = await self._run_plugins(proposals_data) + + scored_proposals = self._calculate_proposal_score(proposals, proposal_scores) + scored_proposals.sort(key=lambda x: x[0], reverse=True) + + return scored_proposals + + async def _run_plugins( + self, proposals_data: Sequence[ProposalData] + ) -> Sequence[Tuple[float, Sequence[float]]]: + proposal_scores = [] + + for plugin in self._plugins: + if isinstance(plugin, (list, tuple)): + weight, plugin = plugin + else: + weight = 1 + + plugin_scores = plugin(proposals_data) + + if inspect.isawaitable(plugin_scores): + plugin_scores = await plugin_scores + + proposal_scores.append((weight, plugin_scores)) + + return proposal_scores + + def _calculate_proposal_score( + self, + proposals: Sequence[Proposal], + plugin_scores: Sequence[Tuple[float, Sequence[float]]], + ) -> List[Tuple[float, Proposal]]: + # FIXME: can this be refactored? + return [ + ( + self._calculate_weighted_score( + self._transpose_plugin_scores(proposal_index, plugin_scores) + ), + proposal, + ) + for proposal_index, proposal in enumerate(proposals) + ] + + def _transpose_plugin_scores( + self, proposal_index: int, plugin_scores: Sequence[Tuple[float, Sequence[float]]] + ) -> Sequence[Tuple[float, float]]: + # FIXME: can this be refactored? + return [ + (plugin_weight, plugin_scores[proposal_index]) + for plugin_weight, plugin_scores in plugin_scores + if plugin_scores[proposal_index] is None + ] + + def _calculate_weighted_score( + self, proposal_weighted_scores: Sequence[Tuple[float, float]] + ) -> float: + if not proposal_weighted_scores: + return 0 + + weighted_sum = sum(pws[0] * pws[1] for pws in proposal_weighted_scores) + weights_sum = sum(pws[0] for pws in proposal_weighted_scores) + + return weighted_sum / weights_sum + + # FIXME: This should be already provided by low level + async def _get_proposals_data_from_proposals( + self, proposals: Sequence[Proposal] + ) -> Sequence[ProposalData]: + result = [] + + for proposal in proposals: + data = await proposal.get_data() + + constraints = self._demand_offer_parser.parse_constraints(data.constraints) + + result.append( + ProposalData( + properties=Properties(data.properties), + constraints=constraints, + proposal_id=data.proposal_id, + issuer_id=data.issuer_id, + state=data.state, + timestamp=cast(datetime, data.timestamp), + prev_proposal_id=data.prev_proposal_id, + ) + ) + + return result diff --git a/golem/managers/proposal/stack.py b/golem/managers/proposal/stack.py deleted file mode 100644 index a048e279..00000000 --- a/golem/managers/proposal/stack.py +++ /dev/null @@ -1,64 +0,0 @@ -import asyncio -import logging -from typing import Optional - -from golem.managers.base import ManagerException, ProposalManager -from golem.node import GolemNode -from golem.resources import Proposal -from golem.utils.asyncio import create_task_with_logging - -logger = logging.getLogger(__name__) - - -class StackProposalManager(ProposalManager): - def __init__(self, golem: GolemNode, get_proposal) -> None: - self._get_proposal = get_proposal - self._proposals: asyncio.Queue[Proposal] = asyncio.Queue() - self._consume_proposals_task: Optional[asyncio.Task] = None - - async def start(self) -> None: - logger.debug("Starting...") - - if self.is_started(): - message = "Already started!" - logger.debug(f"Starting failed with `{message}`") - raise ManagerException(message) - - self._consume_proposals_task = create_task_with_logging(self._consume_proposals()) - - logger.debug("Starting done") - - async def stop(self) -> None: - logger.debug("Stopping...") - - if not self.is_started(): - message = "Already stopped!" - logger.debug(f"Stopping failed with `{message}`") - raise ManagerException(message) - - self._consume_proposals_task.cancel() - self._consume_proposals_task = None - - logger.debug("Stopping done") - - def is_started(self) -> bool: - return self._consume_proposals_task is not None and not self._consume_proposals_task.done() - - async def _consume_proposals(self) -> None: - while True: - proposal = await self._get_proposal() - - logger.debug(f"Adding proposal `{proposal}` on the stack") - - await self._proposals.put(proposal) - - async def get_proposal(self) -> Proposal: - logger.debug("Getting proposal...") - - proposal = await self._proposals.get() - - logger.debug(f"Getting proposal done with `{proposal}`") - - logger.info(f"Proposal `{proposal}` picked") - - return proposal diff --git a/golem/managers/work/mixins.py b/golem/managers/work/mixins.py index 01560f01..c5cc997f 100644 --- a/golem/managers/work/mixins.py +++ b/golem/managers/work/mixins.py @@ -1,25 +1,18 @@ import logging -from typing import List, Optional, Sequence from golem.managers.base import ( WORK_PLUGIN_FIELD_NAME, DoWorkCallable, ManagerPluginsMixin, - TPlugin, Work, - WorkPlugin, + WorkManagerPlugin, WorkResult, ) logger = logging.getLogger(__name__) -class WorkManagerPluginsMixin(ManagerPluginsMixin[WorkPlugin]): - def __init__(self, plugins: Optional[Sequence[TPlugin]] = None, *args, **kwargs) -> None: - self._plugins: List[TPlugin] = list(plugins) if plugins is not None else [] - - super().__init__(*args, **kwargs) - +class WorkManagerPluginsMixin(ManagerPluginsMixin[WorkManagerPlugin]): def _apply_plugins_from_manager(self, do_work: DoWorkCallable) -> DoWorkCallable: do_work_with_plugins = do_work diff --git a/golem/managers/work/plugins.py b/golem/managers/work/plugins.py index 634f035a..be535dcb 100644 --- a/golem/managers/work/plugins.py +++ b/golem/managers/work/plugins.py @@ -2,12 +2,18 @@ import logging from functools import wraps -from golem.managers.base import WORK_PLUGIN_FIELD_NAME, DoWorkCallable, Work, WorkPlugin, WorkResult +from golem.managers.base import ( + WORK_PLUGIN_FIELD_NAME, + DoWorkCallable, + Work, + WorkManagerPlugin, + WorkResult, +) logger = logging.getLogger(__name__) -def work_plugin(plugin: WorkPlugin): +def work_plugin(plugin: WorkManagerPlugin): def _work_plugin(work: Work): if not hasattr(work, WORK_PLUGIN_FIELD_NAME): work._work_plugins = [] diff --git a/golem/payload/__init__.py b/golem/payload/__init__.py index 7d55acf4..25e5afbc 100644 --- a/golem/payload/__init__.py +++ b/golem/payload/__init__.py @@ -1,5 +1,12 @@ from golem.payload.base import Payload, constraint, prop -from golem.payload.constraints import Constraint, ConstraintException, ConstraintGroup, Constraints +from golem.payload.constraints import ( + Constraint, + ConstraintException, + ConstraintGroup, + Constraints, + PropertyName, + PropertyValue, +) from golem.payload.exceptions import InvalidProperties, PayloadException from golem.payload.parsers import PayloadSyntaxParser, SyntaxException from golem.payload.properties import Properties @@ -22,4 +29,6 @@ "InvalidProperties", "SyntaxException", "PayloadSyntaxParser", + "PropertyName", + "PropertyValue", ) diff --git a/golem/resources/__init__.py b/golem/resources/__init__.py index f05d3dc1..4901dd40 100644 --- a/golem/resources/__init__.py +++ b/golem/resources/__init__.py @@ -83,6 +83,7 @@ ProposalClosed, ProposalData, ProposalDataChanged, + ProposalId, default_create_agreement, default_negotiate, ) @@ -143,6 +144,7 @@ "CommandCancelled", "BatchTimeoutError", "Proposal", + "ProposalId", "ProposalData", "NewProposal", "ProposalDataChanged", diff --git a/golem/resources/proposal/__init__.py b/golem/resources/proposal/__init__.py index fde6ef7d..3ce58e48 100644 --- a/golem/resources/proposal/__init__.py +++ b/golem/resources/proposal/__init__.py @@ -1,10 +1,11 @@ from golem.resources.proposal.events import NewProposal, ProposalClosed, ProposalDataChanged from golem.resources.proposal.pipeline import default_create_agreement, default_negotiate -from golem.resources.proposal.proposal import Proposal, ProposalData +from golem.resources.proposal.proposal import Proposal, ProposalData, ProposalId __all__ = ( "Proposal", "ProposalData", + "ProposalId", "NewProposal", "ProposalDataChanged", "ProposalClosed", diff --git a/golem/resources/proposal/proposal.py b/golem/resources/proposal/proposal.py index ed4a4c01..4295ad22 100644 --- a/golem/resources/proposal/proposal.py +++ b/golem/resources/proposal/proposal.py @@ -1,7 +1,7 @@ import asyncio from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, AsyncIterator, Literal, Optional, Union +from typing import TYPE_CHECKING, AsyncIterator, Literal, Optional, Type, Union from ya_market import RequestorApi from ya_market import models as models @@ -16,6 +16,7 @@ from golem.resources.demand import Demand +ProposalId: Type[str] = str ProposalState = Literal["Initial", "Draft", "Rejected", "Accepted", "Expired"] @@ -23,7 +24,7 @@ class ProposalData: properties: Properties constraints: Constraints - proposal_id: str + proposal_id: ProposalId issuer_id: str state: ProposalState timestamp: datetime diff --git a/golem/utils/logging.py b/golem/utils/logging.py index bcc9f42a..32a10226 100644 --- a/golem/utils/logging.py +++ b/golem/utils/logging.py @@ -13,7 +13,12 @@ "format": "[%(asctime)s %(levelname)s %(name)s] %(message)s", }, }, - "handlers": {"console": {"formatter": "default", "class": "logging.StreamHandler"}}, + "handlers": { + "console": { + "formatter": "default", + "class": "logging.StreamHandler", + }, + }, "loggers": { "": { "level": "INFO", @@ -34,7 +39,7 @@ "level": "INFO", }, "golem.managers.proposal": { - "level": "INFO", + "level": "DEBUG", }, "golem.managers.work": { "level": "INFO", diff --git a/tests/unit/test_manager_proposal_plugins.py b/tests/unit/test_manager_proposal_plugins.py new file mode 100644 index 00000000..1ff7f930 --- /dev/null +++ b/tests/unit/test_manager_proposal_plugins.py @@ -0,0 +1,49 @@ +import pytest + +from golem.managers.proposal.plugins import PropertyValueLerpScore, RandomScore + + +@pytest.mark.parametrize( + "kwargs, property_value, expected", + ( + (dict(zero_at=0, one_at=1), 0.5, 0.5), + (dict(zero_at=0, one_at=2), 1, 0.5), + (dict(zero_at=0, one_at=100), 25, 0.25), + (dict(zero_at=0, one_at=100), -1, 0), + (dict(zero_at=0, one_at=100), 200, 1), + (dict(zero_at=-10, one_at=10), 0, 0.5), + (dict(zero_at=-10, one_at=10), -10, 0), + (dict(minus_one_at=10, zero_at=0), 10, -1), + (dict(minus_one_at=10, zero_at=0), 5, -0.5), + (dict(minus_one_at=-1, one_at=1), 0, 0), + (dict(minus_one_at=-1, one_at=1), -0.5, -0.5), + (dict(minus_one_at=0, one_at=100), 50, 0), + (dict(minus_one_at=0, one_at=100), 500, 1), + (dict(minus_one_at=0, one_at=100), -500, -1), + (dict(minus_one_at=0, one_at=100), -500, -1), + ), +) +def test_linear_score_plugin(kwargs, property_value, expected, mocker): + property_name = "some.property" + proposal_id = "foo" + + scorer = PropertyValueLerpScore(property_name=property_name, **kwargs) + proposal_data = mocker.Mock(proposal_id=proposal_id, properties={property_name: property_value}) + + result = scorer([proposal_data]) + + assert result[proposal_id] == expected + + +@pytest.mark.parametrize("expected_value", (0.876, 0.0, 0.2, 0.5, 1)) +def test_random_score_plugin(mocker, expected_value): + mocker.patch("golem.managers.proposal.plugins.random", mocker.Mock(return_value=expected_value)) + + proposal_id = "foo" + + scorer = RandomScore() + proposal_data = mocker.Mock(proposal_id=proposal_id) + + result = scorer([proposal_data]) + + assert result[proposal_id] == expected_value From 029e67e6a82f890bbc64d952b50ddd03ac2962e5 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 10 Jul 2023 13:00:54 +0200 Subject: [PATCH 072/123] Move blender api example to example --- examples/managers/blender/blender.py | 64 ++++++++++++++++++++++++++++ golem/utils/blender_api.py | 46 -------------------- golem/utils/logging.py | 2 +- 3 files changed, 65 insertions(+), 47 deletions(-) delete mode 100644 golem/utils/blender_api.py diff --git a/examples/managers/blender/blender.py b/examples/managers/blender/blender.py index 62b9e628..03efb032 100644 --- a/examples/managers/blender/blender.py +++ b/examples/managers/blender/blender.py @@ -1,18 +1,72 @@ import asyncio import json import logging.config +from datetime import timedelta from pathlib import Path from golem.managers.base import WorkContext +from golem.managers.proposal.plugins import MapScore +from golem.managers.proposal.pricings import LinearAverageCostPricing +from golem.managers.work.plugins import retry from golem.payload import RepositoryVmPayload from golem.utils.blender_api import run_on_golem from golem.utils.logging import DEFAULT_LOGGING + +from typing import List + +from golem.managers.activity.pool import ActivityPoolManager +from golem.managers.agreement.single_use import SingleUseAgreementManager +from golem.managers.base import WorkResult +from golem.managers.negotiation import SequentialNegotiationManager +from golem.managers.negotiation.plugins import AddChosenPaymentPlatform +from golem.managers.payment.pay_all import PayAllPaymentManager +from golem.managers.proposal import ScoredAheadOfTimeProposalManager +from golem.managers.work.asynchronous import AsynchronousWorkManager +from golem.node import GolemNode + BLENDER_IMAGE_HASH = "9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae" FRAME_CONFIG_TEMPLATE = json.loads(Path(__file__).with_name("frame_params.json").read_text()) FRAMES = list(range(0, 60, 3)) +async def run_on_golem( + task_list, + payload, + init_func, + threads=6, + budget=1.0, + market_plugins=[ + AddChosenPaymentPlatform(), + ], + scoring_plugins=None, + task_plugins=None, +): + golem = GolemNode() + + payment_manager = PayAllPaymentManager(golem, budget=budget) + negotiation_manager = SequentialNegotiationManager( + golem, + payment_manager.get_allocation, + payload, + plugins=market_plugins, + ) + proposal_manager = ScoredAheadOfTimeProposalManager( + golem, negotiation_manager.get_proposal, plugins=scoring_plugins + ) + agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) + activity_manager = ActivityPoolManager( + golem, agreement_manager.get_agreement, size=threads, on_activity_start=init_func + ) + work_manager = AsynchronousWorkManager( + golem, activity_manager.do_work, plugins=task_plugins + ) + + async with golem, payment_manager, negotiation_manager, proposal_manager, activity_manager: + results: List[WorkResult] = await work_manager.do_work_list(task_list) + return results + + async def load_blend_file(context: WorkContext): batch = await context.create_batch() batch.deploy() @@ -55,6 +109,16 @@ async def main(): init_func=load_blend_file, threads=6, budget=1.0, + task_plugins=[retry(tries=3)], + scoring_plugins=[ + MapScore( + LinearAverageCostPricing( + average_cpu_load=0.2, average_duration=timedelta(seconds=5) + ), + normalize=True, + normalize_flip=True, + ), + ], ) print(f"\nBLENDER: all frames:{[result.result for result in results]}\n", flush=True) diff --git a/golem/utils/blender_api.py b/golem/utils/blender_api.py deleted file mode 100644 index b1f67523..00000000 --- a/golem/utils/blender_api.py +++ /dev/null @@ -1,46 +0,0 @@ -from typing import List - -from golem.managers.activity.pool import ActivityPoolManager -from golem.managers.agreement.single_use import SingleUseAgreementManager -from golem.managers.base import WorkResult -from golem.managers.negotiation import SequentialNegotiationManager -from golem.managers.negotiation.plugins import AddChosenPaymentPlatform -from golem.managers.payment.pay_all import PayAllPaymentManager -from golem.managers.proposal import StackProposalManager -from golem.managers.work.asynchronous import AsynchronousWorkManager -from golem.managers.work.plugins import retry -from golem.node import GolemNode - - -async def run_on_golem( - task_list, - payload, - init_func, - threads=6, - budget=1.0, - market_plugins=[ - AddChosenPaymentPlatform(), - ], - execution_plugins=[retry(tries=3)], -): - golem = GolemNode() - - payment_manager = PayAllPaymentManager(golem, budget=budget) - negotiation_manager = SequentialNegotiationManager( - golem, - payment_manager.get_allocation, - payload, - plugins=market_plugins, - ) - proposal_manager = StackProposalManager(golem, negotiation_manager.get_proposal) - agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) - activity_manager = ActivityPoolManager( - golem, agreement_manager.get_agreement, size=threads, on_activity_start=init_func - ) - work_manager = AsynchronousWorkManager( - golem, activity_manager.do_work, plugins=execution_plugins - ) - - async with golem, payment_manager, negotiation_manager, proposal_manager, activity_manager: - results: List[WorkResult] = await work_manager.do_work_list(task_list) - return results diff --git a/golem/utils/logging.py b/golem/utils/logging.py index 32a10226..4b680adf 100644 --- a/golem/utils/logging.py +++ b/golem/utils/logging.py @@ -39,7 +39,7 @@ "level": "INFO", }, "golem.managers.proposal": { - "level": "DEBUG", + "level": "INFO", }, "golem.managers.work": { "level": "INFO", From 70ba34ff2992e1d6679760288fb1e95dd279b656 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 12 Jul 2023 12:14:03 +0200 Subject: [PATCH 073/123] Add DemandManager to basic composition example --- examples/managers/basic_composition.py | 13 +- examples/managers/blender/blender.py | 25 ++- examples/managers/ssh.py | 6 +- golem/managers/base.py | 16 +- golem/managers/demand/__init__.py | 0 golem/managers/demand/auto.py | 197 +++++++++++++++++++++++ golem/managers/negotiation/sequential.py | 70 +++----- golem/managers/proposal/plugins.py | 10 +- golem/managers/proposal/scored_aot.py | 16 +- golem/resources/proposal/proposal.py | 2 +- 10 files changed, 267 insertions(+), 88 deletions(-) create mode 100644 golem/managers/demand/__init__.py create mode 100644 golem/managers/demand/auto.py diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index 1288cf1c..efb0807a 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -7,6 +7,7 @@ from golem.managers.activity.single_use import SingleUseActivityManager from golem.managers.agreement.single_use import SingleUseAgreementManager from golem.managers.base import RejectProposal, WorkContext, WorkResult +from golem.managers.demand.auto import AutoDemandManager from golem.managers.negotiation import SequentialNegotiationManager from golem.managers.negotiation.plugins import ( AddChosenPaymentPlatform, @@ -78,10 +79,14 @@ async def main(): ) payment_manager = PayAllPaymentManager(golem, budget=1.0) - negotiation_manager = SequentialNegotiationManager( + demand_manager = AutoDemandManager( golem, payment_manager.get_allocation, payload, + ) + negotiation_manager = SequentialNegotiationManager( + golem, + demand_manager.get_initial_proposal, plugins=[ AddChosenPaymentPlatform(), # class based plugin @@ -99,7 +104,7 @@ async def main(): ) proposal_manager = ScoredAheadOfTimeProposalManager( golem, - negotiation_manager.get_proposal, + negotiation_manager.get_draft_proposal, plugins=[ MapScore(linear_average_cost, normalize=True, normalize_flip=True), [0.5, PropertyValueLerpScore(INF_MEM, zero_at=1, one_at=8)], @@ -108,7 +113,7 @@ async def main(): [0.0, MapScore(lambda proposal_data: random())], ], ) - agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) + agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_draft_proposal) activity_manager = SingleUseActivityManager(golem, agreement_manager.get_agreement) work_manager = SequentialWorkManager( golem, @@ -119,7 +124,7 @@ async def main(): ) async with golem: - async with payment_manager, negotiation_manager, proposal_manager: + async with payment_manager, demand_manager, negotiation_manager, proposal_manager: results: List[WorkResult] = await work_manager.do_work_list(work_list) print(f"\nWORK MANAGER RESULTS:{[result.result for result in results]}\n", flush=True) diff --git a/examples/managers/blender/blender.py b/examples/managers/blender/blender.py index 03efb032..2a3cd7ba 100644 --- a/examples/managers/blender/blender.py +++ b/examples/managers/blender/blender.py @@ -3,27 +3,22 @@ import logging.config from datetime import timedelta from pathlib import Path - -from golem.managers.base import WorkContext -from golem.managers.proposal.plugins import MapScore -from golem.managers.proposal.pricings import LinearAverageCostPricing -from golem.managers.work.plugins import retry -from golem.payload import RepositoryVmPayload -from golem.utils.blender_api import run_on_golem -from golem.utils.logging import DEFAULT_LOGGING - - from typing import List from golem.managers.activity.pool import ActivityPoolManager from golem.managers.agreement.single_use import SingleUseAgreementManager -from golem.managers.base import WorkResult +from golem.managers.base import WorkContext, WorkResult from golem.managers.negotiation import SequentialNegotiationManager from golem.managers.negotiation.plugins import AddChosenPaymentPlatform from golem.managers.payment.pay_all import PayAllPaymentManager from golem.managers.proposal import ScoredAheadOfTimeProposalManager +from golem.managers.proposal.plugins import MapScore +from golem.managers.proposal.pricings import LinearAverageCostPricing from golem.managers.work.asynchronous import AsynchronousWorkManager +from golem.managers.work.plugins import retry from golem.node import GolemNode +from golem.payload import RepositoryVmPayload +from golem.utils.logging import DEFAULT_LOGGING BLENDER_IMAGE_HASH = "9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae" FRAME_CONFIG_TEMPLATE = json.loads(Path(__file__).with_name("frame_params.json").read_text()) @@ -52,15 +47,13 @@ async def run_on_golem( plugins=market_plugins, ) proposal_manager = ScoredAheadOfTimeProposalManager( - golem, negotiation_manager.get_proposal, plugins=scoring_plugins + golem, negotiation_manager.get_draft_proposal, plugins=scoring_plugins ) - agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) + agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_draft_proposal) activity_manager = ActivityPoolManager( golem, agreement_manager.get_agreement, size=threads, on_activity_start=init_func ) - work_manager = AsynchronousWorkManager( - golem, activity_manager.do_work, plugins=task_plugins - ) + work_manager = AsynchronousWorkManager(golem, activity_manager.do_work, plugins=task_plugins) async with golem, payment_manager, negotiation_manager, proposal_manager, activity_manager: results: List[WorkResult] = await work_manager.do_work_list(task_list) diff --git a/examples/managers/ssh.py b/examples/managers/ssh.py index ca2d093b..56c54b92 100644 --- a/examples/managers/ssh.py +++ b/examples/managers/ssh.py @@ -74,8 +74,10 @@ async def main(): negotiation_manager = SequentialNegotiationManager( golem, payment_manager.get_allocation, payload ) - proposal_manager = ScoredAheadOfTimeProposalManager(golem, negotiation_manager.get_proposal) - agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_proposal) + proposal_manager = ScoredAheadOfTimeProposalManager( + golem, negotiation_manager.get_draft_proposal + ) + agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_draft_proposal) activity_manager = SingleUseActivityManager( golem, agreement_manager.get_agreement, diff --git a/golem/managers/base.py b/golem/managers/base.py index 909f1daf..7f7f3700 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -162,15 +162,21 @@ async def get_allocation(self) -> Allocation: ... +class DemandManager(Manager, ABC): + @abstractmethod + async def get_initial_proposal(self) -> Proposal: + ... + + class NegotiationManager(Manager, ABC): @abstractmethod - async def get_proposal(self) -> Proposal: + async def get_draft_proposal(self) -> Proposal: ... class ProposalManager(Manager, ABC): @abstractmethod - async def get_proposal(self) -> Proposal: + async def get_draft_proposal(self) -> Proposal: ... @@ -205,7 +211,7 @@ def __call__( ProposalPluginResult = Sequence[Optional[float]] -class ProposalManagerPlugin(ABC): +class ManagerScorePlugin(ABC): @abstractmethod def __call__( self, proposals_data: Sequence[ProposalData] @@ -213,9 +219,7 @@ def __call__( ... -ProposalManagerPluginWithOptionalWeight = Union[ - ProposalManagerPlugin, Tuple[float, ProposalManagerPlugin] -] +ManagerPluginWithOptionalWeight = Union[ManagerScorePlugin, Tuple[float, ManagerScorePlugin]] class WorkManagerPlugin(ABC): diff --git a/golem/managers/demand/__init__.py b/golem/managers/demand/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/golem/managers/demand/auto.py b/golem/managers/demand/auto.py new file mode 100644 index 00000000..420a2187 --- /dev/null +++ b/golem/managers/demand/auto.py @@ -0,0 +1,197 @@ +import asyncio +import inspect +import logging +from datetime import datetime +from typing import Awaitable, Callable, List, Optional, Sequence, Tuple, cast + +from golem.managers.base import ( + DemandManager, + ManagerException, + ManagerPluginsMixin, + ManagerPluginWithOptionalWeight, +) +from golem.node import GolemNode +from golem.node.node import GolemNode +from golem.payload import Payload, Properties +from golem.payload.parsers.textx.parser import TextXPayloadSyntaxParser +from golem.resources import Allocation, Proposal, ProposalData +from golem.resources.demand.demand_builder import DemandBuilder +from golem.resources.proposal.proposal import Proposal +from golem.utils.asyncio import create_task_with_logging + +logger = logging.getLogger(__name__) + + +class AutoDemandManager(ManagerPluginsMixin[ManagerPluginWithOptionalWeight], DemandManager): + def __init__( + self, + golem: GolemNode, + get_allocation: Callable[[], Awaitable[Allocation]], + payload: Payload, + *args, + **kwargs, + ) -> None: + self._golem = golem + self._get_allocation = get_allocation + self._payload = payload + + self._demand_offer_parser = TextXPayloadSyntaxParser() + + self._scored_proposals: List[Tuple[float, Proposal]] = [] + self._scored_proposals_condition = asyncio.Condition() + + self._manager_loop_task: Optional[asyncio.Task] = None + + super().__init__(*args, **kwargs) + + async def start(self) -> None: + if self.is_started(): + message = "Already started!" + logger.debug(f"Starting failed with `{message}`") + raise ManagerException(message) + + self._manager_loop_task = create_task_with_logging(self._manager_loop()) + + async def stop(self) -> None: + if not self.is_started(): + message = "Already stopped!" + logger.debug(f"Stopping failed with `{message}`") + raise ManagerException(message) + + self._manager_loop_task.cancel() + self._manager_loop_task = None + + def is_started(self) -> bool: + return self._manager_loop_task is not None and not self._manager_loop_task.done() + + async def get_initial_proposal(self) -> Proposal: + async with self._scored_proposals_condition: + await self._scored_proposals_condition.wait_for(lambda: 0 < len(self._scored_proposals)) + + score, proposal = self._scored_proposals.pop(0) + + return proposal + + async def _manager_loop(self) -> None: + allocation = await self._get_allocation() + demand_builder = await self._prepare_demand_builder(allocation) + + demand = await demand_builder.create_demand(self._golem) + demand.start_collecting_events() + + try: + async for initial_proposal in demand.initial_proposals(): + await self._manage_initial(initial_proposal) + finally: + await demand.unsubscribe() + + async def _prepare_demand_builder(self, allocation: Allocation) -> DemandBuilder: + # FIXME: Code looks duplicated as GolemNode.create_demand does the same + demand_builder = DemandBuilder() + + await demand_builder.add_default_parameters( + self._demand_offer_parser, allocations=[allocation] + ) + + await demand_builder.add(self._payload) + + return demand_builder + + async def _manage_initial(self, proposal: Proposal) -> None: + async with self._scored_proposals_condition: + all_proposals = list(sp[1] for sp in self._scored_proposals) + all_proposals.append(proposal) + + self._scored_proposals = await self._do_scoring(all_proposals) + + self._scored_proposals_condition.notify_all() + + async def _do_scoring(self, proposals: Sequence[Proposal]): + proposals_data = await self._get_proposals_data_from_proposals(proposals) + proposal_scores = await self._run_plugins(proposals_data) + + scored_proposals = self._calculate_proposal_score(proposals, proposal_scores) + scored_proposals.sort(key=lambda x: x[0], reverse=True) + + return scored_proposals + + async def _run_plugins( + self, proposals_data: Sequence[ProposalData] + ) -> Sequence[Tuple[float, Sequence[float]]]: + proposal_scores = [] + + for plugin in self._plugins: + if isinstance(plugin, (list, tuple)): + weight, plugin = plugin + else: + weight = 1 + + plugin_scores = plugin(proposals_data) + + if inspect.isawaitable(plugin_scores): + plugin_scores = await plugin_scores + + proposal_scores.append((weight, plugin_scores)) + + return proposal_scores + + def _calculate_proposal_score( + self, + proposals: Sequence[Proposal], + plugin_scores: Sequence[Tuple[float, Sequence[float]]], + ) -> List[Tuple[float, Proposal]]: + # FIXME: can this be refactored? + return [ + ( + self._calculate_weighted_score( + self._transpose_plugin_scores(proposal_index, plugin_scores) + ), + proposal, + ) + for proposal_index, proposal in enumerate(proposals) + ] + + def _calculate_weighted_score( + self, proposal_weighted_scores: Sequence[Tuple[float, float]] + ) -> float: + if not proposal_weighted_scores: + return 0 + + weighted_sum = sum(pws[0] * pws[1] for pws in proposal_weighted_scores) + weights_sum = sum(pws[0] for pws in proposal_weighted_scores) + + return weighted_sum / weights_sum + + def _transpose_plugin_scores( + self, proposal_index: int, plugin_scores: Sequence[Tuple[float, Sequence[float]]] + ) -> Sequence[Tuple[float, float]]: + # FIXME: can this be refactored? + return [ + (plugin_weight, plugin_scores[proposal_index]) + for plugin_weight, plugin_scores in plugin_scores + if plugin_scores[proposal_index] is None + ] + + async def _get_proposals_data_from_proposals( + self, proposals: Sequence[Proposal] + ) -> Sequence[ProposalData]: + result = [] + + for proposal in proposals: + data = await proposal.get_data() + + constraints = self._demand_offer_parser.parse_constraints(data.constraints) + + result.append( + ProposalData( + properties=Properties(data.properties), + constraints=constraints, + proposal_id=data.proposal_id, + issuer_id=data.issuer_id, + state=data.state, + timestamp=cast(datetime, data.timestamp), + prev_proposal_id=data.prev_proposal_id, + ) + ) + + return result diff --git a/golem/managers/negotiation/sequential.py b/golem/managers/negotiation/sequential.py index 5cc6c3e4..d6f22090 100644 --- a/golem/managers/negotiation/sequential.py +++ b/golem/managers/negotiation/sequential.py @@ -14,9 +14,9 @@ RejectProposal, ) from golem.node import GolemNode -from golem.payload import Payload, Properties +from golem.payload import Properties from golem.payload.parsers.textx import TextXPayloadSyntaxParser -from golem.resources import Allocation, Demand, DemandBuilder, DemandData, Proposal, ProposalData +from golem.resources import Allocation, DemandData, Proposal, ProposalData from golem.utils.asyncio import create_task_with_logging logger = logging.getLogger(__name__) @@ -25,17 +25,16 @@ class SequentialNegotiationManager( ManagerPluginsMixin[NegotiationManagerPlugin], NegotiationManager ): + # TODO remove unused methods def __init__( self, golem: GolemNode, - get_allocation: Callable[[], Awaitable[Allocation]], - payload: Payload, + get_initial_proposal: Callable[[], Awaitable[Allocation]], *args, **kwargs, ) -> None: self._golem = golem - self._get_allocation = get_allocation - self._payload = payload + self._get_initial_proposal = get_initial_proposal self._negotiation_loop_task: Optional[asyncio.Task] = None self._eligible_proposals: asyncio.Queue[Proposal] = asyncio.Queue() @@ -43,7 +42,7 @@ def __init__( super().__init__(*args, **kwargs) - async def get_proposal(self) -> Proposal: + async def get_draft_proposal(self) -> Proposal: logger.debug("Getting proposal...") proposal = await self._eligible_proposals.get() @@ -81,49 +80,24 @@ def is_started(self) -> bool: return self._negotiation_loop_task is not None and not self._negotiation_loop_task.done() async def _negotiation_loop(self) -> None: - allocation = await self._get_allocation() - demand_builder = await self._prepare_demand_builder(allocation) + while True: # TODO add buffer + proposal = await self._get_initial_proposal() + offer_proposal = await self._negotiate(proposal) + if offer_proposal is not None: + await self._eligible_proposals.put(offer_proposal) - demand = await demand_builder.create_demand(self._golem) - demand.start_collecting_events() + async def _negotiate(self, initial_proposal: Proposal) -> AsyncIterator[Proposal]: + demand_data = await self._get_demand_data_from_proposal(initial_proposal) - logger.debug("Demand published, waiting for proposals...") + offer_proposal = await self._negotiate_proposal(demand_data, initial_proposal) - try: - async for proposal in self._negotiate(demand): - await self._eligible_proposals.put(proposal) - finally: - await demand.unsubscribe() + if offer_proposal is None: + logger.debug( + f"Negotiating proposal `{initial_proposal}` done and proposal was rejected" + ) + return - async def _prepare_demand_builder(self, allocation: Allocation) -> DemandBuilder: - logger.debug("Preparing demand...") - - # FIXME: Code looks duplicated as GolemNode.create_demand does the same - demand_builder = DemandBuilder() - - await demand_builder.add_default_parameters( - self._demand_offer_parser, allocations=[allocation] - ) - - await demand_builder.add(self._payload) - - logger.debug("Preparing demand done") - - return demand_builder - - async def _negotiate(self, demand: Demand) -> AsyncIterator[Proposal]: - demand_data = await self._get_demand_data_from_demand(demand) - - async for initial_proposal in demand.initial_proposals(): - offer_proposal = await self._negotiate_proposal(demand_data, initial_proposal) - - if offer_proposal is None: - logger.debug( - f"Negotiating proposal `{initial_proposal}` done and proposal was rejected" - ) - continue - - yield offer_proposal + return offer_proposal async def _negotiate_proposal( self, demand_data: DemandData, offer_proposal: Proposal @@ -201,10 +175,10 @@ async def _negotiate_proposal( return offer_proposal - async def _get_demand_data_from_demand(self, demand: Demand) -> DemandData: + async def _get_demand_data_from_proposal(self, proposal: Proposal) -> DemandData: # FIXME: Unnecessary serialisation from DemandBuilder to Demand, # and from Demand to ProposalData - data = await demand.get_data() + data = await proposal.demand.get_data() constraints = self._demand_offer_parser.parse_constraints(data.constraints) diff --git a/golem/managers/proposal/plugins.py b/golem/managers/proposal/plugins.py index 9592794c..11eed61c 100644 --- a/golem/managers/proposal/plugins.py +++ b/golem/managers/proposal/plugins.py @@ -1,7 +1,7 @@ from random import random from typing import Callable, Optional, Sequence, Tuple, Type, Union -from golem.managers.base import ManagerPluginException, ProposalManagerPlugin, ProposalPluginResult +from golem.managers.base import ManagerPluginException, ManagerScorePlugin, ProposalPluginResult from golem.payload.constraints import PropertyName from golem.resources import ProposalData @@ -9,7 +9,7 @@ BoundaryValues = Tuple[Tuple[float, PropertyValueNumeric], Tuple[float, PropertyValueNumeric]] -class PropertyValueLerpScore(ProposalManagerPlugin): +class PropertyValueLerpScore(ManagerScorePlugin): def __init__( self, property_name: PropertyName, @@ -53,7 +53,7 @@ def _get_property_value(self, proposal_data: ProposalData) -> Optional[PropertyV return None - if not isinstance(property_value, PropertyValueNumeric): + if not isinstance(property_value, (int, float)): if self._raise_on_bad_value: raise ManagerPluginException( f"Field `{self._property_name}` value type must be an `int` or `float`!" @@ -90,12 +90,12 @@ def _get_boundary_values( return bounds_min, bounds_max -class RandomScore(ProposalManagerPlugin): +class RandomScore(ManagerScorePlugin): def __call__(self, proposals_data: Sequence[ProposalData]) -> ProposalPluginResult: return [random() for _ in range(len(proposals_data))] -class MapScore(ProposalManagerPlugin): +class MapScore(ManagerScorePlugin): def __init__( self, callback: Callable[[ProposalData], Optional[float]], diff --git a/golem/managers/proposal/scored_aot.py b/golem/managers/proposal/scored_aot.py index 80136402..edf264f0 100644 --- a/golem/managers/proposal/scored_aot.py +++ b/golem/managers/proposal/scored_aot.py @@ -7,8 +7,8 @@ from golem.managers.base import ( ManagerException, ManagerPluginsMixin, + ManagerPluginWithOptionalWeight, ProposalManager, - ProposalManagerPluginWithOptionalWeight, ) from golem.node import GolemNode from golem.payload import Properties @@ -24,12 +24,16 @@ class IgnoreProposal(Exception): class ScoredAheadOfTimeProposalManager( - ManagerPluginsMixin[ProposalManagerPluginWithOptionalWeight], ProposalManager + ManagerPluginsMixin[ManagerPluginWithOptionalWeight], ProposalManager ): def __init__( - self, golem: GolemNode, get_proposal: Callable[[], Awaitable[Proposal]], *args, **kwargs + self, + golem: GolemNode, + get_init_proposal: Callable[[], Awaitable[Proposal]], + *args, + **kwargs, ) -> None: - self._get_proposal = get_proposal + self._get_init_proposal = get_init_proposal self._consume_proposals_task: Optional[asyncio.Task] = None self._demand_offer_parser = TextXPayloadSyntaxParser() @@ -68,7 +72,7 @@ def is_started(self) -> bool: async def _consume_proposals(self) -> None: while True: - proposal = await self._get_proposal() + proposal = await self._get_init_proposal() logger.debug(f"Adding proposal `{proposal}` on the scoring...") @@ -82,7 +86,7 @@ async def _consume_proposals(self) -> None: logger.debug(f"Adding proposal `{proposal}` on the scoring done") - async def get_proposal(self) -> Proposal: + async def get_draft_proposal(self) -> Proposal: logger.debug("Getting proposal...") async with self._scored_proposals_condition: diff --git a/golem/resources/proposal/proposal.py b/golem/resources/proposal/proposal.py index 4295ad22..7f8929a8 100644 --- a/golem/resources/proposal/proposal.py +++ b/golem/resources/proposal/proposal.py @@ -98,7 +98,7 @@ def demand(self) -> "Demand": if isinstance(self.parent, Demand): return self.parent else: - return self.parent.demand + return self.parent.demand # TODO recursion @demand.setter def demand(self, demand: "Demand") -> None: From c8a05b9bdfccac2292ec1eaa5cd802b1b78f9edc Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 12 Jul 2023 13:17:01 +0200 Subject: [PATCH 074/123] Initial span tracing --- golem/event_bus/in_memory/event_bus.py | 1 + golem/managers/negotiation/sequential.py | 74 +++++++++--------------- golem/resources/proposal/proposal.py | 2 + golem/utils/logging.py | 63 +++++++++++++++++++- 4 files changed, 93 insertions(+), 47 deletions(-) diff --git a/golem/event_bus/in_memory/event_bus.py b/golem/event_bus/in_memory/event_bus.py index fd527346..912743af 100644 --- a/golem/event_bus/in_memory/event_bus.py +++ b/golem/event_bus/in_memory/event_bus.py @@ -191,6 +191,7 @@ async def _process_event( logger.debug(f"Calling {callback_info.callback}...") try: + # TODO: Support sync callbacks await callback_info.callback(event) except Exception as e: logger.debug(f"Calling {callback_info.callback} failed with `{e}") diff --git a/golem/managers/negotiation/sequential.py b/golem/managers/negotiation/sequential.py index d6f22090..e06c511f 100644 --- a/golem/managers/negotiation/sequential.py +++ b/golem/managers/negotiation/sequential.py @@ -18,6 +18,7 @@ from golem.payload.parsers.textx import TextXPayloadSyntaxParser from golem.resources import Allocation, DemandData, Proposal, ProposalData from golem.utils.asyncio import create_task_with_logging +from golem.utils.logging import trace_span logger = logging.getLogger(__name__) @@ -42,40 +43,25 @@ def __init__( super().__init__(*args, **kwargs) + @trace_span("Getting proposal") async def get_draft_proposal(self) -> Proposal: - logger.debug("Getting proposal...") - - proposal = await self._eligible_proposals.get() - - logger.debug(f"Getting proposal done with `{proposal}`") - - return proposal + return await self._eligible_proposals.get() + @trace_span('Starting') async def start(self) -> None: - logger.debug("Starting...") - if self.is_started(): - message = "Already started!" - logger.debug(f"Starting failed with `{message}`") - raise ManagerException(message) + raise ManagerException("Already started!") self._negotiation_loop_task = create_task_with_logging(self._negotiation_loop()) - logger.debug("Starting done") - + @trace_span('Stopping') async def stop(self) -> None: - logger.debug("Stopping...") - if not self.is_started(): - message = "Already stopped!" - logger.debug(f"Stopping failed with `{message}`") - raise ManagerException(message) + raise ManagerException("Already stopped!") self._negotiation_loop_task.cancel() self._negotiation_loop_task = None - logger.debug("Stopping done") - def is_started(self) -> bool: return self._negotiation_loop_task is not None and not self._negotiation_loop_task.done() @@ -99,42 +85,22 @@ async def _negotiate(self, initial_proposal: Proposal) -> AsyncIterator[Proposal return offer_proposal + @trace_span("Negotiating proposal") async def _negotiate_proposal( self, demand_data: DemandData, offer_proposal: Proposal ) -> Optional[Proposal]: - logger.debug(f"Negotiating proposal `{offer_proposal}`...") - while True: demand_data_after_plugins = deepcopy(demand_data) - proposal_data = await self._get_proposal_data_from_proposal(offer_proposal) try: - logger.debug(f"Applying plugins on `{offer_proposal}`...") - - for plugin in self._plugins: - plugin_result = plugin(demand_data_after_plugins, proposal_data) - - if asyncio.iscoroutine(plugin_result): - plugin_result = await plugin_result - - if isinstance(plugin_result, RejectProposal): - raise plugin_result - - # Note: Explicit identity to False desired here, not "falsy" check - if plugin_result is False: - raise RejectProposal() - + await self._apply_plugins(demand_data_after_plugins, offer_proposal) except RejectProposal as e: - logger.debug( - f"Applying plugins on `{offer_proposal}` done and proposal was rejected" - ) + logger.debug(f"Proposal `{offer_proposal}` was rejected by plugins") if not offer_proposal.initial: await offer_proposal.reject(str(e)) return None - else: - logger.debug(f"Applying plugins on `{offer_proposal}` done") if offer_proposal.initial or demand_data_after_plugins != demand_data: logger.debug("Sending demand proposal...") @@ -171,15 +137,31 @@ async def _negotiate_proposal( else: break - logger.debug(f"Negotiating proposal `{offer_proposal}` done") - return offer_proposal + @trace_span() + async def _apply_plugins(self, demand_data_after_plugins: DemandData, offer_proposal: Proposal): + proposal_data = await self._get_proposal_data_from_proposal(offer_proposal) + + for plugin in self._plugins: + plugin_result = plugin(demand_data_after_plugins, proposal_data) + + if asyncio.iscoroutine(plugin_result): + plugin_result = await plugin_result + + if isinstance(plugin_result, RejectProposal): + raise plugin_result + + # Note: Explicit identity to False desired here, not "falsy" check + if plugin_result is False: + raise RejectProposal() + async def _get_demand_data_from_proposal(self, proposal: Proposal) -> DemandData: # FIXME: Unnecessary serialisation from DemandBuilder to Demand, # and from Demand to ProposalData data = await proposal.demand.get_data() + # TODO: Make constraints parsing lazy constraints = self._demand_offer_parser.parse_constraints(data.constraints) return DemandData( diff --git a/golem/resources/proposal/proposal.py b/golem/resources/proposal/proposal.py index 7f8929a8..f8de0209 100644 --- a/golem/resources/proposal/proposal.py +++ b/golem/resources/proposal/proposal.py @@ -17,6 +17,8 @@ ProposalId: Type[str] = str + +# TODO: Use Enum ProposalState = Literal["Initial", "Draft", "Rejected", "Accepted", "Expired"] diff --git a/golem/utils/logging.py b/golem/utils/logging.py index 4b680adf..0c635c70 100644 --- a/golem/utils/logging.py +++ b/golem/utils/logging.py @@ -1,10 +1,14 @@ +import inspect import logging from datetime import datetime, timezone +from functools import wraps from typing import TYPE_CHECKING, Optional + if TYPE_CHECKING: from golem.event_bus import Event + DEFAULT_LOGGING = { "version": 1, "disable_existing_loggers": False, @@ -50,7 +54,7 @@ }, } - +logger = logging.getLogger() class _YagnaDatetimeFormatter(logging.Formatter): """Custom log Formatter that formats datetime using the same convention yagna uses.""" @@ -111,3 +115,60 @@ def _prepare_logger(self) -> logging.Logger: async def on_event(self, event: "Event") -> None: """Handle event produced by :any:`EventBus.on`.""" self.logger.info(event) + + +def trace_span(name: Optional[str] = None, show_arguments: bool = False, show_results: bool = True): + def wrapper(f): + span_name = name if name is not None else f.__name__ + + @wraps(f) + def sync_wrapped(*args, **kwargs): + if show_arguments: + args_str = ', '.join(repr(a) for a in args) + kwargs_str = ', '.join('{}={}'.format(k, repr(v)) for (k, v) in kwargs.items()) + final_name = f'{span_name}({args_str}, {kwargs_str})' + else: + final_name = span_name + + logger.debug(f"{final_name}...") + + try: + result = f(*args, **kwargs) + except Exception as e: + logger.debug(f"{final_name} failed with `{e}`") + raise + + if show_results: + logger.debug(f"{final_name} done with `{result}`") + else: + logger.debug(f"{final_name} done") + + return result + + @wraps(f) + async def async_wrapped(*args, **kwargs): + if show_arguments: + args_str = ', '.join(repr(a) for a in args) + kwargs_str = ', '.join('{}={}'.format(k, repr(v)) for (k, v) in kwargs.items()) + final_name = f'{span_name}({args_str}, {kwargs_str})' + else: + final_name = span_name + + logger.debug(f"{final_name}...") + + try: + result = await f(*args, **kwargs) + except Exception as e: + logger.debug(f"{final_name} failed with `{e}`") + raise + + if show_results: + logger.debug(f"{final_name} done with `{result}`") + else: + logger.debug(f"{final_name} done") + + return result + + return async_wrapped if inspect.iscoroutinefunction(f) else sync_wrapped + + return wrapper From f726bedda6ce7f62fee5c638e98d8008c9ea00e0 Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 12 Jul 2023 15:33:14 +0200 Subject: [PATCH 075/123] Initial span tracing in selected managers --- golem/managers/demand/auto.py | 18 +++-- golem/managers/negotiation/sequential.py | 89 ++++++++++------------ golem/managers/payment/pay_all.py | 97 ++++++++++++------------ golem/managers/proposal/scored_aot.py | 43 ++++------- golem/utils/logging.py | 38 +++++++--- 5 files changed, 140 insertions(+), 145 deletions(-) diff --git a/golem/managers/demand/auto.py b/golem/managers/demand/auto.py index 420a2187..cb3a6059 100644 --- a/golem/managers/demand/auto.py +++ b/golem/managers/demand/auto.py @@ -18,6 +18,7 @@ from golem.resources.demand.demand_builder import DemandBuilder from golem.resources.proposal.proposal import Proposal from golem.utils.asyncio import create_task_with_logging +from golem.utils.logging import trace_span logger = logging.getLogger(__name__) @@ -44,19 +45,17 @@ def __init__( super().__init__(*args, **kwargs) + @trace_span() async def start(self) -> None: if self.is_started(): - message = "Already started!" - logger.debug(f"Starting failed with `{message}`") - raise ManagerException(message) + raise ManagerException("Already started!") self._manager_loop_task = create_task_with_logging(self._manager_loop()) + @trace_span() async def stop(self) -> None: if not self.is_started(): - message = "Already stopped!" - logger.debug(f"Stopping failed with `{message}`") - raise ManagerException(message) + raise ManagerException("Already stopped!") self._manager_loop_task.cancel() self._manager_loop_task = None @@ -72,6 +71,7 @@ async def get_initial_proposal(self) -> Proposal: return proposal + @trace_span() async def _manager_loop(self) -> None: allocation = await self._get_allocation() demand_builder = await self._prepare_demand_builder(allocation) @@ -81,7 +81,7 @@ async def _manager_loop(self) -> None: try: async for initial_proposal in demand.initial_proposals(): - await self._manage_initial(initial_proposal) + await self._manage_scoring(initial_proposal) finally: await demand.unsubscribe() @@ -97,7 +97,8 @@ async def _prepare_demand_builder(self, allocation: Allocation) -> DemandBuilder return demand_builder - async def _manage_initial(self, proposal: Proposal) -> None: + @trace_span() + async def _manage_scoring(self, proposal: Proposal) -> None: async with self._scored_proposals_condition: all_proposals = list(sp[1] for sp in self._scored_proposals) all_proposals.append(proposal) @@ -115,6 +116,7 @@ async def _do_scoring(self, proposals: Sequence[Proposal]): return scored_proposals + @trace_span() async def _run_plugins( self, proposals_data: Sequence[ProposalData] ) -> Sequence[Tuple[float, Sequence[float]]]: diff --git a/golem/managers/negotiation/sequential.py b/golem/managers/negotiation/sequential.py index e06c511f..ff063304 100644 --- a/golem/managers/negotiation/sequential.py +++ b/golem/managers/negotiation/sequential.py @@ -2,7 +2,7 @@ import logging from copy import deepcopy from datetime import datetime -from typing import AsyncIterator, Awaitable, Callable, Optional, cast +from typing import Awaitable, Callable, Optional, cast from ya_market import ApiException @@ -16,7 +16,7 @@ from golem.node import GolemNode from golem.payload import Properties from golem.payload.parsers.textx import TextXPayloadSyntaxParser -from golem.resources import Allocation, DemandData, Proposal, ProposalData +from golem.resources import DemandData, Proposal, ProposalData from golem.utils.asyncio import create_task_with_logging from golem.utils.logging import trace_span @@ -30,7 +30,7 @@ class SequentialNegotiationManager( def __init__( self, golem: GolemNode, - get_initial_proposal: Callable[[], Awaitable[Allocation]], + get_initial_proposal: Callable[[], Awaitable[Proposal]], *args, **kwargs, ) -> None: @@ -43,18 +43,18 @@ def __init__( super().__init__(*args, **kwargs) - @trace_span("Getting proposal") + @trace_span() async def get_draft_proposal(self) -> Proposal: return await self._eligible_proposals.get() - @trace_span('Starting') + @trace_span() async def start(self) -> None: if self.is_started(): raise ManagerException("Already started!") self._negotiation_loop_task = create_task_with_logging(self._negotiation_loop()) - @trace_span('Stopping') + @trace_span() async def stop(self) -> None: if not self.is_started(): raise ManagerException("Already stopped!") @@ -65,27 +65,19 @@ async def stop(self) -> None: def is_started(self) -> bool: return self._negotiation_loop_task is not None and not self._negotiation_loop_task.done() + @trace_span() async def _negotiation_loop(self) -> None: while True: # TODO add buffer proposal = await self._get_initial_proposal() - offer_proposal = await self._negotiate(proposal) - if offer_proposal is not None: - await self._eligible_proposals.put(offer_proposal) - async def _negotiate(self, initial_proposal: Proposal) -> AsyncIterator[Proposal]: - demand_data = await self._get_demand_data_from_proposal(initial_proposal) + demand_data = await self._get_demand_data_from_proposal(proposal) - offer_proposal = await self._negotiate_proposal(demand_data, initial_proposal) + offer_proposal = await self._negotiate_proposal(demand_data, proposal) - if offer_proposal is None: - logger.debug( - f"Negotiating proposal `{initial_proposal}` done and proposal was rejected" - ) - return - - return offer_proposal + if offer_proposal is not None: + await self._eligible_proposals.put(offer_proposal) - @trace_span("Negotiating proposal") + @trace_span() async def _negotiate_proposal( self, demand_data: DemandData, offer_proposal: Proposal ) -> Optional[Proposal]: @@ -102,42 +94,27 @@ async def _negotiate_proposal( return None - if offer_proposal.initial or demand_data_after_plugins != demand_data: - logger.debug("Sending demand proposal...") - - demand_data = demand_data_after_plugins + if not offer_proposal.initial and demand_data_after_plugins == demand_data: + return offer_proposal - try: - demand_proposal = await offer_proposal.respond( - demand_data_after_plugins.properties, - demand_data_after_plugins.constraints, - ) - except (ApiException, asyncio.TimeoutError) as e: - logger.debug(f"Sending demand proposal failed with `{e}`") - return None + demand_data = demand_data_after_plugins - logger.debug("Sending demand proposal done") - - logger.debug("Waiting for response...") - - try: - new_offer_proposal = await demand_proposal.responses().__anext__() - except StopAsyncIteration: - logger.debug("Waiting for response failed with provider rejection") - return None + try: + demand_proposal = await self._send_demand_proposal(offer_proposal, demand_data) + except (ApiException, asyncio.TimeoutError): + return None - logger.debug(f"Waiting for response done with `{new_offer_proposal}`") + try: + new_offer_proposal = await self._wait_for_proposal_response(demand_proposal) + except StopAsyncIteration: + return None - logger.debug( - f"Proposal `{offer_proposal}` received counter proposal `{new_offer_proposal}`" - ) - offer_proposal = new_offer_proposal + logger.debug( + f"Proposal `{offer_proposal}` received counter proposal `{new_offer_proposal}`" + ) - continue - else: - break + offer_proposal = new_offer_proposal - return offer_proposal @trace_span() async def _apply_plugins(self, demand_data_after_plugins: DemandData, offer_proposal: Proposal): proposal_data = await self._get_proposal_data_from_proposal(offer_proposal) @@ -155,6 +132,18 @@ async def _apply_plugins(self, demand_data_after_plugins: DemandData, offer_prop if plugin_result is False: raise RejectProposal() + @trace_span() + async def _send_demand_proposal( + self, offer_proposal: Proposal, demand_data: DemandData + ) -> Proposal: + return await offer_proposal.respond( + demand_data.properties, + demand_data.constraints, + ) + + @trace_span() + async def _wait_for_proposal_response(self, demand_proposal: Proposal) -> Proposal: + return await demand_proposal.responses().__anext__() async def _get_demand_data_from_proposal(self, proposal: Proposal) -> DemandData: # FIXME: Unnecessary serialisation from DemandBuilder to Demand, diff --git a/golem/managers/payment/pay_all.py b/golem/managers/payment/pay_all.py index e3189f7b..37ca6820 100644 --- a/golem/managers/payment/pay_all.py +++ b/golem/managers/payment/pay_all.py @@ -14,6 +14,7 @@ NewDebitNote, NewInvoice, ) +from golem.utils.logging import trace_span logger = logging.getLogger(__name__) @@ -37,47 +38,45 @@ def __init__( self._closed_agreements_count = 0 self._payed_invoices_count = 0 - async def start(self) -> None: - logger.debug("Starting...") + self._event_handlers = [] + @trace_span() + async def start(self) -> None: # TODO: Add stop with event_bus.off() - - await self._golem.event_bus.on(NewInvoice, self._pay_invoice_if_received) - await self._golem.event_bus.on(NewDebitNote, self._pay_debit_note_if_received) - - await self._golem.event_bus.on(NewAgreement, self._increment_opened_agreements) - await self._golem.event_bus.on(AgreementClosed, self._increment_closed_agreements) - - logger.debug("Starting done") - + self._event_handlers.extend( + [ + await self._golem.event_bus.on(NewInvoice, self._pay_invoice_if_received), + await self._golem.event_bus.on(NewDebitNote, self._pay_debit_note_if_received), + await self._golem.event_bus.on(NewAgreement, self._increment_opened_agreements), + await self._golem.event_bus.on(AgreementClosed, self._increment_closed_agreements), + ] + ) + + @trace_span() async def stop(self) -> None: - logger.debug("Stopping...") - await self.wait_for_invoices() - logger.debug("Stopping done") + for event_handler in self._event_handlers: + await self._golem.event_bus.off(event_handler) + + @trace_span() + async def _create_allocation(self) -> None: + self._allocation = await Allocation.create_any_account( + self._golem, Decimal(self._budget), self._network, self._driver + ) + + self._golem.add_autoclose_resource(self._allocation) + @trace_span() async def get_allocation(self) -> "Allocation": # TODO handle NoMatchingAccount - logger.debug("Getting allocation...") - if self._allocation is None: - logger.debug("Creating allocation...") - - self._allocation = await Allocation.create_any_account( - self._golem, Decimal(self._budget), self._network, self._driver - ) - self._golem.add_autoclose_resource(self._allocation) - - logger.debug(f"Creating allocation done with `{self._allocation.id}`") - - logger.debug(f"Getting allocation done with `{self._allocation.id}`") + await self._create_allocation() return self._allocation + @trace_span() async def wait_for_invoices(self): - logger.info("Waiting for invoices...") - for _ in range(60): await asyncio.sleep(1) if ( @@ -97,35 +96,35 @@ async def _increment_opened_agreements(self, event: NewAgreement): async def _increment_closed_agreements(self, event: AgreementClosed): self._closed_agreements_count += 1 - async def _pay_invoice_if_received(self, event: NewInvoice) -> None: - logger.debug("Received invoice") + @trace_span() + async def _accept_invoice(self, invoice: Invoice) -> None: + assert self._allocation is not None # TODO think of a better way + await invoice.accept_full(self._allocation) + await invoice.get_data(force=True) + self._payed_invoices_count += 1 + + logger.info(f"Invoice `{invoice.id}` accepted") + + @trace_span() + async def _accept_debit_note(self, debit_note: DebitNote) -> None: + assert self._allocation is not None # TODO think of a better way + await debit_note.accept_full(self._allocation) + await debit_note.get_data(force=True) + + logger.info(f"DebitNote `{debit_note.id}` accepted") + @trace_span() + async def _pay_invoice_if_received(self, event: NewInvoice) -> None: invoice = event.resource assert isinstance(invoice, Invoice) if (await invoice.get_data(force=True)).status == "RECEIVED": - logger.debug(f"Accepting invoice `{invoice.id}`...") - - assert self._allocation is not None # TODO think of a better way - await invoice.accept_full(self._allocation) - await invoice.get_data(force=True) - self._payed_invoices_count += 1 - - logger.debug(f"Accepting invoice `{invoice.id}` done") - logger.info(f"Invoice `{invoice.id}` accepted") + await self._accept_invoice(invoice) + @trace_span() async def _pay_debit_note_if_received(self, event: NewDebitNote) -> None: - logger.debug("Received debit note") - debit_note = event.resource assert isinstance(debit_note, DebitNote) if (await debit_note.get_data(force=True)).status == "RECEIVED": - logger.debug(f"Accepting DebitNote `{debit_note.id}`...") - - assert self._allocation is not None # TODO think of a better way - await debit_note.accept_full(self._allocation) - await debit_note.get_data(force=True) - - logger.debug(f"Accepting DebitNote `{debit_note.id}` done") - logger.debug(f"DebitNote `{debit_note.id}` accepted") + await self._accept_debit_note(debit_note) diff --git a/golem/managers/proposal/scored_aot.py b/golem/managers/proposal/scored_aot.py index edf264f0..e0cf92c5 100644 --- a/golem/managers/proposal/scored_aot.py +++ b/golem/managers/proposal/scored_aot.py @@ -15,6 +15,7 @@ from golem.payload.parsers.textx import TextXPayloadSyntaxParser from golem.resources import Proposal, ProposalData from golem.utils.asyncio import create_task_with_logging +from golem.utils.logging import trace_span logger = logging.getLogger(__name__) @@ -42,31 +43,21 @@ def __init__( super().__init__(*args, **kwargs) + @trace_span() async def start(self) -> None: - logger.debug("Starting...") - if self.is_started(): - message = "Already started!" - logger.debug(f"Starting failed with `{message}`") - raise ManagerException(message) + raise ManagerException("Already started!") self._consume_proposals_task = create_task_with_logging(self._consume_proposals()) - logger.debug("Starting done") - + @trace_span() async def stop(self) -> None: - logger.debug("Stopping...") - if not self.is_started(): - message = "Already stopped!" - logger.debug(f"Stopping failed with `{message}`") - raise ManagerException(message) + raise ManagerException("Already stopped!") self._consume_proposals_task.cancel() self._consume_proposals_task = None - logger.debug("Stopping done") - def is_started(self) -> bool: return self._consume_proposals_task is not None and not self._consume_proposals_task.done() @@ -74,29 +65,26 @@ async def _consume_proposals(self) -> None: while True: proposal = await self._get_init_proposal() - logger.debug(f"Adding proposal `{proposal}` on the scoring...") + await self._manage_scoring(proposal) - async with self._scored_proposals_condition: - all_proposals = list(sp[1] for sp in self._scored_proposals) - all_proposals.append(proposal) - - self._scored_proposals = await self._do_scoring(all_proposals) + @trace_span() + async def _manage_scoring(self, proposal: Proposal) -> None: + async with self._scored_proposals_condition: + all_proposals = list(sp[1] for sp in self._scored_proposals) + all_proposals.append(proposal) - self._scored_proposals_condition.notify_all() + self._scored_proposals = await self._do_scoring(all_proposals) - logger.debug(f"Adding proposal `{proposal}` on the scoring done") + self._scored_proposals_condition.notify_all() + @trace_span() async def get_draft_proposal(self) -> Proposal: - logger.debug("Getting proposal...") - async with self._scored_proposals_condition: await self._scored_proposals_condition.wait_for(lambda: 0 < len(self._scored_proposals)) score, proposal = self._scored_proposals.pop(0) - logger.debug(f"Getting proposal done with `{proposal}` with score `{score}`") - - logger.info(f"Proposal `{proposal}` picked") + logger.info(f"Proposal `{proposal}` picked with score `{score}`") return proposal @@ -109,6 +97,7 @@ async def _do_scoring(self, proposals: Sequence[Proposal]): return scored_proposals + @trace_span() async def _run_plugins( self, proposals_data: Sequence[ProposalData] ) -> Sequence[Tuple[float, Sequence[float]]]: diff --git a/golem/utils/logging.py b/golem/utils/logging.py index 0c635c70..d09ee6ec 100644 --- a/golem/utils/logging.py +++ b/golem/utils/logging.py @@ -4,7 +4,6 @@ from functools import wraps from typing import TYPE_CHECKING, Optional - if TYPE_CHECKING: from golem.event_bus import Event @@ -31,30 +30,47 @@ ], }, "asyncio": { - "level": "DEBUG", + "level": "INFO", }, "golem": { "level": "INFO", }, + "golem.utils.logging": { + "level": "INFO", + }, "golem.managers": { "level": "INFO", }, + "golem.managers.payment": { + "level": "INFO", + }, + "golem.managers.network": { + "level": "INFO", + }, + "golem.managers.demand": { + "level": "INFO", + }, "golem.managers.negotiation": { "level": "INFO", }, "golem.managers.proposal": { "level": "INFO", }, - "golem.managers.work": { + "golem.managers.agreement": { "level": "INFO", }, - "golem.managers.agreement": { + "golem.managers.activity": { + "level": "INFO", + }, + "golem.managers.work": { "level": "INFO", }, }, } -logger = logging.getLogger() +logger = logging.getLogger(__name__) + + class _YagnaDatetimeFormatter(logging.Formatter): """Custom log Formatter that formats datetime using the same convention yagna uses.""" @@ -124,9 +140,9 @@ def wrapper(f): @wraps(f) def sync_wrapped(*args, **kwargs): if show_arguments: - args_str = ', '.join(repr(a) for a in args) - kwargs_str = ', '.join('{}={}'.format(k, repr(v)) for (k, v) in kwargs.items()) - final_name = f'{span_name}({args_str}, {kwargs_str})' + args_str = ", ".join(repr(a) for a in args) + kwargs_str = ", ".join("{}={}".format(k, repr(v)) for (k, v) in kwargs.items()) + final_name = f"{span_name}({args_str}, {kwargs_str})" else: final_name = span_name @@ -148,9 +164,9 @@ def sync_wrapped(*args, **kwargs): @wraps(f) async def async_wrapped(*args, **kwargs): if show_arguments: - args_str = ', '.join(repr(a) for a in args) - kwargs_str = ', '.join('{}={}'.format(k, repr(v)) for (k, v) in kwargs.items()) - final_name = f'{span_name}({args_str}, {kwargs_str})' + args_str = ", ".join(repr(a) for a in args) + kwargs_str = ", ".join("{}={}".format(k, repr(v)) for (k, v) in kwargs.items()) + final_name = f"{span_name}({args_str}, {kwargs_str})" else: final_name = span_name From 4a488bc228876c20ade34333cdf815cd2becfd0f Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 12 Jul 2023 15:41:26 +0200 Subject: [PATCH 076/123] Add trace_span to managers --- golem/managers/activity/mixins.py | 3 +++ golem/managers/activity/pool.py | 6 +++++ golem/managers/activity/single_use.py | 32 +++----------------------- golem/managers/agreement/single_use.py | 27 +++------------------- golem/managers/base.py | 3 +++ golem/managers/network/single.py | 6 +++++ golem/managers/work/asynchronous.py | 5 ++-- golem/managers/work/mixins.py | 4 ++++ golem/managers/work/sequential.py | 11 +++------ 9 files changed, 34 insertions(+), 63 deletions(-) diff --git a/golem/managers/activity/mixins.py b/golem/managers/activity/mixins.py index fabfb1a2..48af52e0 100644 --- a/golem/managers/activity/mixins.py +++ b/golem/managers/activity/mixins.py @@ -5,6 +5,7 @@ from golem.managers.agreement.events import AgreementReleased from golem.managers.base import WorkContext from golem.resources import Activity +from golem.utils.logging import trace_span logger = logging.getLogger(__name__) @@ -26,6 +27,7 @@ def __init__( super().__init__(*args, **kwargs) + @trace_span() async def _prepare_activity(self, agreement) -> Activity: activity = await agreement.create_activity() logger.info(f"Activity `{activity}` created") @@ -34,6 +36,7 @@ async def _prepare_activity(self, agreement) -> Activity: await self._on_activity_start(work_context) return activity + @trace_span() async def _release_activity(self, activity: Activity) -> None: if self._on_activity_stop: work_context = WorkContext(activity) diff --git a/golem/managers/activity/pool.py b/golem/managers/activity/pool.py index a85de911..adb255f6 100644 --- a/golem/managers/activity/pool.py +++ b/golem/managers/activity/pool.py @@ -8,6 +8,7 @@ from golem.node import GolemNode from golem.resources import Agreement from golem.utils.asyncio import create_task_with_logging +from golem.utils.logging import trace_span logger = logging.getLogger(__name__) @@ -28,9 +29,11 @@ def __init__( self._pool = asyncio.Queue() super().__init__(*args, **kwargs) + @trace_span() async def start(self): self._manage_pool_task = create_task_with_logging(self._manage_pool()) + @trace_span() async def stop(self): self._pool_target_size = 0 # TODO cancel prepare_activity tasks @@ -57,11 +60,13 @@ async def _manage_pool(self): ) await asyncio.sleep(0.01) + @trace_span() async def _release_activity_and_pop_from_pool(self): activity = await self._pool.get() await self._release_activity(activity) logger.info(f"Activity `{activity}` removed from the pool") + @trace_span() async def _prepare_activity_and_put_in_pool(self): agreement = await self._get_agreement() activity = await self._prepare_activity(agreement) @@ -76,6 +81,7 @@ async def _get_activity_from_pool(self): self._pool.put_nowait(activity) logger.info(f"Activity `{activity}` back in the pool") + @trace_span() async def do_work(self, work: Work) -> WorkResult: async with self._get_activity_from_pool() as activity: work_context = WorkContext(activity) diff --git a/golem/managers/activity/single_use.py b/golem/managers/activity/single_use.py index a21170e6..33870563 100644 --- a/golem/managers/activity/single_use.py +++ b/golem/managers/activity/single_use.py @@ -6,6 +6,7 @@ from golem.managers.base import ActivityManager, Work, WorkContext, WorkResult from golem.node import GolemNode from golem.resources import Activity, Agreement +from golem.utils.logging import trace_span logger = logging.getLogger(__name__) @@ -22,52 +23,25 @@ def __init__( @asynccontextmanager async def _prepare_single_use_activity(self) -> Activity: while True: - logger.debug("Getting agreement...") - agreement = await self._get_agreement() - - logger.debug(f"Getting agreement done with `{agreement}`") - try: - logger.debug("Creating activity...") - activity = await self._prepare_activity(agreement) - - logger.debug(f"Creating activity done with `{activity}`") logger.info(f"Activity `{activity}` created") - - logger.debug("Yielding activity...") - yield activity - await self._release_activity(activity) - - logger.debug("Yielding activity done") - break except Exception: logger.exception("Creating activity failed, but will be retried with new agreement") + @trace_span() async def do_work(self, work: Work) -> WorkResult: - logger.debug(f"Doing work `{work}`...") - async with self._prepare_single_use_activity() as activity: work_context = WorkContext(activity) - try: - logger.debug(f"Calling `{work}`...") work_result = await work(work_context) except Exception as e: - logger.debug(f"Calling `{work}` done with exception `{e}`") work_result = WorkResult(exception=e) else: - if isinstance(work_result, WorkResult): - logger.debug(f"Calling `{work}` done with explicit result `{work_result}`") - else: - logger.debug(f"Calling `{work}` done with implicit result `{work_result}`") - + if not isinstance(work_result, WorkResult): work_result = WorkResult(result=work_result) - - logger.debug(f"Doing work `{work}` done") - return work_result diff --git a/golem/managers/agreement/single_use.py b/golem/managers/agreement/single_use.py index 6dd0e93d..4fe024da 100644 --- a/golem/managers/agreement/single_use.py +++ b/golem/managers/agreement/single_use.py @@ -5,6 +5,7 @@ from golem.managers.base import AgreementManager from golem.node import GolemNode from golem.resources import Agreement, Proposal +from golem.utils.logging import trace_span logger = logging.getLogger(__name__) @@ -14,32 +15,17 @@ def __init__(self, golem: GolemNode, get_proposal: Callable[[], Awaitable[Propos self._get_proposal = get_proposal self._event_bus = golem.event_bus + @trace_span() async def get_agreement(self) -> Agreement: - logger.debug("Getting agreement...") - while True: - logger.debug("Getting proposal...") - proposal = await self._get_proposal() - - logger.debug(f"Getting proposal done with {proposal}") - try: - logger.debug("Creating agreement...") - agreement = await proposal.create_agreement() - - logger.debug("Sending agreement to provider...") - await agreement.confirm() - - logger.debug("Waiting for provider approval...") - await agreement.wait_for_approval() except Exception as e: logger.debug(f"Creating agreement failed with `{e}`. Retrying...") else: - logger.debug(f"Creating agreement done with `{agreement}`") logger.info(f"Agreement `{agreement}` created") # TODO: Support removing callback on resource close @@ -48,17 +34,10 @@ async def get_agreement(self) -> Agreement: self._terminate_agreement, lambda event: event.resource.id == agreement.id, ) - - logger.debug(f"Getting agreement done with `{agreement}`") - return agreement + @trace_span() async def _terminate_agreement(self, event: AgreementReleased) -> None: - logger.debug("Calling `_terminate_agreement`...") - agreement: Agreement = event.resource await agreement.terminate() - - logger.debug("Calling `_terminate_agreement` done") - logger.info(f"Agreement `{agreement}` closed") diff --git a/golem/managers/base.py b/golem/managers/base.py index 7f7f3700..42ce6ce7 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -26,6 +26,7 @@ Script, ) from golem.resources.activity import commands +from golem.utils.logging import trace_span class Batch: @@ -145,9 +146,11 @@ def __init__(self, plugins: Optional[Sequence[TPlugin]] = None, *args, **kwargs) super().__init__(*args, **kwargs) + @trace_span() def register_plugin(self, plugin: TPlugin): self._plugins.append(plugin) + @trace_span() def unregister_plugin(self, plugin: TPlugin): self._plugins.remove(plugin) diff --git a/golem/managers/network/single.py b/golem/managers/network/single.py index b2d06ad3..7cef682e 100644 --- a/golem/managers/network/single.py +++ b/golem/managers/network/single.py @@ -6,6 +6,7 @@ from golem.managers.base import NetworkManager from golem.node import GolemNode from golem.resources import DeployArgsType, Network, NewAgreement +from golem.utils.logging import trace_span logger = logging.getLogger(__name__) @@ -16,6 +17,7 @@ def __init__(self, golem: GolemNode, ip: str) -> None: self._ip = ip self._nodes: Dict[str, str] = {} + @trace_span() async def start(self): self._network = await Network.create(self._golem, self._ip, None, None) await self._network.add_requestor_ip(None) @@ -23,6 +25,7 @@ async def start(self): await self._golem.event_bus.on(NewAgreement, self._add_provider_to_network) + @trace_span() async def get_node_id(self, provider_id: str) -> str: while True: node_ip = self._nodes.get(provider_id) @@ -30,10 +33,12 @@ async def get_node_id(self, provider_id: str) -> str: return node_ip await asyncio.sleep(0.1) + @trace_span() async def get_deploy_args(self, provider_id: str) -> DeployArgsType: node_ip = await self.get_node_id(provider_id) return self._network.deploy_args(node_ip) + @trace_span() async def get_provider_uri(self, provider_id: str, protocol: str = "http") -> str: node_ip = await self.get_node_id(provider_id) url = self._network.node._api_config.net_url @@ -41,6 +46,7 @@ async def get_provider_uri(self, provider_id: str, protocol: str = "http") -> st connection_uri = f"{net_api_ws}/net/{self._network.id}/tcp/{node_ip}/22" return connection_uri + @trace_span() async def _add_provider_to_network(self, event: NewAgreement): await event.resource.get_data() provider_id = event.resource.data.offer.provider_id diff --git a/golem/managers/work/asynchronous.py b/golem/managers/work/asynchronous.py index b7476e60..c97c2598 100644 --- a/golem/managers/work/asynchronous.py +++ b/golem/managers/work/asynchronous.py @@ -5,6 +5,7 @@ from golem.managers.base import DoWorkCallable, Work, WorkManager, WorkResult from golem.managers.work.mixins import WorkManagerPluginsMixin from golem.node import GolemNode +from golem.utils.logging import trace_span logger = logging.getLogger(__name__) @@ -15,13 +16,13 @@ def __init__(self, golem: GolemNode, do_work: DoWorkCallable, *args, **kwargs): super().__init__(*args, **kwargs) + @trace_span() async def do_work(self, work: Work) -> WorkResult: result = await self._do_work_with_plugins(self._do_work, work) - logger.info(f"Work `{work}` completed") - return result + @trace_span() async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: results = await asyncio.gather(*[self.do_work(work) for work in work_list]) return results diff --git a/golem/managers/work/mixins.py b/golem/managers/work/mixins.py index c5cc997f..3f205e7e 100644 --- a/golem/managers/work/mixins.py +++ b/golem/managers/work/mixins.py @@ -8,11 +8,13 @@ WorkManagerPlugin, WorkResult, ) +from golem.utils.logging import trace_span logger = logging.getLogger(__name__) class WorkManagerPluginsMixin(ManagerPluginsMixin[WorkManagerPlugin]): + @trace_span() def _apply_plugins_from_manager(self, do_work: DoWorkCallable) -> DoWorkCallable: do_work_with_plugins = do_work @@ -21,6 +23,7 @@ def _apply_plugins_from_manager(self, do_work: DoWorkCallable) -> DoWorkCallable return do_work_with_plugins + @trace_span() def _apply_plugins_from_work(self, do_work: DoWorkCallable, work: Work) -> DoWorkCallable: work_plugins = getattr(work, WORK_PLUGIN_FIELD_NAME, []) @@ -34,6 +37,7 @@ def _apply_plugins_from_work(self, do_work: DoWorkCallable, work: Work) -> DoWor return do_work_with_plugins + @trace_span() async def _do_work_with_plugins(self, do_work: DoWorkCallable, work: Work) -> WorkResult: do_work_with_plugins = self._apply_plugins_from_manager(do_work) do_work_with_plugins = self._apply_plugins_from_work(do_work_with_plugins, work) diff --git a/golem/managers/work/sequential.py b/golem/managers/work/sequential.py index d549556b..ceae43fb 100644 --- a/golem/managers/work/sequential.py +++ b/golem/managers/work/sequential.py @@ -4,6 +4,7 @@ from golem.managers.base import DoWorkCallable, Work, WorkManager, WorkResult from golem.managers.work.mixins import WorkManagerPluginsMixin from golem.node import GolemNode +from golem.utils.logging import trace_span logger = logging.getLogger(__name__) @@ -14,6 +15,7 @@ def __init__(self, golem: GolemNode, do_work: DoWorkCallable, *args, **kwargs): super().__init__(*args, **kwargs) + @trace_span() async def do_work(self, work: Work) -> WorkResult: result = await self._do_work_with_plugins(self._do_work, work) @@ -21,18 +23,11 @@ async def do_work(self, work: Work) -> WorkResult: return result + @trace_span() async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: - logger.debug(f"Running work sequence `{work_list}`...") - results = [] for i, work in enumerate(work_list): - logger.debug(f"Doing work sequence #{i}...") - results.append(await self.do_work(work)) - logger.debug(f"Doing work sequence #{i} done") - - logger.debug(f"Running work sequence `{work_list}` done") - return results From d185187646ed2247b77ede64ebef502d49347ce8 Mon Sep 17 00:00:00 2001 From: approxit Date: Wed, 12 Jul 2023 16:47:54 +0200 Subject: [PATCH 077/123] todos --- golem/managers/activity/pool.py | 3 +++ golem/managers/negotiation/sequential.py | 34 +++++++++++++----------- golem/managers/payment/pay_all.py | 1 + golem/resources/proposal/proposal.py | 1 + 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/golem/managers/activity/pool.py b/golem/managers/activity/pool.py index adb255f6..5e932d4f 100644 --- a/golem/managers/activity/pool.py +++ b/golem/managers/activity/pool.py @@ -44,6 +44,7 @@ async def _manage_pool(self): pool_current_size = 0 release_tasks = [] prepare_tasks = [] + # TODO: After reducing to zero its not possible to manage pool afterwards while self._pool_target_size > 0 or pool_current_size > 0: # TODO observe tasks status and add fallback if pool_current_size > self._pool_target_size: @@ -58,6 +59,8 @@ async def _manage_pool(self): prepare_tasks.append( create_task_with_logging(self._prepare_activity_and_put_in_pool()) ) + + # TODO: Use events instead of sleep await asyncio.sleep(0.01) @trace_span() diff --git a/golem/managers/negotiation/sequential.py b/golem/managers/negotiation/sequential.py index ff063304..cf580deb 100644 --- a/golem/managers/negotiation/sequential.py +++ b/golem/managers/negotiation/sequential.py @@ -85,7 +85,7 @@ async def _negotiate_proposal( demand_data_after_plugins = deepcopy(demand_data) try: - await self._apply_plugins(demand_data_after_plugins, offer_proposal) + await self._run_plugins(demand_data_after_plugins, offer_proposal) except RejectProposal as e: logger.debug(f"Proposal `{offer_proposal}` was rejected by plugins") @@ -99,14 +99,12 @@ async def _negotiate_proposal( demand_data = demand_data_after_plugins - try: - demand_proposal = await self._send_demand_proposal(offer_proposal, demand_data) - except (ApiException, asyncio.TimeoutError): + demand_proposal = await self._send_demand_proposal(offer_proposal, demand_data) + if demand_proposal is None: return None - try: - new_offer_proposal = await self._wait_for_proposal_response(demand_proposal) - except StopAsyncIteration: + new_offer_proposal = await self._wait_for_proposal_response(demand_proposal) + if new_offer_proposal is None: return None logger.debug( @@ -116,7 +114,7 @@ async def _negotiate_proposal( offer_proposal = new_offer_proposal @trace_span() - async def _apply_plugins(self, demand_data_after_plugins: DemandData, offer_proposal: Proposal): + async def _run_plugins(self, demand_data_after_plugins: DemandData, offer_proposal: Proposal): proposal_data = await self._get_proposal_data_from_proposal(offer_proposal) for plugin in self._plugins: @@ -135,15 +133,21 @@ async def _apply_plugins(self, demand_data_after_plugins: DemandData, offer_prop @trace_span() async def _send_demand_proposal( self, offer_proposal: Proposal, demand_data: DemandData - ) -> Proposal: - return await offer_proposal.respond( - demand_data.properties, - demand_data.constraints, - ) + ) -> Optional[Proposal]: + try: + return await offer_proposal.respond( + demand_data.properties, + demand_data.constraints, + ) + except (ApiException, asyncio.TimeoutError): + return None @trace_span() - async def _wait_for_proposal_response(self, demand_proposal: Proposal) -> Proposal: - return await demand_proposal.responses().__anext__() + async def _wait_for_proposal_response(self, demand_proposal: Proposal) -> Optional[Proposal]: + try: + return await demand_proposal.responses().__anext__() + except StopAsyncIteration: + return None async def _get_demand_data_from_proposal(self, proposal: Proposal) -> DemandData: # FIXME: Unnecessary serialisation from DemandBuilder to Demand, diff --git a/golem/managers/payment/pay_all.py b/golem/managers/payment/pay_all.py index 37ca6820..c22fef73 100644 --- a/golem/managers/payment/pay_all.py +++ b/golem/managers/payment/pay_all.py @@ -65,6 +65,7 @@ async def _create_allocation(self) -> None: self._golem, Decimal(self._budget), self._network, self._driver ) + # TODO: We should not rely on golem node with cleanups, manager should do it by itself self._golem.add_autoclose_resource(self._allocation) @trace_span() diff --git a/golem/resources/proposal/proposal.py b/golem/resources/proposal/proposal.py index f8de0209..242d0cb2 100644 --- a/golem/resources/proposal/proposal.py +++ b/golem/resources/proposal/proposal.py @@ -115,6 +115,7 @@ def add_event(self, event: Union[models.ProposalEvent, models.ProposalRejectedEv if isinstance(event, models.ProposalRejectedEvent): self.set_no_more_children() + # TODO: Add support for rejection reason from provider async def responses(self) -> AsyncIterator["Proposal"]: """Yield responses to this proposal. From 3b7dda8605500ce95f32958499890c08056274b0 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 13 Jul 2023 09:10:15 +0200 Subject: [PATCH 078/123] Updated managers examples with DemandManager --- examples/managers/blender/blender.py | 11 ++++++++--- examples/managers/ssh.py | 10 +++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/examples/managers/blender/blender.py b/examples/managers/blender/blender.py index 2a3cd7ba..542f7306 100644 --- a/examples/managers/blender/blender.py +++ b/examples/managers/blender/blender.py @@ -8,6 +8,7 @@ from golem.managers.activity.pool import ActivityPoolManager from golem.managers.agreement.single_use import SingleUseAgreementManager from golem.managers.base import WorkContext, WorkResult +from golem.managers.demand.auto import AutoDemandManager from golem.managers.negotiation import SequentialNegotiationManager from golem.managers.negotiation.plugins import AddChosenPaymentPlatform from golem.managers.payment.pay_all import PayAllPaymentManager @@ -22,7 +23,7 @@ BLENDER_IMAGE_HASH = "9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae" FRAME_CONFIG_TEMPLATE = json.loads(Path(__file__).with_name("frame_params.json").read_text()) -FRAMES = list(range(0, 60, 3)) +FRAMES = list(range(0, 60, 5)) async def run_on_golem( @@ -40,10 +41,14 @@ async def run_on_golem( golem = GolemNode() payment_manager = PayAllPaymentManager(golem, budget=budget) - negotiation_manager = SequentialNegotiationManager( + demand_manager = AutoDemandManager( golem, payment_manager.get_allocation, payload, + ) + negotiation_manager = SequentialNegotiationManager( + golem, + demand_manager.get_initial_proposal, plugins=market_plugins, ) proposal_manager = ScoredAheadOfTimeProposalManager( @@ -55,7 +60,7 @@ async def run_on_golem( ) work_manager = AsynchronousWorkManager(golem, activity_manager.do_work, plugins=task_plugins) - async with golem, payment_manager, negotiation_manager, proposal_manager, activity_manager: + async with golem, payment_manager, demand_manager, negotiation_manager, proposal_manager, activity_manager: results: List[WorkResult] = await work_manager.do_work_list(task_list) return results diff --git a/examples/managers/ssh.py b/examples/managers/ssh.py index 56c54b92..5f1b3eaa 100644 --- a/examples/managers/ssh.py +++ b/examples/managers/ssh.py @@ -7,6 +7,7 @@ from golem.managers.activity.single_use import SingleUseActivityManager from golem.managers.agreement.single_use import SingleUseAgreementManager from golem.managers.base import WorkContext, WorkResult +from golem.managers.demand.auto import AutoDemandManager from golem.managers.negotiation import SequentialNegotiationManager from golem.managers.network.single import SingleNetworkManager from golem.managers.payment.pay_all import PayAllPaymentManager @@ -71,9 +72,12 @@ async def main(): network_manager = SingleNetworkManager(golem, network_ip) payment_manager = PayAllPaymentManager(golem, budget=1.0) - negotiation_manager = SequentialNegotiationManager( - golem, payment_manager.get_allocation, payload + demand_manager = AutoDemandManager( + golem, + payment_manager.get_allocation, + payload, ) + negotiation_manager = SequentialNegotiationManager(golem, demand_manager.get_initial_proposal) proposal_manager = ScoredAheadOfTimeProposalManager( golem, negotiation_manager.get_draft_proposal ) @@ -85,7 +89,7 @@ async def main(): ) work_manager = SequentialWorkManager(golem, activity_manager.do_work) # TODO use different managers so it allows to finish work func without destroying activity - async with golem, network_manager, payment_manager, negotiation_manager, proposal_manager: + async with golem, network_manager, payment_manager, demand_manager, negotiation_manager, proposal_manager: result: WorkResult = await work_manager.do_work( work(golem._api_config.app_key, network_manager.get_provider_uri) ) From ad68a67311a17af6fa3714a01fb3cd67b78726d1 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 13 Jul 2023 09:28:36 +0200 Subject: [PATCH 079/123] Merge proposal and agreement managers --- examples/managers/basic_composition.py | 12 ++--- examples/managers/blender/blender.py | 12 ++--- examples/managers/ssh.py | 8 ++- golem/managers/agreement/__init__.py | 4 +- .../{proposal => agreement}/plugins.py | 0 .../{proposal => agreement}/pricings.py | 0 .../{proposal => agreement}/scored_aot.py | 52 ++++++++++++++----- golem/managers/agreement/single_use.py | 43 --------------- golem/managers/base.py | 6 --- golem/managers/proposal/__init__.py | 3 -- ...s.py => test_manager_agreement_plugins.py} | 2 +- 11 files changed, 55 insertions(+), 87 deletions(-) rename golem/managers/{proposal => agreement}/plugins.py (100%) rename golem/managers/{proposal => agreement}/pricings.py (100%) rename golem/managers/{proposal => agreement}/scored_aot.py (77%) delete mode 100644 golem/managers/agreement/single_use.py delete mode 100644 golem/managers/proposal/__init__.py rename tests/unit/{test_manager_proposal_plugins.py => test_manager_agreement_plugins.py} (95%) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index efb0807a..43a9788a 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -5,7 +5,9 @@ from typing import List from golem.managers.activity.single_use import SingleUseActivityManager -from golem.managers.agreement.single_use import SingleUseAgreementManager +from golem.managers.agreement.plugins import MapScore, PropertyValueLerpScore, RandomScore +from golem.managers.agreement.pricings import LinearAverageCostPricing +from golem.managers.agreement.scored_aot import ScoredAheadOfTimeAgreementManager from golem.managers.base import RejectProposal, WorkContext, WorkResult from golem.managers.demand.auto import AutoDemandManager from golem.managers.negotiation import SequentialNegotiationManager @@ -15,9 +17,6 @@ RejectIfCostsExceeds, ) from golem.managers.payment.pay_all import PayAllPaymentManager -from golem.managers.proposal import ScoredAheadOfTimeProposalManager -from golem.managers.proposal.plugins import MapScore, PropertyValueLerpScore, RandomScore -from golem.managers.proposal.pricings import LinearAverageCostPricing from golem.managers.work.plugins import redundancy_cancel_others_on_first_done, retry, work_plugin from golem.managers.work.sequential import SequentialWorkManager from golem.node import GolemNode @@ -102,7 +101,7 @@ async def main(): else None, ], ) - proposal_manager = ScoredAheadOfTimeProposalManager( + agreement_manager = ScoredAheadOfTimeAgreementManager( golem, negotiation_manager.get_draft_proposal, plugins=[ @@ -113,7 +112,6 @@ async def main(): [0.0, MapScore(lambda proposal_data: random())], ], ) - agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_draft_proposal) activity_manager = SingleUseActivityManager(golem, agreement_manager.get_agreement) work_manager = SequentialWorkManager( golem, @@ -124,7 +122,7 @@ async def main(): ) async with golem: - async with payment_manager, demand_manager, negotiation_manager, proposal_manager: + async with payment_manager, demand_manager, negotiation_manager, agreement_manager: results: List[WorkResult] = await work_manager.do_work_list(work_list) print(f"\nWORK MANAGER RESULTS:{[result.result for result in results]}\n", flush=True) diff --git a/examples/managers/blender/blender.py b/examples/managers/blender/blender.py index 542f7306..42276f31 100644 --- a/examples/managers/blender/blender.py +++ b/examples/managers/blender/blender.py @@ -6,15 +6,14 @@ from typing import List from golem.managers.activity.pool import ActivityPoolManager -from golem.managers.agreement.single_use import SingleUseAgreementManager +from golem.managers.agreement.plugins import MapScore +from golem.managers.agreement.pricings import LinearAverageCostPricing +from golem.managers.agreement.scored_aot import ScoredAheadOfTimeAgreementManager from golem.managers.base import WorkContext, WorkResult from golem.managers.demand.auto import AutoDemandManager from golem.managers.negotiation import SequentialNegotiationManager from golem.managers.negotiation.plugins import AddChosenPaymentPlatform from golem.managers.payment.pay_all import PayAllPaymentManager -from golem.managers.proposal import ScoredAheadOfTimeProposalManager -from golem.managers.proposal.plugins import MapScore -from golem.managers.proposal.pricings import LinearAverageCostPricing from golem.managers.work.asynchronous import AsynchronousWorkManager from golem.managers.work.plugins import retry from golem.node import GolemNode @@ -51,16 +50,15 @@ async def run_on_golem( demand_manager.get_initial_proposal, plugins=market_plugins, ) - proposal_manager = ScoredAheadOfTimeProposalManager( + agreement_manager = ScoredAheadOfTimeAgreementManager( golem, negotiation_manager.get_draft_proposal, plugins=scoring_plugins ) - agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_draft_proposal) activity_manager = ActivityPoolManager( golem, agreement_manager.get_agreement, size=threads, on_activity_start=init_func ) work_manager = AsynchronousWorkManager(golem, activity_manager.do_work, plugins=task_plugins) - async with golem, payment_manager, demand_manager, negotiation_manager, proposal_manager, activity_manager: + async with golem, payment_manager, demand_manager, negotiation_manager, agreement_manager, activity_manager: results: List[WorkResult] = await work_manager.do_work_list(task_list) return results diff --git a/examples/managers/ssh.py b/examples/managers/ssh.py index 5f1b3eaa..29052c45 100644 --- a/examples/managers/ssh.py +++ b/examples/managers/ssh.py @@ -5,13 +5,12 @@ from uuid import uuid4 from golem.managers.activity.single_use import SingleUseActivityManager -from golem.managers.agreement.single_use import SingleUseAgreementManager +from golem.managers.agreement.scored_aot import ScoredAheadOfTimeAgreementManager from golem.managers.base import WorkContext, WorkResult from golem.managers.demand.auto import AutoDemandManager from golem.managers.negotiation import SequentialNegotiationManager from golem.managers.network.single import SingleNetworkManager from golem.managers.payment.pay_all import PayAllPaymentManager -from golem.managers.proposal import ScoredAheadOfTimeProposalManager from golem.managers.work.sequential import SequentialWorkManager from golem.node import GolemNode from golem.payload import RepositoryVmPayload @@ -78,10 +77,9 @@ async def main(): payload, ) negotiation_manager = SequentialNegotiationManager(golem, demand_manager.get_initial_proposal) - proposal_manager = ScoredAheadOfTimeProposalManager( + agreement_manager = ScoredAheadOfTimeAgreementManager( golem, negotiation_manager.get_draft_proposal ) - agreement_manager = SingleUseAgreementManager(golem, proposal_manager.get_draft_proposal) activity_manager = SingleUseActivityManager( golem, agreement_manager.get_agreement, @@ -89,7 +87,7 @@ async def main(): ) work_manager = SequentialWorkManager(golem, activity_manager.do_work) # TODO use different managers so it allows to finish work func without destroying activity - async with golem, network_manager, payment_manager, demand_manager, negotiation_manager, proposal_manager: + async with golem, network_manager, payment_manager, demand_manager, negotiation_manager, agreement_manager: result: WorkResult = await work_manager.do_work( work(golem._api_config.app_key, network_manager.get_provider_uri) ) diff --git a/golem/managers/agreement/__init__.py b/golem/managers/agreement/__init__.py index 2ba5ea00..9437cbd1 100644 --- a/golem/managers/agreement/__init__.py +++ b/golem/managers/agreement/__init__.py @@ -1,7 +1,7 @@ from golem.managers.agreement.events import AgreementReleased -from golem.managers.agreement.single_use import SingleUseAgreementManager +from golem.managers.agreement.scored_aot import ScoredAheadOfTimeAgreementManager __all__ = ( "AgreementReleased", - "SingleUseAgreementManager", + "ScoredAheadOfTimeAgreementManager", ) diff --git a/golem/managers/proposal/plugins.py b/golem/managers/agreement/plugins.py similarity index 100% rename from golem/managers/proposal/plugins.py rename to golem/managers/agreement/plugins.py diff --git a/golem/managers/proposal/pricings.py b/golem/managers/agreement/pricings.py similarity index 100% rename from golem/managers/proposal/pricings.py rename to golem/managers/agreement/pricings.py diff --git a/golem/managers/proposal/scored_aot.py b/golem/managers/agreement/scored_aot.py similarity index 77% rename from golem/managers/proposal/scored_aot.py rename to golem/managers/agreement/scored_aot.py index e0cf92c5..9a5fa76c 100644 --- a/golem/managers/proposal/scored_aot.py +++ b/golem/managers/agreement/scored_aot.py @@ -4,37 +4,36 @@ from datetime import datetime from typing import Awaitable, Callable, List, Optional, Sequence, Tuple, cast +from golem.managers.agreement.events import AgreementReleased from golem.managers.base import ( + AgreementManager, ManagerException, ManagerPluginsMixin, ManagerPluginWithOptionalWeight, - ProposalManager, ) from golem.node import GolemNode from golem.payload import Properties from golem.payload.parsers.textx import TextXPayloadSyntaxParser -from golem.resources import Proposal, ProposalData +from golem.resources import Agreement, Proposal, ProposalData from golem.utils.asyncio import create_task_with_logging from golem.utils.logging import trace_span logger = logging.getLogger(__name__) -class IgnoreProposal(Exception): - pass - - -class ScoredAheadOfTimeProposalManager( - ManagerPluginsMixin[ManagerPluginWithOptionalWeight], ProposalManager +class ScoredAheadOfTimeAgreementManager( + ManagerPluginsMixin[ManagerPluginWithOptionalWeight], AgreementManager ): def __init__( self, golem: GolemNode, - get_init_proposal: Callable[[], Awaitable[Proposal]], + get_draft_proposal: Callable[[], Awaitable[Proposal]], *args, **kwargs, - ) -> None: - self._get_init_proposal = get_init_proposal + ): + self._get_draft_proposal = get_draft_proposal + self._event_bus = golem.event_bus + self._consume_proposals_task: Optional[asyncio.Task] = None self._demand_offer_parser = TextXPayloadSyntaxParser() @@ -63,7 +62,7 @@ def is_started(self) -> bool: async def _consume_proposals(self) -> None: while True: - proposal = await self._get_init_proposal() + proposal = await self._get_draft_proposal() await self._manage_scoring(proposal) @@ -78,7 +77,7 @@ async def _manage_scoring(self, proposal: Proposal) -> None: self._scored_proposals_condition.notify_all() @trace_span() - async def get_draft_proposal(self) -> Proposal: + async def _get_scored_proposal(self): async with self._scored_proposals_condition: await self._scored_proposals_condition.wait_for(lambda: 0 < len(self._scored_proposals)) @@ -88,6 +87,33 @@ async def get_draft_proposal(self) -> Proposal: return proposal + @trace_span() + async def get_agreement(self) -> Agreement: + while True: + proposal = await self._get_scored_proposal() + try: + agreement = await proposal.create_agreement() + await agreement.confirm() + await agreement.wait_for_approval() + except Exception as e: + logger.debug(f"Creating agreement failed with `{e}`. Retrying...") + else: + logger.info(f"Agreement `{agreement}` created") + + # TODO: Support removing callback on resource close + await self._event_bus.on_once( + AgreementReleased, + self._terminate_agreement, + lambda event: event.resource.id == agreement.id, + ) + return agreement + + @trace_span() + async def _terminate_agreement(self, event: AgreementReleased) -> None: + agreement: Agreement = event.resource + await agreement.terminate() + logger.info(f"Agreement `{agreement}` closed") + async def _do_scoring(self, proposals: Sequence[Proposal]): proposals_data = await self._get_proposals_data_from_proposals(proposals) proposal_scores = await self._run_plugins(proposals_data) diff --git a/golem/managers/agreement/single_use.py b/golem/managers/agreement/single_use.py deleted file mode 100644 index 4fe024da..00000000 --- a/golem/managers/agreement/single_use.py +++ /dev/null @@ -1,43 +0,0 @@ -import logging -from typing import Awaitable, Callable - -from golem.managers.agreement.events import AgreementReleased -from golem.managers.base import AgreementManager -from golem.node import GolemNode -from golem.resources import Agreement, Proposal -from golem.utils.logging import trace_span - -logger = logging.getLogger(__name__) - - -class SingleUseAgreementManager(AgreementManager): - def __init__(self, golem: GolemNode, get_proposal: Callable[[], Awaitable[Proposal]]): - self._get_proposal = get_proposal - self._event_bus = golem.event_bus - - @trace_span() - async def get_agreement(self) -> Agreement: - while True: - proposal = await self._get_proposal() - try: - agreement = await proposal.create_agreement() - await agreement.confirm() - await agreement.wait_for_approval() - except Exception as e: - logger.debug(f"Creating agreement failed with `{e}`. Retrying...") - else: - logger.info(f"Agreement `{agreement}` created") - - # TODO: Support removing callback on resource close - await self._event_bus.on_once( - AgreementReleased, - self._terminate_agreement, - lambda event: event.resource.id == agreement.id, - ) - return agreement - - @trace_span() - async def _terminate_agreement(self, event: AgreementReleased) -> None: - agreement: Agreement = event.resource - await agreement.terminate() - logger.info(f"Agreement `{agreement}` closed") diff --git a/golem/managers/base.py b/golem/managers/base.py index 42ce6ce7..8b71c3ef 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -177,12 +177,6 @@ async def get_draft_proposal(self) -> Proposal: ... -class ProposalManager(Manager, ABC): - @abstractmethod - async def get_draft_proposal(self) -> Proposal: - ... - - class AgreementManager(Manager, ABC): @abstractmethod async def get_agreement(self) -> Agreement: diff --git a/golem/managers/proposal/__init__.py b/golem/managers/proposal/__init__.py deleted file mode 100644 index 82e9a6aa..00000000 --- a/golem/managers/proposal/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from golem.managers.proposal.scored_aot import ScoredAheadOfTimeProposalManager - -__all__ = ("ScoredAheadOfTimeProposalManager",) diff --git a/tests/unit/test_manager_proposal_plugins.py b/tests/unit/test_manager_agreement_plugins.py similarity index 95% rename from tests/unit/test_manager_proposal_plugins.py rename to tests/unit/test_manager_agreement_plugins.py index 1ff7f930..f47131e7 100644 --- a/tests/unit/test_manager_proposal_plugins.py +++ b/tests/unit/test_manager_agreement_plugins.py @@ -1,6 +1,6 @@ import pytest -from golem.managers.proposal.plugins import PropertyValueLerpScore, RandomScore +from golem.managers.agreement.plugins import PropertyValueLerpScore, RandomScore @pytest.mark.parametrize( From 248d89abf1182dffc8e5070def474026c9b765d7 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 13 Jul 2023 10:12:24 +0200 Subject: [PATCH 080/123] Add ContextManagerLoopMixin --- .../exception_handling/exception_handling.py | 3 +- examples/rate_providers/rate_providers.py | 3 +- .../score_based_providers.py | 3 +- .../examples/pipeline_example.py | 3 +- examples/task_api_draft/examples/yacat.py | 3 +- .../task_api_draft/task_api/execute_tasks.py | 3 +- golem/managers/__init__.py | 2 - golem/managers/activity/pool.py | 71 ++++++++++--------- golem/managers/agreement/scored_aot.py | 28 ++------ golem/managers/base.py | 29 +++++++- golem/managers/demand/auto.py | 29 ++------ golem/managers/negotiation/sequential.py | 26 +------ golem/managers/network/single.py | 2 +- golem/managers/payment/__init__.py | 4 +- golem/managers/payment/pay_all.py | 4 +- golem/pipeline/__init__.py | 2 + .../default_payment_handler.py} | 0 17 files changed, 87 insertions(+), 128 deletions(-) rename golem/{managers/payment/default.py => pipeline/default_payment_handler.py} (100%) diff --git a/examples/exception_handling/exception_handling.py b/examples/exception_handling/exception_handling.py index d63a52a9..868b497d 100644 --- a/examples/exception_handling/exception_handling.py +++ b/examples/exception_handling/exception_handling.py @@ -2,10 +2,9 @@ from typing import Callable, Tuple from golem.event_bus import Event -from golem.managers import DefaultPaymentManager from golem.node import GolemNode from golem.payload import RepositoryVmPayload -from golem.pipeline import Buffer, Chain, Limit, Map +from golem.pipeline import Buffer, Chain, DefaultPaymentManager, Limit, Map from golem.resources import ( BatchError, BatchTimeoutError, diff --git a/examples/rate_providers/rate_providers.py b/examples/rate_providers/rate_providers.py index 9989db2f..1b86a31a 100644 --- a/examples/rate_providers/rate_providers.py +++ b/examples/rate_providers/rate_providers.py @@ -5,10 +5,9 @@ from typing import Any, AsyncIterator, Callable, Dict, Optional, Tuple from golem.event_bus import Event -from golem.managers import DefaultPaymentManager from golem.node import GolemNode from golem.payload import RepositoryVmPayload -from golem.pipeline import Buffer, Chain, Limit, Map +from golem.pipeline import Buffer, Chain, DefaultPaymentManager, Limit, Map from golem.resources import ( Proposal, default_create_activity, diff --git a/examples/score_based_providers/score_based_providers.py b/examples/score_based_providers/score_based_providers.py index 493b43fc..0afd0362 100644 --- a/examples/score_based_providers/score_based_providers.py +++ b/examples/score_based_providers/score_based_providers.py @@ -3,10 +3,9 @@ from typing import Callable, Dict, Optional, Tuple from golem.event_bus import Event -from golem.managers import DefaultPaymentManager from golem.node import GolemNode from golem.payload import RepositoryVmPayload -from golem.pipeline import Buffer, Chain, Limit, Map, Sort +from golem.pipeline import Buffer, Chain, DefaultPaymentManager, Limit, Map, Sort from golem.resources import ( Proposal, default_create_activity, diff --git a/examples/task_api_draft/examples/pipeline_example.py b/examples/task_api_draft/examples/pipeline_example.py index b184133c..5585693f 100644 --- a/examples/task_api_draft/examples/pipeline_example.py +++ b/examples/task_api_draft/examples/pipeline_example.py @@ -5,10 +5,9 @@ from examples.task_api_draft.task_api.activity_pool import ActivityPool from golem.events_bus import Event -from golem.managers import DefaultPaymentManager from golem.node import GolemNode from golem.payload import RepositoryVmPayload -from golem.pipeline import Buffer, Chain, Map, Sort, Zip +from golem.pipeline import Buffer, Chain, DefaultPaymentManager, Map, Sort, Zip from golem.resources import ( Proposal, default_create_activity, diff --git a/examples/task_api_draft/examples/yacat.py b/examples/task_api_draft/examples/yacat.py index 5d55ee73..65eb0744 100644 --- a/examples/task_api_draft/examples/yacat.py +++ b/examples/task_api_draft/examples/yacat.py @@ -24,9 +24,8 @@ ) from examples.task_api_draft.task_api.activity_pool import ActivityPool from golem.event_bus import Event -from golem.managers import DefaultPaymentManager from golem.node import GolemNode -from golem.pipeline import Buffer, Chain, Map, Sort, Zip +from golem.pipeline import Buffer, Chain, DefaultPaymentManager, Map, Sort, Zip from golem.resources import ( Activity, DebitNote, diff --git a/examples/task_api_draft/task_api/execute_tasks.py b/examples/task_api_draft/task_api/execute_tasks.py index 8e605249..a6fcbb69 100644 --- a/examples/task_api_draft/task_api/execute_tasks.py +++ b/examples/task_api_draft/task_api/execute_tasks.py @@ -3,10 +3,9 @@ from typing import AsyncIterator, Awaitable, Callable, Iterable, Optional, Tuple, TypeVar from golem.event_bus import Event -from golem.managers import DefaultPaymentManager from golem.node import GolemNode from golem.payload import Payload -from golem.pipeline import Buffer, Chain, Map, Sort, Zip +from golem.pipeline import Buffer, Chain, DefaultPaymentManager, Map, Sort, Zip from golem.resources import ( Activity, Demand, diff --git a/golem/managers/__init__.py b/golem/managers/__init__.py index aadc1973..8b137891 100644 --- a/golem/managers/__init__.py +++ b/golem/managers/__init__.py @@ -1,3 +1 @@ -from golem.managers.payment import DefaultPaymentManager -__all__ = ("DefaultPaymentManager",) diff --git a/golem/managers/activity/pool.py b/golem/managers/activity/pool.py index 5e932d4f..da71b50d 100644 --- a/golem/managers/activity/pool.py +++ b/golem/managers/activity/pool.py @@ -4,16 +4,21 @@ from typing import Awaitable, Callable from golem.managers.activity.mixins import ActivityPrepareReleaseMixin -from golem.managers.base import ActivityManager, Work, WorkContext, WorkResult +from golem.managers.base import ( + ActivityManager, + ContextManagerLoopMixin, + Work, + WorkContext, + WorkResult, +) from golem.node import GolemNode from golem.resources import Agreement -from golem.utils.asyncio import create_task_with_logging from golem.utils.logging import trace_span logger = logging.getLogger(__name__) -class ActivityPoolManager(ActivityPrepareReleaseMixin, ActivityManager): +class ActivityPoolManager(ContextManagerLoopMixin, ActivityPrepareReleaseMixin, ActivityManager): def __init__( self, golem: GolemNode, @@ -29,39 +34,35 @@ def __init__( self._pool = asyncio.Queue() super().__init__(*args, **kwargs) - @trace_span() - async def start(self): - self._manage_pool_task = create_task_with_logging(self._manage_pool()) - - @trace_span() - async def stop(self): - self._pool_target_size = 0 - # TODO cancel prepare_activity tasks - await self._manage_pool_task - assert self._pool.empty() - - async def _manage_pool(self): + async def _manager_loop(self): pool_current_size = 0 - release_tasks = [] - prepare_tasks = [] - # TODO: After reducing to zero its not possible to manage pool afterwards - while self._pool_target_size > 0 or pool_current_size > 0: - # TODO observe tasks status and add fallback - if pool_current_size > self._pool_target_size: - pool_current_size -= 1 - logger.debug(f"Releasing activity from the pool, new size: {pool_current_size}") - release_tasks.append( - create_task_with_logging(self._release_activity_and_pop_from_pool()) - ) - elif pool_current_size < self._pool_target_size: - pool_current_size += 1 - logger.debug(f"Adding activity to the pool, new size: {pool_current_size}") - prepare_tasks.append( - create_task_with_logging(self._prepare_activity_and_put_in_pool()) - ) - - # TODO: Use events instead of sleep - await asyncio.sleep(0.01) + try: + while True: + if pool_current_size > self._pool_target_size: + # TODO check tasks results and add fallback + await asyncio.gather( + *[ + self._release_activity_and_pop_from_pool() + for _ in range(pool_current_size - self._pool_target_size) + ] + ) + pool_current_size -= pool_current_size - self._pool_target_size + elif pool_current_size < self._pool_target_size: + # TODO check tasks results and add fallback + await asyncio.gather( + *[ + self._prepare_activity_and_put_in_pool() + for _ in range(self._pool_target_size - pool_current_size) + ] + ) + pool_current_size += self._pool_target_size - pool_current_size + # TODO: Use events instead of sleep + await asyncio.sleep(0.01) + finally: + logger.info(f"Releasing all {pool_current_size} activity from the pool") + await asyncio.gather( + *[self._release_activity_and_pop_from_pool() for _ in range(pool_current_size)] + ) @trace_span() async def _release_activity_and_pop_from_pool(self): diff --git a/golem/managers/agreement/scored_aot.py b/golem/managers/agreement/scored_aot.py index 9a5fa76c..6f0c70f0 100644 --- a/golem/managers/agreement/scored_aot.py +++ b/golem/managers/agreement/scored_aot.py @@ -2,12 +2,12 @@ import inspect import logging from datetime import datetime -from typing import Awaitable, Callable, List, Optional, Sequence, Tuple, cast +from typing import Awaitable, Callable, List, Sequence, Tuple, cast from golem.managers.agreement.events import AgreementReleased from golem.managers.base import ( AgreementManager, - ManagerException, + ContextManagerLoopMixin, ManagerPluginsMixin, ManagerPluginWithOptionalWeight, ) @@ -15,14 +15,13 @@ from golem.payload import Properties from golem.payload.parsers.textx import TextXPayloadSyntaxParser from golem.resources import Agreement, Proposal, ProposalData -from golem.utils.asyncio import create_task_with_logging from golem.utils.logging import trace_span logger = logging.getLogger(__name__) class ScoredAheadOfTimeAgreementManager( - ManagerPluginsMixin[ManagerPluginWithOptionalWeight], AgreementManager + ContextManagerLoopMixin, ManagerPluginsMixin[ManagerPluginWithOptionalWeight], AgreementManager ): def __init__( self, @@ -34,7 +33,6 @@ def __init__( self._get_draft_proposal = get_draft_proposal self._event_bus = golem.event_bus - self._consume_proposals_task: Optional[asyncio.Task] = None self._demand_offer_parser = TextXPayloadSyntaxParser() self._scored_proposals: List[Tuple[float, Proposal]] = [] @@ -42,25 +40,7 @@ def __init__( super().__init__(*args, **kwargs) - @trace_span() - async def start(self) -> None: - if self.is_started(): - raise ManagerException("Already started!") - - self._consume_proposals_task = create_task_with_logging(self._consume_proposals()) - - @trace_span() - async def stop(self) -> None: - if not self.is_started(): - raise ManagerException("Already stopped!") - - self._consume_proposals_task.cancel() - self._consume_proposals_task = None - - def is_started(self) -> bool: - return self._consume_proposals_task is not None and not self._consume_proposals_task.done() - - async def _consume_proposals(self) -> None: + async def _manager_loop(self) -> None: while True: proposal = await self._get_draft_proposal() diff --git a/golem/managers/base.py b/golem/managers/base.py index 8b71c3ef..661f85a5 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -1,3 +1,4 @@ +import asyncio from abc import ABC, abstractmethod from dataclasses import dataclass, field from typing import ( @@ -26,6 +27,7 @@ Script, ) from golem.resources.activity import commands +from golem.utils.asyncio import create_task_with_logging from golem.utils.logging import trace_span @@ -123,6 +125,12 @@ class ManagerPluginException(ManagerException): class Manager(ABC): + ... + + +class ContextManagerLoopMixin: + _manager_loop_task: Optional[asyncio.Task] = None + async def __aenter__(self): await self.start() return self @@ -130,10 +138,25 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc, tb): await self.stop() - async def start(self): - ... + @trace_span() + async def start(self) -> None: + if self.is_started(): + raise ManagerException("Already started!") + + self._manager_loop_task = create_task_with_logging(self._manager_loop()) + + @trace_span() + async def stop(self) -> None: + if not self.is_started(): + raise ManagerException("Already stopped!") + + self._manager_loop_task.cancel() + self._manager_loop_task = None + + def is_started(self) -> bool: + return self._manager_loop_task is not None and not self._manager_loop_task.done() - async def stop(self): + async def _manager_loop(self) -> None: ... diff --git a/golem/managers/demand/auto.py b/golem/managers/demand/auto.py index cb3a6059..29f799ff 100644 --- a/golem/managers/demand/auto.py +++ b/golem/managers/demand/auto.py @@ -2,11 +2,11 @@ import inspect import logging from datetime import datetime -from typing import Awaitable, Callable, List, Optional, Sequence, Tuple, cast +from typing import Awaitable, Callable, List, Sequence, Tuple, cast from golem.managers.base import ( + ContextManagerLoopMixin, DemandManager, - ManagerException, ManagerPluginsMixin, ManagerPluginWithOptionalWeight, ) @@ -17,13 +17,14 @@ from golem.resources import Allocation, Proposal, ProposalData from golem.resources.demand.demand_builder import DemandBuilder from golem.resources.proposal.proposal import Proposal -from golem.utils.asyncio import create_task_with_logging from golem.utils.logging import trace_span logger = logging.getLogger(__name__) -class AutoDemandManager(ManagerPluginsMixin[ManagerPluginWithOptionalWeight], DemandManager): +class AutoDemandManager( + ContextManagerLoopMixin, ManagerPluginsMixin[ManagerPluginWithOptionalWeight], DemandManager +): def __init__( self, golem: GolemNode, @@ -41,28 +42,8 @@ def __init__( self._scored_proposals: List[Tuple[float, Proposal]] = [] self._scored_proposals_condition = asyncio.Condition() - self._manager_loop_task: Optional[asyncio.Task] = None - super().__init__(*args, **kwargs) - @trace_span() - async def start(self) -> None: - if self.is_started(): - raise ManagerException("Already started!") - - self._manager_loop_task = create_task_with_logging(self._manager_loop()) - - @trace_span() - async def stop(self) -> None: - if not self.is_started(): - raise ManagerException("Already stopped!") - - self._manager_loop_task.cancel() - self._manager_loop_task = None - - def is_started(self) -> bool: - return self._manager_loop_task is not None and not self._manager_loop_task.done() - async def get_initial_proposal(self) -> Proposal: async with self._scored_proposals_condition: await self._scored_proposals_condition.wait_for(lambda: 0 < len(self._scored_proposals)) diff --git a/golem/managers/negotiation/sequential.py b/golem/managers/negotiation/sequential.py index cf580deb..ff236a7e 100644 --- a/golem/managers/negotiation/sequential.py +++ b/golem/managers/negotiation/sequential.py @@ -7,7 +7,7 @@ from ya_market import ApiException from golem.managers.base import ( - ManagerException, + ContextManagerLoopMixin, ManagerPluginsMixin, NegotiationManager, NegotiationManagerPlugin, @@ -17,14 +17,13 @@ from golem.payload import Properties from golem.payload.parsers.textx import TextXPayloadSyntaxParser from golem.resources import DemandData, Proposal, ProposalData -from golem.utils.asyncio import create_task_with_logging from golem.utils.logging import trace_span logger = logging.getLogger(__name__) class SequentialNegotiationManager( - ManagerPluginsMixin[NegotiationManagerPlugin], NegotiationManager + ContextManagerLoopMixin, ManagerPluginsMixin[NegotiationManagerPlugin], NegotiationManager ): # TODO remove unused methods def __init__( @@ -37,7 +36,6 @@ def __init__( self._golem = golem self._get_initial_proposal = get_initial_proposal - self._negotiation_loop_task: Optional[asyncio.Task] = None self._eligible_proposals: asyncio.Queue[Proposal] = asyncio.Queue() self._demand_offer_parser = TextXPayloadSyntaxParser() @@ -48,25 +46,7 @@ async def get_draft_proposal(self) -> Proposal: return await self._eligible_proposals.get() @trace_span() - async def start(self) -> None: - if self.is_started(): - raise ManagerException("Already started!") - - self._negotiation_loop_task = create_task_with_logging(self._negotiation_loop()) - - @trace_span() - async def stop(self) -> None: - if not self.is_started(): - raise ManagerException("Already stopped!") - - self._negotiation_loop_task.cancel() - self._negotiation_loop_task = None - - def is_started(self) -> bool: - return self._negotiation_loop_task is not None and not self._negotiation_loop_task.done() - - @trace_span() - async def _negotiation_loop(self) -> None: + async def _manager_loop(self) -> None: while True: # TODO add buffer proposal = await self._get_initial_proposal() diff --git a/golem/managers/network/single.py b/golem/managers/network/single.py index 7cef682e..0424245b 100644 --- a/golem/managers/network/single.py +++ b/golem/managers/network/single.py @@ -18,7 +18,7 @@ def __init__(self, golem: GolemNode, ip: str) -> None: self._nodes: Dict[str, str] = {} @trace_span() - async def start(self): + async def __aenter__(self): self._network = await Network.create(self._golem, self._ip, None, None) await self._network.add_requestor_ip(None) self._golem.add_autoclose_resource(self._network) diff --git a/golem/managers/payment/__init__.py b/golem/managers/payment/__init__.py index 3a592759..0acf8649 100644 --- a/golem/managers/payment/__init__.py +++ b/golem/managers/payment/__init__.py @@ -1,3 +1,3 @@ -from golem.managers.payment.default import DefaultPaymentManager +from golem.managers.payment.pay_all import PayAllPaymentManager -__all__ = ("DefaultPaymentManager",) +__all__ = ("PayAllPaymentManager",) diff --git a/golem/managers/payment/pay_all.py b/golem/managers/payment/pay_all.py index c22fef73..310861cf 100644 --- a/golem/managers/payment/pay_all.py +++ b/golem/managers/payment/pay_all.py @@ -41,7 +41,7 @@ def __init__( self._event_handlers = [] @trace_span() - async def start(self) -> None: + async def __aenter__(self): # TODO: Add stop with event_bus.off() self._event_handlers.extend( [ @@ -53,7 +53,7 @@ async def start(self) -> None: ) @trace_span() - async def stop(self) -> None: + async def __aexit__(self, exc_type, exc, tb): await self.wait_for_invoices() for event_handler in self._event_handlers: diff --git a/golem/pipeline/__init__.py b/golem/pipeline/__init__.py index 19624116..40336809 100644 --- a/golem/pipeline/__init__.py +++ b/golem/pipeline/__init__.py @@ -1,5 +1,6 @@ from golem.pipeline.buffer import Buffer from golem.pipeline.chain import Chain +from golem.pipeline.default_payment_handler import DefaultPaymentManager from golem.pipeline.exceptions import InputStreamExhausted from golem.pipeline.limit import Limit from golem.pipeline.map import Map @@ -14,4 +15,5 @@ "Buffer", "Sort", "InputStreamExhausted", + "DefaultPaymentManager", ) diff --git a/golem/managers/payment/default.py b/golem/pipeline/default_payment_handler.py similarity index 100% rename from golem/managers/payment/default.py rename to golem/pipeline/default_payment_handler.py From 216a2351219e49def2d0c285b5076b20bbc71f75 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 13 Jul 2023 11:07:23 +0200 Subject: [PATCH 081/123] Refactor ContextManager Mixin into BackgroundLoopMixin --- golem/managers/activity/pool.py | 12 +++-------- golem/managers/agreement/scored_aot.py | 6 +++--- golem/managers/base.py | 26 ++++++++++++++---------- golem/managers/demand/auto.py | 6 +++--- golem/managers/negotiation/sequential.py | 6 +++--- golem/managers/network/single.py | 2 +- golem/managers/payment/pay_all.py | 4 ++-- 7 files changed, 30 insertions(+), 32 deletions(-) diff --git a/golem/managers/activity/pool.py b/golem/managers/activity/pool.py index da71b50d..a25e8982 100644 --- a/golem/managers/activity/pool.py +++ b/golem/managers/activity/pool.py @@ -4,13 +4,7 @@ from typing import Awaitable, Callable from golem.managers.activity.mixins import ActivityPrepareReleaseMixin -from golem.managers.base import ( - ActivityManager, - ContextManagerLoopMixin, - Work, - WorkContext, - WorkResult, -) +from golem.managers.base import ActivityManager, BackgroundLoopMixin, Work, WorkContext, WorkResult from golem.node import GolemNode from golem.resources import Agreement from golem.utils.logging import trace_span @@ -18,7 +12,7 @@ logger = logging.getLogger(__name__) -class ActivityPoolManager(ContextManagerLoopMixin, ActivityPrepareReleaseMixin, ActivityManager): +class ActivityPoolManager(BackgroundLoopMixin, ActivityPrepareReleaseMixin, ActivityManager): def __init__( self, golem: GolemNode, @@ -34,7 +28,7 @@ def __init__( self._pool = asyncio.Queue() super().__init__(*args, **kwargs) - async def _manager_loop(self): + async def _background_loop(self): pool_current_size = 0 try: while True: diff --git a/golem/managers/agreement/scored_aot.py b/golem/managers/agreement/scored_aot.py index 6f0c70f0..5b3d9f7e 100644 --- a/golem/managers/agreement/scored_aot.py +++ b/golem/managers/agreement/scored_aot.py @@ -7,7 +7,7 @@ from golem.managers.agreement.events import AgreementReleased from golem.managers.base import ( AgreementManager, - ContextManagerLoopMixin, + BackgroundLoopMixin, ManagerPluginsMixin, ManagerPluginWithOptionalWeight, ) @@ -21,7 +21,7 @@ class ScoredAheadOfTimeAgreementManager( - ContextManagerLoopMixin, ManagerPluginsMixin[ManagerPluginWithOptionalWeight], AgreementManager + BackgroundLoopMixin, ManagerPluginsMixin[ManagerPluginWithOptionalWeight], AgreementManager ): def __init__( self, @@ -40,7 +40,7 @@ def __init__( super().__init__(*args, **kwargs) - async def _manager_loop(self) -> None: + async def _background_loop(self) -> None: while True: proposal = await self._get_draft_proposal() diff --git a/golem/managers/base.py b/golem/managers/base.py index 661f85a5..c732cebd 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -125,12 +125,6 @@ class ManagerPluginException(ManagerException): class Manager(ABC): - ... - - -class ContextManagerLoopMixin: - _manager_loop_task: Optional[asyncio.Task] = None - async def __aenter__(self): await self.start() return self @@ -138,25 +132,35 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc, tb): await self.stop() + async def start(self) -> None: + ... + + async def stop(self) -> None: + ... + + +class BackgroundLoopMixin: + _background_loop_task: Optional[asyncio.Task] = None + @trace_span() async def start(self) -> None: if self.is_started(): raise ManagerException("Already started!") - self._manager_loop_task = create_task_with_logging(self._manager_loop()) + self._background_loop_task = create_task_with_logging(self._background_loop()) @trace_span() async def stop(self) -> None: if not self.is_started(): raise ManagerException("Already stopped!") - self._manager_loop_task.cancel() - self._manager_loop_task = None + self._background_loop_task.cancel() + self._background_loop_task = None def is_started(self) -> bool: - return self._manager_loop_task is not None and not self._manager_loop_task.done() + return self._background_loop_task is not None and not self._background_loop_task.done() - async def _manager_loop(self) -> None: + async def _background_loop(self) -> None: ... diff --git a/golem/managers/demand/auto.py b/golem/managers/demand/auto.py index 29f799ff..fdce572b 100644 --- a/golem/managers/demand/auto.py +++ b/golem/managers/demand/auto.py @@ -5,7 +5,7 @@ from typing import Awaitable, Callable, List, Sequence, Tuple, cast from golem.managers.base import ( - ContextManagerLoopMixin, + BackgroundLoopMixin, DemandManager, ManagerPluginsMixin, ManagerPluginWithOptionalWeight, @@ -23,7 +23,7 @@ class AutoDemandManager( - ContextManagerLoopMixin, ManagerPluginsMixin[ManagerPluginWithOptionalWeight], DemandManager + BackgroundLoopMixin, ManagerPluginsMixin[ManagerPluginWithOptionalWeight], DemandManager ): def __init__( self, @@ -53,7 +53,7 @@ async def get_initial_proposal(self) -> Proposal: return proposal @trace_span() - async def _manager_loop(self) -> None: + async def _background_loop(self) -> None: allocation = await self._get_allocation() demand_builder = await self._prepare_demand_builder(allocation) diff --git a/golem/managers/negotiation/sequential.py b/golem/managers/negotiation/sequential.py index ff236a7e..555c1d1b 100644 --- a/golem/managers/negotiation/sequential.py +++ b/golem/managers/negotiation/sequential.py @@ -7,7 +7,7 @@ from ya_market import ApiException from golem.managers.base import ( - ContextManagerLoopMixin, + BackgroundLoopMixin, ManagerPluginsMixin, NegotiationManager, NegotiationManagerPlugin, @@ -23,7 +23,7 @@ class SequentialNegotiationManager( - ContextManagerLoopMixin, ManagerPluginsMixin[NegotiationManagerPlugin], NegotiationManager + BackgroundLoopMixin, ManagerPluginsMixin[NegotiationManagerPlugin], NegotiationManager ): # TODO remove unused methods def __init__( @@ -46,7 +46,7 @@ async def get_draft_proposal(self) -> Proposal: return await self._eligible_proposals.get() @trace_span() - async def _manager_loop(self) -> None: + async def _background_loop(self) -> None: while True: # TODO add buffer proposal = await self._get_initial_proposal() diff --git a/golem/managers/network/single.py b/golem/managers/network/single.py index 0424245b..7cef682e 100644 --- a/golem/managers/network/single.py +++ b/golem/managers/network/single.py @@ -18,7 +18,7 @@ def __init__(self, golem: GolemNode, ip: str) -> None: self._nodes: Dict[str, str] = {} @trace_span() - async def __aenter__(self): + async def start(self): self._network = await Network.create(self._golem, self._ip, None, None) await self._network.add_requestor_ip(None) self._golem.add_autoclose_resource(self._network) diff --git a/golem/managers/payment/pay_all.py b/golem/managers/payment/pay_all.py index 310861cf..07e9ff28 100644 --- a/golem/managers/payment/pay_all.py +++ b/golem/managers/payment/pay_all.py @@ -41,7 +41,7 @@ def __init__( self._event_handlers = [] @trace_span() - async def __aenter__(self): + async def start(self): # TODO: Add stop with event_bus.off() self._event_handlers.extend( [ @@ -53,7 +53,7 @@ async def __aenter__(self): ) @trace_span() - async def __aexit__(self, exc_type, exc, tb): + async def stop(self): await self.wait_for_invoices() for event_handler in self._event_handlers: From e323091bfe006af84fd50f0149323ef640e3a432 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Fri, 14 Jul 2023 14:39:34 +0200 Subject: [PATCH 082/123] Add mixins module; move mixins, add WeightProposalScoringPluginsMixin --- golem/managers/activity/pool.py | 3 +- golem/managers/agreement/scored_aot.py | 151 ++----------------- golem/managers/base.py | 61 +------- golem/managers/demand/auto.py | 139 ++--------------- golem/managers/mixins.py | 180 +++++++++++++++++++++++ golem/managers/negotiation/sequential.py | 9 +- golem/managers/work/mixins.py | 2 +- 7 files changed, 212 insertions(+), 333 deletions(-) create mode 100644 golem/managers/mixins.py diff --git a/golem/managers/activity/pool.py b/golem/managers/activity/pool.py index a25e8982..21bed377 100644 --- a/golem/managers/activity/pool.py +++ b/golem/managers/activity/pool.py @@ -4,7 +4,8 @@ from typing import Awaitable, Callable from golem.managers.activity.mixins import ActivityPrepareReleaseMixin -from golem.managers.base import ActivityManager, BackgroundLoopMixin, Work, WorkContext, WorkResult +from golem.managers.base import ActivityManager, Work, WorkContext, WorkResult +from golem.managers.mixins import BackgroundLoopMixin from golem.node import GolemNode from golem.resources import Agreement from golem.utils.logging import trace_span diff --git a/golem/managers/agreement/scored_aot.py b/golem/managers/agreement/scored_aot.py index 5b3d9f7e..ae232912 100644 --- a/golem/managers/agreement/scored_aot.py +++ b/golem/managers/agreement/scored_aot.py @@ -1,27 +1,18 @@ -import asyncio -import inspect import logging -from datetime import datetime -from typing import Awaitable, Callable, List, Sequence, Tuple, cast +from typing import Awaitable, Callable from golem.managers.agreement.events import AgreementReleased -from golem.managers.base import ( - AgreementManager, - BackgroundLoopMixin, - ManagerPluginsMixin, - ManagerPluginWithOptionalWeight, -) +from golem.managers.base import AgreementManager +from golem.managers.mixins import BackgroundLoopMixin, WeightProposalScoringPluginsMixin from golem.node import GolemNode -from golem.payload import Properties -from golem.payload.parsers.textx import TextXPayloadSyntaxParser -from golem.resources import Agreement, Proposal, ProposalData +from golem.resources import Agreement, Proposal from golem.utils.logging import trace_span logger = logging.getLogger(__name__) class ScoredAheadOfTimeAgreementManager( - BackgroundLoopMixin, ManagerPluginsMixin[ManagerPluginWithOptionalWeight], AgreementManager + BackgroundLoopMixin, WeightProposalScoringPluginsMixin, AgreementManager ): def __init__( self, @@ -33,44 +24,12 @@ def __init__( self._get_draft_proposal = get_draft_proposal self._event_bus = golem.event_bus - self._demand_offer_parser = TextXPayloadSyntaxParser() - - self._scored_proposals: List[Tuple[float, Proposal]] = [] - self._scored_proposals_condition = asyncio.Condition() - super().__init__(*args, **kwargs) - async def _background_loop(self) -> None: - while True: - proposal = await self._get_draft_proposal() - - await self._manage_scoring(proposal) - - @trace_span() - async def _manage_scoring(self, proposal: Proposal) -> None: - async with self._scored_proposals_condition: - all_proposals = list(sp[1] for sp in self._scored_proposals) - all_proposals.append(proposal) - - self._scored_proposals = await self._do_scoring(all_proposals) - - self._scored_proposals_condition.notify_all() - - @trace_span() - async def _get_scored_proposal(self): - async with self._scored_proposals_condition: - await self._scored_proposals_condition.wait_for(lambda: 0 < len(self._scored_proposals)) - - score, proposal = self._scored_proposals.pop(0) - - logger.info(f"Proposal `{proposal}` picked with score `{score}`") - - return proposal - @trace_span() async def get_agreement(self) -> Agreement: while True: - proposal = await self._get_scored_proposal() + proposal = await self.get_scored_proposal() try: agreement = await proposal.create_agreement() await agreement.confirm() @@ -88,100 +47,14 @@ async def get_agreement(self) -> Agreement: ) return agreement + async def _background_loop(self) -> None: + while True: + proposal = await self._get_draft_proposal() + + await self.manage_scoring(proposal) + @trace_span() async def _terminate_agreement(self, event: AgreementReleased) -> None: agreement: Agreement = event.resource await agreement.terminate() logger.info(f"Agreement `{agreement}` closed") - - async def _do_scoring(self, proposals: Sequence[Proposal]): - proposals_data = await self._get_proposals_data_from_proposals(proposals) - proposal_scores = await self._run_plugins(proposals_data) - - scored_proposals = self._calculate_proposal_score(proposals, proposal_scores) - scored_proposals.sort(key=lambda x: x[0], reverse=True) - - return scored_proposals - - @trace_span() - async def _run_plugins( - self, proposals_data: Sequence[ProposalData] - ) -> Sequence[Tuple[float, Sequence[float]]]: - proposal_scores = [] - - for plugin in self._plugins: - if isinstance(plugin, (list, tuple)): - weight, plugin = plugin - else: - weight = 1 - - plugin_scores = plugin(proposals_data) - - if inspect.isawaitable(plugin_scores): - plugin_scores = await plugin_scores - - proposal_scores.append((weight, plugin_scores)) - - return proposal_scores - - def _calculate_proposal_score( - self, - proposals: Sequence[Proposal], - plugin_scores: Sequence[Tuple[float, Sequence[float]]], - ) -> List[Tuple[float, Proposal]]: - # FIXME: can this be refactored? - return [ - ( - self._calculate_weighted_score( - self._transpose_plugin_scores(proposal_index, plugin_scores) - ), - proposal, - ) - for proposal_index, proposal in enumerate(proposals) - ] - - def _transpose_plugin_scores( - self, proposal_index: int, plugin_scores: Sequence[Tuple[float, Sequence[float]]] - ) -> Sequence[Tuple[float, float]]: - # FIXME: can this be refactored? - return [ - (plugin_weight, plugin_scores[proposal_index]) - for plugin_weight, plugin_scores in plugin_scores - if plugin_scores[proposal_index] is None - ] - - def _calculate_weighted_score( - self, proposal_weighted_scores: Sequence[Tuple[float, float]] - ) -> float: - if not proposal_weighted_scores: - return 0 - - weighted_sum = sum(pws[0] * pws[1] for pws in proposal_weighted_scores) - weights_sum = sum(pws[0] for pws in proposal_weighted_scores) - - return weighted_sum / weights_sum - - # FIXME: This should be already provided by low level - async def _get_proposals_data_from_proposals( - self, proposals: Sequence[Proposal] - ) -> Sequence[ProposalData]: - result = [] - - for proposal in proposals: - data = await proposal.get_data() - - constraints = self._demand_offer_parser.parse_constraints(data.constraints) - - result.append( - ProposalData( - properties=Properties(data.properties), - constraints=constraints, - proposal_id=data.proposal_id, - issuer_id=data.issuer_id, - state=data.state, - timestamp=cast(datetime, data.timestamp), - prev_proposal_id=data.prev_proposal_id, - ) - ) - - return result diff --git a/golem/managers/base.py b/golem/managers/base.py index c732cebd..814c6dfe 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -1,19 +1,7 @@ -import asyncio +import logging from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import ( - Any, - Awaitable, - Callable, - Dict, - Generic, - List, - Optional, - Sequence, - Tuple, - TypeVar, - Union, -) +from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, Tuple, TypeVar, Union from golem.exceptions import GolemException from golem.resources import ( @@ -27,8 +15,9 @@ Script, ) from golem.resources.activity import commands -from golem.utils.asyncio import create_task_with_logging -from golem.utils.logging import trace_span +from golem.resources.proposal.proposal import Proposal + +logger = logging.getLogger(__name__) class Batch: @@ -139,49 +128,9 @@ async def stop(self) -> None: ... -class BackgroundLoopMixin: - _background_loop_task: Optional[asyncio.Task] = None - - @trace_span() - async def start(self) -> None: - if self.is_started(): - raise ManagerException("Already started!") - - self._background_loop_task = create_task_with_logging(self._background_loop()) - - @trace_span() - async def stop(self) -> None: - if not self.is_started(): - raise ManagerException("Already stopped!") - - self._background_loop_task.cancel() - self._background_loop_task = None - - def is_started(self) -> bool: - return self._background_loop_task is not None and not self._background_loop_task.done() - - async def _background_loop(self) -> None: - ... - - TPlugin = TypeVar("TPlugin") -class ManagerPluginsMixin(Generic[TPlugin]): - def __init__(self, plugins: Optional[Sequence[TPlugin]] = None, *args, **kwargs) -> None: - self._plugins: List[TPlugin] = list(plugins) if plugins is not None else [] - - super().__init__(*args, **kwargs) - - @trace_span() - def register_plugin(self, plugin: TPlugin): - self._plugins.append(plugin) - - @trace_span() - def unregister_plugin(self, plugin: TPlugin): - self._plugins.remove(plugin) - - class NetworkManager(Manager, ABC): ... diff --git a/golem/managers/demand/auto.py b/golem/managers/demand/auto.py index fdce572b..1a71450c 100644 --- a/golem/managers/demand/auto.py +++ b/golem/managers/demand/auto.py @@ -1,20 +1,12 @@ -import asyncio -import inspect import logging -from datetime import datetime -from typing import Awaitable, Callable, List, Sequence, Tuple, cast +from typing import Awaitable, Callable -from golem.managers.base import ( - BackgroundLoopMixin, - DemandManager, - ManagerPluginsMixin, - ManagerPluginWithOptionalWeight, -) +from golem.managers.base import DemandManager +from golem.managers.mixins import BackgroundLoopMixin, WeightProposalScoringPluginsMixin from golem.node import GolemNode from golem.node.node import GolemNode -from golem.payload import Payload, Properties -from golem.payload.parsers.textx.parser import TextXPayloadSyntaxParser -from golem.resources import Allocation, Proposal, ProposalData +from golem.payload import Payload +from golem.resources import Allocation, Proposal from golem.resources.demand.demand_builder import DemandBuilder from golem.resources.proposal.proposal import Proposal from golem.utils.logging import trace_span @@ -22,9 +14,7 @@ logger = logging.getLogger(__name__) -class AutoDemandManager( - BackgroundLoopMixin, ManagerPluginsMixin[ManagerPluginWithOptionalWeight], DemandManager -): +class AutoDemandManager(BackgroundLoopMixin, WeightProposalScoringPluginsMixin, DemandManager): def __init__( self, golem: GolemNode, @@ -37,20 +27,11 @@ def __init__( self._get_allocation = get_allocation self._payload = payload - self._demand_offer_parser = TextXPayloadSyntaxParser() - - self._scored_proposals: List[Tuple[float, Proposal]] = [] - self._scored_proposals_condition = asyncio.Condition() - super().__init__(*args, **kwargs) + @trace_span() async def get_initial_proposal(self) -> Proposal: - async with self._scored_proposals_condition: - await self._scored_proposals_condition.wait_for(lambda: 0 < len(self._scored_proposals)) - - score, proposal = self._scored_proposals.pop(0) - - return proposal + return await self.get_scored_proposal() @trace_span() async def _background_loop(self) -> None: @@ -62,10 +43,11 @@ async def _background_loop(self) -> None: try: async for initial_proposal in demand.initial_proposals(): - await self._manage_scoring(initial_proposal) + await self.manage_scoring(initial_proposal) finally: await demand.unsubscribe() + @trace_span() async def _prepare_demand_builder(self, allocation: Allocation) -> DemandBuilder: # FIXME: Code looks duplicated as GolemNode.create_demand does the same demand_builder = DemandBuilder() @@ -77,104 +59,3 @@ async def _prepare_demand_builder(self, allocation: Allocation) -> DemandBuilder await demand_builder.add(self._payload) return demand_builder - - @trace_span() - async def _manage_scoring(self, proposal: Proposal) -> None: - async with self._scored_proposals_condition: - all_proposals = list(sp[1] for sp in self._scored_proposals) - all_proposals.append(proposal) - - self._scored_proposals = await self._do_scoring(all_proposals) - - self._scored_proposals_condition.notify_all() - - async def _do_scoring(self, proposals: Sequence[Proposal]): - proposals_data = await self._get_proposals_data_from_proposals(proposals) - proposal_scores = await self._run_plugins(proposals_data) - - scored_proposals = self._calculate_proposal_score(proposals, proposal_scores) - scored_proposals.sort(key=lambda x: x[0], reverse=True) - - return scored_proposals - - @trace_span() - async def _run_plugins( - self, proposals_data: Sequence[ProposalData] - ) -> Sequence[Tuple[float, Sequence[float]]]: - proposal_scores = [] - - for plugin in self._plugins: - if isinstance(plugin, (list, tuple)): - weight, plugin = plugin - else: - weight = 1 - - plugin_scores = plugin(proposals_data) - - if inspect.isawaitable(plugin_scores): - plugin_scores = await plugin_scores - - proposal_scores.append((weight, plugin_scores)) - - return proposal_scores - - def _calculate_proposal_score( - self, - proposals: Sequence[Proposal], - plugin_scores: Sequence[Tuple[float, Sequence[float]]], - ) -> List[Tuple[float, Proposal]]: - # FIXME: can this be refactored? - return [ - ( - self._calculate_weighted_score( - self._transpose_plugin_scores(proposal_index, plugin_scores) - ), - proposal, - ) - for proposal_index, proposal in enumerate(proposals) - ] - - def _calculate_weighted_score( - self, proposal_weighted_scores: Sequence[Tuple[float, float]] - ) -> float: - if not proposal_weighted_scores: - return 0 - - weighted_sum = sum(pws[0] * pws[1] for pws in proposal_weighted_scores) - weights_sum = sum(pws[0] for pws in proposal_weighted_scores) - - return weighted_sum / weights_sum - - def _transpose_plugin_scores( - self, proposal_index: int, plugin_scores: Sequence[Tuple[float, Sequence[float]]] - ) -> Sequence[Tuple[float, float]]: - # FIXME: can this be refactored? - return [ - (plugin_weight, plugin_scores[proposal_index]) - for plugin_weight, plugin_scores in plugin_scores - if plugin_scores[proposal_index] is None - ] - - async def _get_proposals_data_from_proposals( - self, proposals: Sequence[Proposal] - ) -> Sequence[ProposalData]: - result = [] - - for proposal in proposals: - data = await proposal.get_data() - - constraints = self._demand_offer_parser.parse_constraints(data.constraints) - - result.append( - ProposalData( - properties=Properties(data.properties), - constraints=constraints, - proposal_id=data.proposal_id, - issuer_id=data.issuer_id, - state=data.state, - timestamp=cast(datetime, data.timestamp), - prev_proposal_id=data.prev_proposal_id, - ) - ) - - return result diff --git a/golem/managers/mixins.py b/golem/managers/mixins.py new file mode 100644 index 00000000..966118a7 --- /dev/null +++ b/golem/managers/mixins.py @@ -0,0 +1,180 @@ +import asyncio +import inspect +import logging +from datetime import datetime +from typing import Generic, List, Optional, Sequence, Tuple, cast + +from golem.managers.base import ManagerException, ManagerPluginWithOptionalWeight, TPlugin +from golem.payload import Properties +from golem.payload.parsers.base import PayloadSyntaxParser +from golem.payload.parsers.textx.parser import TextXPayloadSyntaxParser +from golem.resources import Proposal, ProposalData +from golem.resources.proposal.proposal import Proposal +from golem.utils.asyncio import create_task_with_logging +from golem.utils.logging import trace_span + +logger = logging.getLogger(__name__) + + +class BackgroundLoopMixin: + _background_loop_task: Optional[asyncio.Task] = None + + @trace_span() + async def start(self) -> None: + if self.is_started(): + raise ManagerException("Already started!") + + self._background_loop_task = create_task_with_logging(self._background_loop()) + + @trace_span() + async def stop(self) -> None: + if not self.is_started(): + raise ManagerException("Already stopped!") + + self._background_loop_task.cancel() + self._background_loop_task = None + + def is_started(self) -> bool: + return self._background_loop_task is not None and not self._background_loop_task.done() + + async def _background_loop(self) -> None: + ... + + +class ManagerPluginsMixin(Generic[TPlugin]): + def __init__(self, plugins: Optional[Sequence[TPlugin]] = None, *args, **kwargs) -> None: + self._plugins: List[TPlugin] = list(plugins) if plugins is not None else [] + + super().__init__(*args, **kwargs) + + @trace_span() + def register_plugin(self, plugin: TPlugin): + self._plugins.append(plugin) + + @trace_span() + def unregister_plugin(self, plugin: TPlugin): + self._plugins.remove(plugin) + + +class WeightProposalScoringPluginsMixin(ManagerPluginsMixin[ManagerPluginWithOptionalWeight]): + def __init__( + self, demand_offer_parser: Optional[PayloadSyntaxParser] = None, *args, **kwargs + ) -> None: + self._demand_offer_parser = demand_offer_parser or TextXPayloadSyntaxParser() + + self._scored_proposals: List[Tuple[float, Proposal]] = [] + self._scored_proposals_condition = asyncio.Condition() + super().__init__(*args, **kwargs) + + @trace_span() + async def manage_scoring(self, proposal: Proposal) -> None: + async with self._scored_proposals_condition: + all_proposals = list(sp[1] for sp in self._scored_proposals) + all_proposals.append(proposal) + + self._scored_proposals = await self._do_scoring(all_proposals) + + self._scored_proposals_condition.notify_all() + + @trace_span() + async def get_scored_proposal(self): + async with self._scored_proposals_condition: + await self._scored_proposals_condition.wait_for(lambda: 0 < len(self._scored_proposals)) + + score, proposal = self._scored_proposals.pop(0) + + logger.info(f"Proposal `{proposal}` picked with score `{score}`") + + return proposal + + async def _do_scoring(self, proposals: Sequence[Proposal]): + proposals_data = await self._get_proposals_data_from_proposals(proposals) + proposal_scores = await self._run_plugins(proposals_data) + + scored_proposals = self._calculate_proposal_score(proposals, proposal_scores) + scored_proposals.sort(key=lambda x: x[0], reverse=True) + + return scored_proposals + + @trace_span() + async def _run_plugins( + self, proposals_data: Sequence[ProposalData] + ) -> Sequence[Tuple[float, Sequence[float]]]: + proposal_scores = [] + + for plugin in self._plugins: + if isinstance(plugin, (list, tuple)): + weight, plugin = plugin + else: + weight = 1 + + plugin_scores = plugin(proposals_data) + + if inspect.isawaitable(plugin_scores): + plugin_scores = await plugin_scores + + proposal_scores.append((weight, plugin_scores)) + + return proposal_scores + + def _calculate_proposal_score( + self, + proposals: Sequence[Proposal], + plugin_scores: Sequence[Tuple[float, Sequence[float]]], + ) -> List[Tuple[float, Proposal]]: + # FIXME: can this be refactored? + return [ + ( + self._calculate_weighted_score( + self._transpose_plugin_scores(proposal_index, plugin_scores) + ), + proposal, + ) + for proposal_index, proposal in enumerate(proposals) + ] + + def _calculate_weighted_score( + self, proposal_weighted_scores: Sequence[Tuple[float, float]] + ) -> float: + if not proposal_weighted_scores: + return 0 + + weighted_sum = sum(pws[0] * pws[1] for pws in proposal_weighted_scores) + weights_sum = sum(pws[0] for pws in proposal_weighted_scores) + + return weighted_sum / weights_sum + + def _transpose_plugin_scores( + self, proposal_index: int, plugin_scores: Sequence[Tuple[float, Sequence[float]]] + ) -> Sequence[Tuple[float, float]]: + # FIXME: can this be refactored? + return [ + (plugin_weight, plugin_scores[proposal_index]) + for plugin_weight, plugin_scores in plugin_scores + if plugin_scores[proposal_index] is None + ] + + # FIXME: This should be already provided by low level + async def _get_proposals_data_from_proposals( + self, proposals: Sequence[Proposal] + ) -> Sequence[ProposalData]: + result = [] + + for proposal in proposals: + data = await proposal.get_data() + + constraints = self._demand_offer_parser.parse_constraints(data.constraints) + + result.append( + ProposalData( + properties=Properties(data.properties), + constraints=constraints, + proposal_id=data.proposal_id, + issuer_id=data.issuer_id, + state=data.state, + timestamp=cast(datetime, data.timestamp), + prev_proposal_id=data.prev_proposal_id, + ) + ) + + return result diff --git a/golem/managers/negotiation/sequential.py b/golem/managers/negotiation/sequential.py index 555c1d1b..f1a9f07a 100644 --- a/golem/managers/negotiation/sequential.py +++ b/golem/managers/negotiation/sequential.py @@ -6,13 +6,8 @@ from ya_market import ApiException -from golem.managers.base import ( - BackgroundLoopMixin, - ManagerPluginsMixin, - NegotiationManager, - NegotiationManagerPlugin, - RejectProposal, -) +from golem.managers.base import NegotiationManager, NegotiationManagerPlugin, RejectProposal +from golem.managers.mixins import BackgroundLoopMixin, ManagerPluginsMixin from golem.node import GolemNode from golem.payload import Properties from golem.payload.parsers.textx import TextXPayloadSyntaxParser diff --git a/golem/managers/work/mixins.py b/golem/managers/work/mixins.py index 3f205e7e..650aa354 100644 --- a/golem/managers/work/mixins.py +++ b/golem/managers/work/mixins.py @@ -3,11 +3,11 @@ from golem.managers.base import ( WORK_PLUGIN_FIELD_NAME, DoWorkCallable, - ManagerPluginsMixin, Work, WorkManagerPlugin, WorkResult, ) +from golem.managers.mixins import ManagerPluginsMixin from golem.utils.logging import trace_span logger = logging.getLogger(__name__) From 8bda0cf242e612b899646582a44631acb0e02a87 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 17 Jul 2023 13:10:14 +0200 Subject: [PATCH 083/123] Rename pipeline DefaultPaymentManager to DefaultPaymentHandler --- docs/sphinx/api.rst | 6 ------ examples/exception_handling/exception_handling.py | 8 ++++---- examples/rate_providers/rate_providers.py | 8 ++++---- .../score_based_providers/score_based_providers.py | 8 ++++---- examples/task_api_draft/examples/pipeline_example.py | 8 ++++---- examples/task_api_draft/examples/yacat.py | 8 ++++---- examples/task_api_draft/task_api/execute_tasks.py | 8 ++++---- golem/pipeline/__init__.py | 4 ++-- golem/pipeline/default_payment_handler.py | 12 ++++++------ 9 files changed, 32 insertions(+), 38 deletions(-) diff --git a/docs/sphinx/api.rst b/docs/sphinx/api.rst index 6dc33ef2..1d4bb739 100644 --- a/docs/sphinx/api.rst +++ b/docs/sphinx/api.rst @@ -164,9 +164,3 @@ Logging .. autoclass:: golem.utils.logging.DefaultLogger :members: __init__, file_name, logger, on_event - -Managers -======== - -.. autoclass:: golem.managers.DefaultPaymentManager - :members: __init__, terminate_agreements, wait_for_invoices diff --git a/examples/exception_handling/exception_handling.py b/examples/exception_handling/exception_handling.py index 868b497d..91f5359a 100644 --- a/examples/exception_handling/exception_handling.py +++ b/examples/exception_handling/exception_handling.py @@ -4,7 +4,7 @@ from golem.event_bus import Event from golem.node import GolemNode from golem.payload import RepositoryVmPayload -from golem.pipeline import Buffer, Chain, DefaultPaymentManager, Limit, Map +from golem.pipeline import Buffer, Chain, DefaultPaymentHandler, Limit, Map from golem.resources import ( BatchError, BatchTimeoutError, @@ -47,7 +47,7 @@ async def main() -> None: async with golem: allocation = await golem.create_allocation(1.0) - payment_manager = DefaultPaymentManager(golem, allocation) + payment_handler = DefaultPaymentHandler(golem, allocation) demand = await golem.create_demand(PAYLOAD, allocations=[allocation]) chain = Chain( @@ -64,8 +64,8 @@ async def main() -> None: print(f"Finished with {result}") print("TASK DONE") - await payment_manager.terminate_agreements() - await payment_manager.wait_for_invoices() + await payment_handler.terminate_agreements() + await payment_handler.wait_for_invoices() if __name__ == "__main__": diff --git a/examples/rate_providers/rate_providers.py b/examples/rate_providers/rate_providers.py index 1b86a31a..885b93e4 100644 --- a/examples/rate_providers/rate_providers.py +++ b/examples/rate_providers/rate_providers.py @@ -7,7 +7,7 @@ from golem.event_bus import Event from golem.node import GolemNode from golem.payload import RepositoryVmPayload -from golem.pipeline import Buffer, Chain, DefaultPaymentManager, Limit, Map +from golem.pipeline import Buffer, Chain, DefaultPaymentHandler, Limit, Map from golem.resources import ( Proposal, default_create_activity, @@ -86,7 +86,7 @@ async def main() -> None: async with golem: allocation = await golem.create_allocation(1.0) - payment_manager = DefaultPaymentManager(golem, allocation) + payment_handler = DefaultPaymentHandler(golem, allocation) demand = await golem.create_demand(PAYLOAD, allocations=[allocation]) # `set_no_more_children` has to be called so `initial_proposals` will eventually stop @@ -111,8 +111,8 @@ async def main() -> None: print(f"{[(s, scores[s]) for s in scores if scores[s] is not None]}") print("TASK DONE") - await payment_manager.terminate_agreements() - await payment_manager.wait_for_invoices() + await payment_handler.terminate_agreements() + await payment_handler.wait_for_invoices() if __name__ == "__main__": diff --git a/examples/score_based_providers/score_based_providers.py b/examples/score_based_providers/score_based_providers.py index 0afd0362..4f7d51cb 100644 --- a/examples/score_based_providers/score_based_providers.py +++ b/examples/score_based_providers/score_based_providers.py @@ -5,7 +5,7 @@ from golem.event_bus import Event from golem.node import GolemNode from golem.payload import RepositoryVmPayload -from golem.pipeline import Buffer, Chain, DefaultPaymentManager, Limit, Map, Sort +from golem.pipeline import Buffer, Chain, DefaultPaymentHandler, Limit, Map, Sort from golem.resources import ( Proposal, default_create_activity, @@ -100,7 +100,7 @@ async def main() -> None: async with golem: allocation = await golem.create_allocation(1) - payment_manager = DefaultPaymentManager(golem, allocation) + payment_handler = DefaultPaymentHandler(golem, allocation) demand = await golem.create_demand(PAYLOAD, allocations=[allocation]) chain = Chain( @@ -123,8 +123,8 @@ async def main() -> None: print(f"RESULT: {result}") print("TASK DONE") - await payment_manager.terminate_agreements() - await payment_manager.wait_for_invoices() + await payment_handler.terminate_agreements() + await payment_handler.wait_for_invoices() if __name__ == "__main__": diff --git a/examples/task_api_draft/examples/pipeline_example.py b/examples/task_api_draft/examples/pipeline_example.py index 5585693f..a8bb7f6b 100644 --- a/examples/task_api_draft/examples/pipeline_example.py +++ b/examples/task_api_draft/examples/pipeline_example.py @@ -7,7 +7,7 @@ from golem.events_bus import Event from golem.node import GolemNode from golem.payload import RepositoryVmPayload -from golem.pipeline import Buffer, Chain, DefaultPaymentManager, Map, Sort, Zip +from golem.pipeline import Buffer, Chain, DefaultPaymentHandler, Map, Sort, Zip from golem.resources import ( Proposal, default_create_activity, @@ -73,7 +73,7 @@ async def main() -> None: async with golem: allocation = await golem.create_allocation(1) - payment_manager = DefaultPaymentManager(golem, allocation) + payment_handler = DefaultPaymentHandler(golem, allocation) demand = await golem.create_demand(PAYLOAD, allocations=[allocation]) @@ -106,8 +106,8 @@ async def task_stream() -> AsyncIterator[int]: print("ALL TASKS DONE") - await payment_manager.terminate_agreements() - await payment_manager.wait_for_invoices() + await payment_handler.terminate_agreements() + await payment_handler.wait_for_invoices() if __name__ == "__main__": diff --git a/examples/task_api_draft/examples/yacat.py b/examples/task_api_draft/examples/yacat.py index 65eb0744..a24a8bac 100644 --- a/examples/task_api_draft/examples/yacat.py +++ b/examples/task_api_draft/examples/yacat.py @@ -25,7 +25,7 @@ from examples.task_api_draft.task_api.activity_pool import ActivityPool from golem.event_bus import Event from golem.node import GolemNode -from golem.pipeline import Buffer, Chain, DefaultPaymentManager, Map, Sort, Zip +from golem.pipeline import Buffer, Chain, DefaultPaymentHandler, Map, Sort, Zip from golem.resources import ( Activity, DebitNote, @@ -189,7 +189,7 @@ async def main() -> None: async with golem: allocation = await golem.create_allocation(amount=1) - payment_manager = DefaultPaymentManager(golem, allocation) + payment_handler = DefaultPaymentHandler(golem, allocation) demand = await golem.create_demand(PAYLOAD, allocations=[allocation]) @@ -211,8 +211,8 @@ async def main() -> None: ): pass except asyncio.CancelledError: - await payment_manager.terminate_agreements() - await payment_manager.wait_for_invoices() + await payment_handler.terminate_agreements() + await payment_handler.wait_for_invoices() # TODO: This removes the "destroyed but pending" messages, probably there's # some even prettier solution available? diff --git a/examples/task_api_draft/task_api/execute_tasks.py b/examples/task_api_draft/task_api/execute_tasks.py index a6fcbb69..d56f77ca 100644 --- a/examples/task_api_draft/task_api/execute_tasks.py +++ b/examples/task_api_draft/task_api/execute_tasks.py @@ -5,7 +5,7 @@ from golem.event_bus import Event from golem.node import GolemNode from golem.payload import Payload -from golem.pipeline import Buffer, Chain, DefaultPaymentManager, Map, Sort, Zip +from golem.pipeline import Buffer, Chain, DefaultPaymentHandler, Map, Sort, Zip from golem.resources import ( Activity, Demand, @@ -149,7 +149,7 @@ async def execute_tasks( async with golem: allocation = await golem.create_allocation(budget) - payment_manager = DefaultPaymentManager(golem, allocation) + payment_handler = DefaultPaymentHandler(golem, allocation) demand = await golem.create_demand(payload, allocations=[allocation]) chain = get_chain( @@ -169,5 +169,5 @@ async def execute_tasks( if task_stream.in_stream_empty and returned == task_stream.task_cnt: break - await payment_manager.terminate_agreements() - await payment_manager.wait_for_invoices() + await payment_handler.terminate_agreements() + await payment_handler.wait_for_invoices() diff --git a/golem/pipeline/__init__.py b/golem/pipeline/__init__.py index 40336809..2c55b562 100644 --- a/golem/pipeline/__init__.py +++ b/golem/pipeline/__init__.py @@ -1,6 +1,6 @@ from golem.pipeline.buffer import Buffer from golem.pipeline.chain import Chain -from golem.pipeline.default_payment_handler import DefaultPaymentManager +from golem.pipeline.default_payment_handler import DefaultPaymentHandler from golem.pipeline.exceptions import InputStreamExhausted from golem.pipeline.limit import Limit from golem.pipeline.map import Map @@ -15,5 +15,5 @@ "Buffer", "Sort", "InputStreamExhausted", - "DefaultPaymentManager", + "DefaultPaymentHandler", ) diff --git a/golem/pipeline/default_payment_handler.py b/golem/pipeline/default_payment_handler.py index 441c445c..192d7cff 100644 --- a/golem/pipeline/default_payment_handler.py +++ b/golem/pipeline/default_payment_handler.py @@ -9,7 +9,7 @@ from golem.resources import Allocation, NewAgreement -class DefaultPaymentManager: +class DefaultPaymentHandler: """Accepts all incoming debit_notes and invoices. Calls `get_data(force=True)` on invoices/debit notes after they are accepted, @@ -19,21 +19,21 @@ class DefaultPaymentManager: async with GolemNode() as golem: allocation = await golem.create_allocation(BUDGET) - payment_manager = DefaultPaymentManager(golem, allocation) + payment_handler = DefaultPaymentHandler(golem, allocation) try: # ... interact with the Golem Network ... finally: - await payment_manager.terminate_agreements() - await payment_manager.wait_for_invoices() + await payment_handler.terminate_agreements() + await payment_handler.wait_for_invoices() """ def __init__(self, node: "GolemNode", allocation: "Allocation"): - """Init DefaultPaymentManager. + """Init DefaultPaymentHandler. :param node: Debit notes/invoices received by this node will be accepted. - :any:`DefaultPaymentManager` will only work if the :any:`GolemNode` was started with + :any:`DefaultPaymentHandler` will only work if the :any:`GolemNode` was started with `collect_payment_events = True`. :param allocation: Allocation that will be used to accept debit notes/invoices. """ From de54e9d2879df3a8c853347384f51a2e494bfe47 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 17 Jul 2023 13:18:03 +0200 Subject: [PATCH 084/123] Fix checks_codestyle --- examples/managers/blender/blender.py | 2 +- examples/managers/ssh.py | 2 +- golem/managers/__init__.py | 1 - golem/managers/agreement/pricings.py | 3 ++- golem/managers/base.py | 1 - golem/managers/demand/auto.py | 2 -- golem/managers/mixins.py | 1 - golem/managers/work/plugins.py | 3 ++- 8 files changed, 6 insertions(+), 9 deletions(-) diff --git a/examples/managers/blender/blender.py b/examples/managers/blender/blender.py index 42276f31..cc594460 100644 --- a/examples/managers/blender/blender.py +++ b/examples/managers/blender/blender.py @@ -58,7 +58,7 @@ async def run_on_golem( ) work_manager = AsynchronousWorkManager(golem, activity_manager.do_work, plugins=task_plugins) - async with golem, payment_manager, demand_manager, negotiation_manager, agreement_manager, activity_manager: + async with golem, payment_manager, demand_manager, negotiation_manager, agreement_manager, activity_manager: # noqa: E501 line too long results: List[WorkResult] = await work_manager.do_work_list(task_list) return results diff --git a/examples/managers/ssh.py b/examples/managers/ssh.py index 29052c45..888702e0 100644 --- a/examples/managers/ssh.py +++ b/examples/managers/ssh.py @@ -87,7 +87,7 @@ async def main(): ) work_manager = SequentialWorkManager(golem, activity_manager.do_work) # TODO use different managers so it allows to finish work func without destroying activity - async with golem, network_manager, payment_manager, demand_manager, negotiation_manager, agreement_manager: + async with golem, network_manager, payment_manager, demand_manager, negotiation_manager, agreement_manager: # noqa: E501 line too long result: WorkResult = await work_manager.do_work( work(golem._api_config.app_key, network_manager.get_provider_uri) ) diff --git a/golem/managers/__init__.py b/golem/managers/__init__.py index 8b137891..e69de29b 100644 --- a/golem/managers/__init__.py +++ b/golem/managers/__init__.py @@ -1 +0,0 @@ - diff --git a/golem/managers/agreement/pricings.py b/golem/managers/agreement/pricings.py index 7cbcb288..954b2d8a 100644 --- a/golem/managers/agreement/pricings.py +++ b/golem/managers/agreement/pricings.py @@ -34,7 +34,8 @@ def _get_linear_coeffs( if not (isinstance(coeffs, (list, tuple)) and len(coeffs) == 3): logging.debug( - f"Proposal `{proposal_data.proposal_id}` linear pricing coeffs must be a 3 element sequence, ignoring" + f"Proposal `{proposal_data.proposal_id}` linear pricing coeffs must be a 3 element" + "sequence, ignoring" ) return None diff --git a/golem/managers/base.py b/golem/managers/base.py index 814c6dfe..9a9e3fbe 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -15,7 +15,6 @@ Script, ) from golem.resources.activity import commands -from golem.resources.proposal.proposal import Proposal logger = logging.getLogger(__name__) diff --git a/golem/managers/demand/auto.py b/golem/managers/demand/auto.py index 1a71450c..dd6e40d9 100644 --- a/golem/managers/demand/auto.py +++ b/golem/managers/demand/auto.py @@ -4,11 +4,9 @@ from golem.managers.base import DemandManager from golem.managers.mixins import BackgroundLoopMixin, WeightProposalScoringPluginsMixin from golem.node import GolemNode -from golem.node.node import GolemNode from golem.payload import Payload from golem.resources import Allocation, Proposal from golem.resources.demand.demand_builder import DemandBuilder -from golem.resources.proposal.proposal import Proposal from golem.utils.logging import trace_span logger = logging.getLogger(__name__) diff --git a/golem/managers/mixins.py b/golem/managers/mixins.py index 966118a7..8f0e4609 100644 --- a/golem/managers/mixins.py +++ b/golem/managers/mixins.py @@ -9,7 +9,6 @@ from golem.payload.parsers.base import PayloadSyntaxParser from golem.payload.parsers.textx.parser import TextXPayloadSyntaxParser from golem.resources import Proposal, ProposalData -from golem.resources.proposal.proposal import Proposal from golem.utils.asyncio import create_task_with_logging from golem.utils.logging import trace_span diff --git a/golem/managers/work/plugins.py b/golem/managers/work/plugins.py index be535dcb..35187125 100644 --- a/golem/managers/work/plugins.py +++ b/golem/managers/work/plugins.py @@ -43,7 +43,8 @@ async def wrapper(work: Work) -> WorkResult: errors.append(work_result.exception) logger.info( - f"Got an exception {work_result.exception} on {count} attempt {tries-count} attempts left" + f"Got an exception {work_result.exception} on {count} attempt {tries-count}" + "attempts left" ) work_result.extras["retry"] = { From eadfbe40c829eef2c436e9d74987b329c0ae8647 Mon Sep 17 00:00:00 2001 From: approxit Date: Sun, 16 Jul 2023 01:03:41 +0200 Subject: [PATCH 085/123] trace_span tests --- golem/managers/activity/mixins.py | 4 +- golem/managers/activity/pool.py | 2 +- golem/managers/activity/single_use.py | 2 +- golem/managers/agreement/events.py | 3 +- golem/managers/agreement/scored_aot.py | 6 +- golem/managers/base.py | 12 +- golem/managers/demand/auto.py | 2 + golem/managers/mixins.py | 10 +- golem/managers/negotiation/sequential.py | 4 +- golem/managers/network/single.py | 8 +- golem/managers/payment/pay_all.py | 8 +- golem/managers/work/asynchronous.py | 4 +- golem/managers/work/sequential.py | 4 +- golem/utils/logging.py | 129 +++++++++------ tests/unit/utils/test_logging.py | 195 +++++++++++++++++++++++ 15 files changed, 311 insertions(+), 82 deletions(-) create mode 100644 tests/unit/utils/test_logging.py diff --git a/golem/managers/activity/mixins.py b/golem/managers/activity/mixins.py index 48af52e0..898deb06 100644 --- a/golem/managers/activity/mixins.py +++ b/golem/managers/activity/mixins.py @@ -27,7 +27,7 @@ def __init__( super().__init__(*args, **kwargs) - @trace_span() + @trace_span(show_arguments=True, show_results=True) async def _prepare_activity(self, agreement) -> Activity: activity = await agreement.create_activity() logger.info(f"Activity `{activity}` created") @@ -36,7 +36,7 @@ async def _prepare_activity(self, agreement) -> Activity: await self._on_activity_start(work_context) return activity - @trace_span() + @trace_span(show_arguments=True) async def _release_activity(self, activity: Activity) -> None: if self._on_activity_stop: work_context = WorkContext(activity) diff --git a/golem/managers/activity/pool.py b/golem/managers/activity/pool.py index 21bed377..10796ae8 100644 --- a/golem/managers/activity/pool.py +++ b/golem/managers/activity/pool.py @@ -80,7 +80,7 @@ async def _get_activity_from_pool(self): self._pool.put_nowait(activity) logger.info(f"Activity `{activity}` back in the pool") - @trace_span() + @trace_span(show_arguments=True, show_results=True) async def do_work(self, work: Work) -> WorkResult: async with self._get_activity_from_pool() as activity: work_context = WorkContext(activity) diff --git a/golem/managers/activity/single_use.py b/golem/managers/activity/single_use.py index 33870563..86c08aac 100644 --- a/golem/managers/activity/single_use.py +++ b/golem/managers/activity/single_use.py @@ -33,7 +33,7 @@ async def _prepare_single_use_activity(self) -> Activity: except Exception: logger.exception("Creating activity failed, but will be retried with new agreement") - @trace_span() + @trace_span(show_arguments=True, show_results=True) async def do_work(self, work: Work) -> WorkResult: async with self._prepare_single_use_activity() as activity: work_context = WorkContext(activity) diff --git a/golem/managers/agreement/events.py b/golem/managers/agreement/events.py index 5ad374c4..5cac1f5d 100644 --- a/golem/managers/agreement/events.py +++ b/golem/managers/agreement/events.py @@ -1,5 +1,6 @@ from golem.managers.base import ManagerEvent +from golem.resources import Agreement, ResourceEvent -class AgreementReleased(ManagerEvent): +class AgreementReleased(ManagerEvent, ResourceEvent[Agreement]): pass diff --git a/golem/managers/agreement/scored_aot.py b/golem/managers/agreement/scored_aot.py index ae232912..92aa3978 100644 --- a/golem/managers/agreement/scored_aot.py +++ b/golem/managers/agreement/scored_aot.py @@ -26,7 +26,7 @@ def __init__( super().__init__(*args, **kwargs) - @trace_span() + @trace_span(show_arguments=True) async def get_agreement(self) -> Agreement: while True: proposal = await self.get_scored_proposal() @@ -42,7 +42,7 @@ async def get_agreement(self) -> Agreement: # TODO: Support removing callback on resource close await self._event_bus.on_once( AgreementReleased, - self._terminate_agreement, + self._terminate_agreement_if_released, lambda event: event.resource.id == agreement.id, ) return agreement @@ -53,7 +53,7 @@ async def _background_loop(self) -> None: await self.manage_scoring(proposal) - @trace_span() + @trace_span(show_arguments=True) async def _terminate_agreement(self, event: AgreementReleased) -> None: agreement: Agreement = event.resource await agreement.terminate() diff --git a/golem/managers/base.py b/golem/managers/base.py index 9a9e3fbe..0f22d7eb 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -11,7 +11,6 @@ DemandData, Proposal, ProposalData, - ResourceEvent, Script, ) from golem.resources.activity import commands @@ -100,7 +99,7 @@ def __call__(self, context: WorkContext) -> Awaitable[Optional[WorkResult]]: DoWorkCallable = Callable[[Work], Awaitable[WorkResult]] -class ManagerEvent(ResourceEvent, ABC): +class ManagerEvent(ABC): pass @@ -121,17 +120,17 @@ async def __aexit__(self, exc_type, exc, tb): await self.stop() async def start(self) -> None: - ... + pass async def stop(self) -> None: - ... + pass TPlugin = TypeVar("TPlugin") class NetworkManager(Manager, ABC): - ... + pass class PaymentManager(Manager, ABC): @@ -165,7 +164,7 @@ async def do_work(self, work: Work) -> WorkResult: class WorkManager(Manager, ABC): - ... + pass class RejectProposal(ManagerPluginException): @@ -195,6 +194,7 @@ def __call__( class WorkManagerPlugin(ABC): + @abstractmethod def __call__(self, do_work: DoWorkCallable) -> DoWorkCallable: ... diff --git a/golem/managers/demand/auto.py b/golem/managers/demand/auto.py index dd6e40d9..c04ceb0b 100644 --- a/golem/managers/demand/auto.py +++ b/golem/managers/demand/auto.py @@ -39,6 +39,8 @@ async def _background_loop(self) -> None: demand = await demand_builder.create_demand(self._golem) demand.start_collecting_events() + logger.debug(f"`{demand}` posted on market with `{demand_builder}`") + try: async for initial_proposal in demand.initial_proposals(): await self.manage_scoring(initial_proposal) diff --git a/golem/managers/mixins.py b/golem/managers/mixins.py index 8f0e4609..6ecdeb75 100644 --- a/golem/managers/mixins.py +++ b/golem/managers/mixins.py @@ -16,7 +16,10 @@ class BackgroundLoopMixin: - _background_loop_task: Optional[asyncio.Task] = None + def __init__(self, *args, **kwargs) -> None: + self._background_loop_task: Optional[asyncio.Task] = None + + super().__init__(*args, **kwargs) @trace_span() async def start(self) -> None: @@ -37,7 +40,7 @@ def is_started(self) -> bool: return self._background_loop_task is not None and not self._background_loop_task.done() async def _background_loop(self) -> None: - ... + pass class ManagerPluginsMixin(Generic[TPlugin]): @@ -63,9 +66,10 @@ def __init__( self._scored_proposals: List[Tuple[float, Proposal]] = [] self._scored_proposals_condition = asyncio.Condition() + super().__init__(*args, **kwargs) - @trace_span() + @trace_span(show_arguments=True) async def manage_scoring(self, proposal: Proposal) -> None: async with self._scored_proposals_condition: all_proposals = list(sp[1] for sp in self._scored_proposals) diff --git a/golem/managers/negotiation/sequential.py b/golem/managers/negotiation/sequential.py index f1a9f07a..384cae1b 100644 --- a/golem/managers/negotiation/sequential.py +++ b/golem/managers/negotiation/sequential.py @@ -36,7 +36,7 @@ def __init__( super().__init__(*args, **kwargs) - @trace_span() + @trace_span(show_results=True) async def get_draft_proposal(self) -> Proposal: return await self._eligible_proposals.get() @@ -52,7 +52,7 @@ async def _background_loop(self) -> None: if offer_proposal is not None: await self._eligible_proposals.put(offer_proposal) - @trace_span() + @trace_span(show_arguments=True, show_results=True) async def _negotiate_proposal( self, demand_data: DemandData, offer_proposal: Proposal ) -> Optional[Proposal]: diff --git a/golem/managers/network/single.py b/golem/managers/network/single.py index 7cef682e..f89e7e11 100644 --- a/golem/managers/network/single.py +++ b/golem/managers/network/single.py @@ -31,14 +31,16 @@ async def get_node_id(self, provider_id: str) -> str: node_ip = self._nodes.get(provider_id) if node_ip: return node_ip + + # TODO: get rid of sleep await asyncio.sleep(0.1) - @trace_span() + @trace_span(show_arguments=True, show_results=True) async def get_deploy_args(self, provider_id: str) -> DeployArgsType: node_ip = await self.get_node_id(provider_id) return self._network.deploy_args(node_ip) - @trace_span() + @trace_span(show_arguments=True, show_results=True) async def get_provider_uri(self, provider_id: str, protocol: str = "http") -> str: node_ip = await self.get_node_id(provider_id) url = self._network.node._api_config.net_url @@ -46,7 +48,7 @@ async def get_provider_uri(self, provider_id: str, protocol: str = "http") -> st connection_uri = f"{net_api_ws}/net/{self._network.id}/tcp/{node_ip}/22" return connection_uri - @trace_span() + @trace_span(show_arguments=True) async def _add_provider_to_network(self, event: NewAgreement): await event.resource.get_data() provider_id = event.resource.data.offer.provider_id diff --git a/golem/managers/payment/pay_all.py b/golem/managers/payment/pay_all.py index 07e9ff28..99b4ba49 100644 --- a/golem/managers/payment/pay_all.py +++ b/golem/managers/payment/pay_all.py @@ -68,7 +68,7 @@ async def _create_allocation(self) -> None: # TODO: We should not rely on golem node with cleanups, manager should do it by itself self._golem.add_autoclose_resource(self._allocation) - @trace_span() + @trace_span(show_results=True) async def get_allocation(self) -> "Allocation": # TODO handle NoMatchingAccount if self._allocation is None: @@ -97,7 +97,7 @@ async def _increment_opened_agreements(self, event: NewAgreement): async def _increment_closed_agreements(self, event: AgreementClosed): self._closed_agreements_count += 1 - @trace_span() + @trace_span(show_arguments=True) async def _accept_invoice(self, invoice: Invoice) -> None: assert self._allocation is not None # TODO think of a better way await invoice.accept_full(self._allocation) @@ -106,7 +106,7 @@ async def _accept_invoice(self, invoice: Invoice) -> None: logger.info(f"Invoice `{invoice.id}` accepted") - @trace_span() + @trace_span(show_arguments=True) async def _accept_debit_note(self, debit_note: DebitNote) -> None: assert self._allocation is not None # TODO think of a better way await debit_note.accept_full(self._allocation) @@ -117,7 +117,6 @@ async def _accept_debit_note(self, debit_note: DebitNote) -> None: @trace_span() async def _pay_invoice_if_received(self, event: NewInvoice) -> None: invoice = event.resource - assert isinstance(invoice, Invoice) if (await invoice.get_data(force=True)).status == "RECEIVED": await self._accept_invoice(invoice) @@ -125,7 +124,6 @@ async def _pay_invoice_if_received(self, event: NewInvoice) -> None: @trace_span() async def _pay_debit_note_if_received(self, event: NewDebitNote) -> None: debit_note = event.resource - assert isinstance(debit_note, DebitNote) if (await debit_note.get_data(force=True)).status == "RECEIVED": await self._accept_debit_note(debit_note) diff --git a/golem/managers/work/asynchronous.py b/golem/managers/work/asynchronous.py index c97c2598..5713ead2 100644 --- a/golem/managers/work/asynchronous.py +++ b/golem/managers/work/asynchronous.py @@ -16,13 +16,13 @@ def __init__(self, golem: GolemNode, do_work: DoWorkCallable, *args, **kwargs): super().__init__(*args, **kwargs) - @trace_span() + @trace_span(show_arguments=True, show_results=True) async def do_work(self, work: Work) -> WorkResult: result = await self._do_work_with_plugins(self._do_work, work) logger.info(f"Work `{work}` completed") return result - @trace_span() + @trace_span(show_arguments=True, show_results=True) async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: results = await asyncio.gather(*[self.do_work(work) for work in work_list]) return results diff --git a/golem/managers/work/sequential.py b/golem/managers/work/sequential.py index ceae43fb..69a63692 100644 --- a/golem/managers/work/sequential.py +++ b/golem/managers/work/sequential.py @@ -15,7 +15,7 @@ def __init__(self, golem: GolemNode, do_work: DoWorkCallable, *args, **kwargs): super().__init__(*args, **kwargs) - @trace_span() + @trace_span(show_arguments=True, show_results=True) async def do_work(self, work: Work) -> WorkResult: result = await self._do_work_with_plugins(self._do_work, work) @@ -23,7 +23,7 @@ async def do_work(self, work: Work) -> WorkResult: return result - @trace_span() + @trace_span(show_arguments=True, show_results=True) async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: results = [] diff --git a/golem/utils/logging.py b/golem/utils/logging.py index d09ee6ec..fff7a07c 100644 --- a/golem/utils/logging.py +++ b/golem/utils/logging.py @@ -2,7 +2,7 @@ import logging from datetime import datetime, timezone from functools import wraps -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Sequence if TYPE_CHECKING: from golem.event_bus import Event @@ -13,7 +13,7 @@ "disable_existing_loggers": False, "formatters": { "default": { - "format": "[%(asctime)s %(levelname)s %(name)s] %(message)s", + "format": "[%(asctime)s %(levelname)-7s %(name)s] %(message)s", }, }, "handlers": { @@ -35,9 +35,6 @@ "golem": { "level": "INFO", }, - "golem.utils.logging": { - "level": "INFO", - }, "golem.managers": { "level": "INFO", }, @@ -60,7 +57,7 @@ "level": "INFO", }, "golem.managers.activity": { - "level": "INFO", + "level": "DEBUG", }, "golem.managers.work": { "level": "INFO", @@ -68,8 +65,6 @@ }, } -logger = logging.getLogger(__name__) - class _YagnaDatetimeFormatter(logging.Formatter): """Custom log Formatter that formats datetime using the same convention yagna uses.""" @@ -133,58 +128,90 @@ async def on_event(self, event: "Event") -> None: self.logger.info(event) -def trace_span(name: Optional[str] = None, show_arguments: bool = False, show_results: bool = True): - def wrapper(f): - span_name = name if name is not None else f.__name__ +class TraceSpan: + def __init__( + self, name: Optional[str] = None, show_arguments: bool = False, show_results: bool = False + ) -> None: + self._name = name + self._show_arguments = show_arguments + self._show_results = show_results + + def __call__(self, func): + wrapper = self._async_wrapper if inspect.iscoroutinefunction(func) else self._sync_wrapper + + # TODO: partial instead of decorator() + def decorator(*args, **kwargs): + return wrapper(func, args, kwargs) + + return wraps(func)(decorator) + + def _get_span_name(self, func: Callable, args: Sequence, kwargs: Dict) -> str: + if self._name is not None: + return self._name + + # TODO: check type of func in different cases + contextmanager + span_name = ( + func.__qualname__.split(">.")[-1] if self._is_instance_method(func) else func.__name__ + ) + + if self._show_arguments: + arguments = ", ".join( + [ + *[repr(a) for a in (args[1:] if self._is_instance_method(func) else args)], + *["{}={}".format(k, repr(v)) for (k, v) in kwargs.items()], + ] + ) + return f"{span_name}({arguments})" + + return span_name + + def _get_logger(self, func: Callable, args: Sequence) -> logging.Logger: + module_name = ( + args[0].__class__.__module__ if self._is_instance_method(func) else func.__module__ + ) + + return logging.getLogger(module_name) + + def _is_instance_method(self, func: Callable) -> bool: + return inspect.isfunction(func) and func.__name__ != func.__qualname__.split(">.")[-1] - @wraps(f) - def sync_wrapped(*args, **kwargs): - if show_arguments: - args_str = ", ".join(repr(a) for a in args) - kwargs_str = ", ".join("{}={}".format(k, repr(v)) for (k, v) in kwargs.items()) - final_name = f"{span_name}({args_str}, {kwargs_str})" - else: - final_name = span_name + def _sync_wrapper(self, func: Callable, args: Sequence, kwargs: Dict) -> Any: + span_name = self._get_span_name(func, args, kwargs) + logger = self._get_logger(func, args) - logger.debug(f"{final_name}...") + logger.debug("Calling %s...", span_name) - try: - result = f(*args, **kwargs) - except Exception as e: - logger.debug(f"{final_name} failed with `{e}`") - raise + try: + result = func(*args, **kwargs) + except Exception as e: + logger.debug("Calling %s failed with `%s`", span_name, e) + raise - if show_results: - logger.debug(f"{final_name} done with `{result}`") - else: - logger.debug(f"{final_name} done") + if self._show_results: + logger.debug("Calling %s done with `%s`", span_name, result) + else: + logger.debug("Calling %s done", span_name) - return result + return result - @wraps(f) - async def async_wrapped(*args, **kwargs): - if show_arguments: - args_str = ", ".join(repr(a) for a in args) - kwargs_str = ", ".join("{}={}".format(k, repr(v)) for (k, v) in kwargs.items()) - final_name = f"{span_name}({args_str}, {kwargs_str})" - else: - final_name = span_name + async def _async_wrapper(self, func: Callable, args: Sequence, kwargs: Dict) -> Any: + span_name = self._get_span_name(func, args, kwargs) + logger = self._get_logger(func, args) - logger.debug(f"{final_name}...") + logger.debug("Calling %s...", span_name) - try: - result = await f(*args, **kwargs) - except Exception as e: - logger.debug(f"{final_name} failed with `{e}`") - raise + try: + result = await func(*args, **kwargs) + except Exception as e: + logger.debug("Calling %s failed with `%s`", span_name, e) + raise - if show_results: - logger.debug(f"{final_name} done with `{result}`") - else: - logger.debug(f"{final_name} done") + if self._show_results: + logger.debug("Calling %s done with `%s`", span_name, result) + else: + logger.debug("Calling %s done", span_name) - return result + return result - return async_wrapped if inspect.iscoroutinefunction(f) else sync_wrapped - return wrapper +trace_span = TraceSpan diff --git a/tests/unit/utils/test_logging.py b/tests/unit/utils/test_logging.py new file mode 100644 index 00000000..1ffc1468 --- /dev/null +++ b/tests/unit/utils/test_logging.py @@ -0,0 +1,195 @@ +import asyncio +import logging + +import pytest + +from golem.utils.logging import trace_span + + +def test_trace_span_on_standalone_function(caplog): + caplog.set_level(logging.DEBUG) + + @trace_span() + def foobar(a): + return a + + value = "foobar" + assert foobar(value) == value + + assert caplog.record_tuples == [ + ("tests.unit.utils.test_logging", logging.DEBUG, "Calling foobar..."), + ("tests.unit.utils.test_logging", logging.DEBUG, "Calling foobar done"), + ] + + +def test_trace_span_on_nested_function(caplog): + caplog.set_level(logging.DEBUG) + + def foo(a): + @trace_span() + def bar(b): + return b + + return bar(a) + + value = "foobar" + assert foo(value) == value + + assert caplog.record_tuples == [ + ("tests.unit.utils.test_logging", logging.DEBUG, "Calling bar..."), + ("tests.unit.utils.test_logging", logging.DEBUG, "Calling bar done"), + ] + + +def test_trace_span_with_results(caplog): + caplog.set_level(logging.DEBUG) + + @trace_span(show_results=True) + def foobar(a): + return a + + value = "foo" + assert foobar(value) == value + + assert caplog.record_tuples == [ + ("tests.unit.utils.test_logging", logging.DEBUG, "Calling foobar..."), + ("tests.unit.utils.test_logging", logging.DEBUG, f"Calling foobar done with `{value}`"), + ] + + +def test_trace_span_with_exception(caplog): + caplog.set_level(logging.DEBUG) + + exc_message = "some exception message" + + @trace_span() + def foobar(): + raise Exception(exc_message) + + with pytest.raises(Exception, match=exc_message) as exc_info: + foobar() + + assert caplog.record_tuples == [ + ("tests.unit.utils.test_logging", logging.DEBUG, "Calling foobar..."), + ( + "tests.unit.utils.test_logging", + logging.DEBUG, + f"Calling foobar failed with `{exc_info.value}`", + ), + ] + + +def test_trace_span_on_async_function(caplog): + caplog.set_level(logging.DEBUG, logger="tests.unit.utils.test_logging") + + @trace_span() + async def foobar(a): + return a + + loop = asyncio.get_event_loop() + value = "foobar" + assert loop.run_until_complete(foobar(value)) == value + + assert caplog.record_tuples == [ + ("tests.unit.utils.test_logging", logging.DEBUG, "Calling foobar..."), + ("tests.unit.utils.test_logging", logging.DEBUG, "Calling foobar done"), + ] + + +def test_trace_span_on_async_function_with_result(caplog): + caplog.set_level(logging.DEBUG, logger="tests.unit.utils.test_logging") + + @trace_span(show_results=True) + async def foobar(a): + return a + + loop = asyncio.get_event_loop() + value = "foobar" + assert loop.run_until_complete(foobar(value)) == value + + assert caplog.record_tuples == [ + ("tests.unit.utils.test_logging", logging.DEBUG, "Calling foobar..."), + ("tests.unit.utils.test_logging", logging.DEBUG, f"Calling foobar done with `{value}`"), + ] + + +def test_trace_span_on_async_function_with_exception(caplog): + caplog.set_level(logging.DEBUG, logger="tests.unit.utils.test_logging") + exc_message = "some exception message" + + @trace_span() + async def foobar(): + raise Exception(exc_message) + + loop = asyncio.get_event_loop() + with pytest.raises(Exception, match=exc_message) as exc_info: + loop.run_until_complete(foobar()) + + assert caplog.record_tuples == [ + ("tests.unit.utils.test_logging", logging.DEBUG, "Calling foobar..."), + ( + "tests.unit.utils.test_logging", + logging.DEBUG, + f"Calling foobar failed with `{exc_info.value}`", + ), + ] + + +def test_trace_span_on_method(caplog): + caplog.set_level(logging.DEBUG) + + class Foo: + @trace_span() + def bar(self, a): + return a + + value = "foobar" + assert Foo().bar(value) == value + + assert caplog.record_tuples == [ + ("tests.unit.utils.test_logging", logging.DEBUG, "Calling Foo.bar..."), + ("tests.unit.utils.test_logging", logging.DEBUG, "Calling Foo.bar done"), + ] + + +def test_trace_span_with_custom_name(caplog): + caplog.set_level(logging.DEBUG) + custom_name = "custom-name" + + @trace_span(name=custom_name) + def foobar(a): + return a + + value = "foobar" + assert foobar(value) == value + + assert caplog.record_tuples == [ + ("tests.unit.utils.test_logging", logging.DEBUG, f"Calling {custom_name}..."), + ("tests.unit.utils.test_logging", logging.DEBUG, f"Calling {custom_name} done"), + ] + + +def test_trace_span_with_arguments(caplog): + caplog.set_level(logging.DEBUG) + + @trace_span(show_arguments=True) + def foobar(a, b, c=None, d=None): + return a + + a = "value_a" + b = "value_b" + c = "value_c" + assert foobar(a, b, c=c) == a + + assert caplog.record_tuples == [ + ( + "tests.unit.utils.test_logging", + logging.DEBUG, + f"Calling foobar({repr(a)}, {repr(b)}, c={repr(c)})...", + ), + ( + "tests.unit.utils.test_logging", + logging.DEBUG, + f"Calling foobar({repr(a)}, {repr(b)}, c={repr(c)}) done", + ), + ] From 12b532d9e8ced3542d327f890d1cd76ca291ca4a Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 19 Jul 2023 14:45:49 +0200 Subject: [PATCH 086/123] Add batch of mypy fixes --- examples/attach.py | 2 +- examples/task_api_draft/examples/yacat.py | 6 +++--- golem/event_bus/base.py | 4 ++-- golem/event_bus/in_memory/event_bus.py | 22 +++++++++++++--------- golem/managers/activity/mixins.py | 3 ++- golem/managers/activity/pool.py | 4 ++-- golem/managers/activity/single_use.py | 4 ++-- golem/managers/agreement/plugins.py | 4 ++-- golem/managers/agreement/scored_aot.py | 2 +- golem/managers/base.py | 1 + golem/managers/mixins.py | 5 +++-- golem/managers/network/single.py | 1 + golem/managers/payment/pay_all.py | 7 ++++--- golem/managers/work/plugins.py | 2 +- golem/payload/constraints.py | 6 +++--- golem/payload/properties.py | 4 ++-- golem/resources/agreement/agreement.py | 4 ++-- golem/resources/allocation/allocation.py | 4 ++-- golem/resources/debit_note/debit_note.py | 4 ++-- golem/resources/demand/demand.py | 8 ++++---- golem/resources/exceptions.py | 6 +++--- golem/resources/invoice/invoice.py | 4 ++-- golem/resources/proposal/proposal.py | 12 ++++++------ 23 files changed, 64 insertions(+), 55 deletions(-) diff --git a/examples/attach.py b/examples/attach.py index 6ac2f18f..7e5604cf 100644 --- a/examples/attach.py +++ b/examples/attach.py @@ -10,7 +10,7 @@ async def accept_debit_note(payment_event: NewResource) -> None: - debit_note: DebitNote = payment_event.resource # type: ignore + debit_note: DebitNote = payment_event.resource this_activity_id = (await debit_note.get_data()).activity_id if this_activity_id == ACTIVITY_ID: diff --git a/examples/task_api_draft/examples/yacat.py b/examples/task_api_draft/examples/yacat.py index a24a8bac..257c4636 100644 --- a/examples/task_api_draft/examples/yacat.py +++ b/examples/task_api_draft/examples/yacat.py @@ -72,7 +72,7 @@ async def count_batches(event: NewResource) -> None: async def gather_debit_note_log(event: NewResource) -> None: - debit_note: DebitNote = event.resource # type: ignore + debit_note: DebitNote = event.resource activity_id = (await debit_note.get_data()).activity_id if not any(activity.id == activity_id for activity in activity_data): # This is a debit note for an unknown activity (e.g. from a previous run) @@ -86,7 +86,7 @@ async def gather_debit_note_log(event: NewResource) -> None: async def note_activity_destroyed(event: ResourceClosed) -> None: - activity: Activity = event.resource # type: ignore + activity: Activity = event.resource if activity not in activity_data: # Destroyed activity from a previous run return @@ -97,7 +97,7 @@ async def note_activity_destroyed(event: ResourceClosed) -> None: async def update_new_activity_status(event: NewResource) -> None: - activity: Activity = event.resource # type: ignore + activity: Activity = event.resource activity_data[activity]["status"] = "new" if not activity.has_parent: diff --git a/golem/event_bus/base.py b/golem/event_bus/base.py index 676ed82e..abcc9df7 100644 --- a/golem/event_bus/base.py +++ b/golem/event_bus/base.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Awaitable, Callable, Optional, Type, TypeVar +from typing import Awaitable, Callable, Generic, Optional, Type, TypeVar from golem.exceptions import GolemException @@ -16,7 +16,7 @@ class EventBusError(GolemException): pass -class EventBus(ABC): +class EventBus(ABC, Generic[TCallbackHandler]): @abstractmethod async def start(self) -> None: ... diff --git a/golem/event_bus/in_memory/event_bus.py b/golem/event_bus/in_memory/event_bus.py index 912743af..88bce4d1 100644 --- a/golem/event_bus/in_memory/event_bus.py +++ b/golem/event_bus/in_memory/event_bus.py @@ -2,9 +2,9 @@ import logging from collections import defaultdict from dataclasses import dataclass -from typing import Awaitable, Callable, DefaultDict, List, Optional, Type +from typing import Awaitable, Callable, DefaultDict, List, Optional, Tuple, Type -from golem.event_bus.base import Event, EventBus, EventBusError, TCallbackHandler, TEvent +from golem.event_bus.base import Event, EventBus, EventBusError, TEvent from golem.utils.asyncio import create_task_with_logging logger = logging.getLogger(__name__) @@ -17,10 +17,13 @@ class _CallbackInfo: once: bool -class InMemoryEventBus(EventBus): +_CallbackHandler = Tuple[Type[TEvent], _CallbackInfo] + + +class InMemoryEventBus(EventBus[_CallbackHandler]): def __init__(self): self._callbacks: DefaultDict[Type[TEvent], List[_CallbackInfo]] = defaultdict(list) - self._event_queue = asyncio.Queue() + self._event_queue: asyncio.Queue[TEvent] = asyncio.Queue() self._process_event_queue_loop_task: Optional[asyncio.Task] = None async def start(self): @@ -47,8 +50,9 @@ async def stop(self): await self._event_queue.join() - self._process_event_queue_loop_task.cancel() - self._process_event_queue_loop_task = None + if self._process_event_queue_loop_task is not None: + self._process_event_queue_loop_task.cancel() + self._process_event_queue_loop_task = None logger.debug("Stopping event bus done") @@ -63,7 +67,7 @@ async def on( event_type: Type[TEvent], callback: Callable[[TEvent], Awaitable[None]], filter_func: Optional[Callable[[TEvent], bool]] = None, - ) -> TCallbackHandler: + ) -> _CallbackHandler: logger.debug( f"Adding callback handler for `{event_type}` with callback `{callback}`" f" and filter `{filter_func}`..." @@ -88,7 +92,7 @@ async def on_once( event_type: Type[TEvent], callback: Callable[[TEvent], Awaitable[None]], filter_func: Optional[Callable[[TEvent], bool]] = None, - ) -> TCallbackHandler: + ) -> _CallbackHandler: logger.debug( f"Adding one-time callback handler for `{event_type}` with callback `{callback}`" f" and filter `{filter_func}`..." @@ -108,7 +112,7 @@ async def on_once( return callback_handler - async def off(self, callback_handler: TCallbackHandler) -> None: + async def off(self, callback_handler: _CallbackHandler) -> None: logger.debug(f"Removing callback handler `{id(callback_handler)}`...") event_type, callback_info = callback_handler diff --git a/golem/managers/activity/mixins.py b/golem/managers/activity/mixins.py index 898deb06..c2b6b4db 100644 --- a/golem/managers/activity/mixins.py +++ b/golem/managers/activity/mixins.py @@ -52,4 +52,5 @@ async def _release_activity(self, activity: Activity) -> None: ) event = AgreementReleased(activity.parent) - await self._event_bus.emit(event) + # TODO how to give access to event bus without coupling with GolemNode? + await self._event_bus.emit(event) # type: ignore[attr-defined] diff --git a/golem/managers/activity/pool.py b/golem/managers/activity/pool.py index 10796ae8..68906ec6 100644 --- a/golem/managers/activity/pool.py +++ b/golem/managers/activity/pool.py @@ -7,7 +7,7 @@ from golem.managers.base import ActivityManager, Work, WorkContext, WorkResult from golem.managers.mixins import BackgroundLoopMixin from golem.node import GolemNode -from golem.resources import Agreement +from golem.resources import Activity, Agreement from golem.utils.logging import trace_span logger = logging.getLogger(__name__) @@ -26,7 +26,7 @@ def __init__( self._event_bus = golem.event_bus self._pool_target_size = size - self._pool = asyncio.Queue() + self._pool: asyncio.Queue[Activity] = asyncio.Queue() super().__init__(*args, **kwargs) async def _background_loop(self): diff --git a/golem/managers/activity/single_use.py b/golem/managers/activity/single_use.py index 86c08aac..572187a6 100644 --- a/golem/managers/activity/single_use.py +++ b/golem/managers/activity/single_use.py @@ -1,6 +1,6 @@ import logging from contextlib import asynccontextmanager -from typing import Awaitable, Callable +from typing import AsyncGenerator, Awaitable, Callable from golem.managers.activity.mixins import ActivityPrepareReleaseMixin from golem.managers.base import ActivityManager, Work, WorkContext, WorkResult @@ -21,7 +21,7 @@ def __init__( super().__init__(*args, **kwargs) @asynccontextmanager - async def _prepare_single_use_activity(self) -> Activity: + async def _prepare_single_use_activity(self) -> AsyncGenerator[Activity, None]: while True: agreement = await self._get_agreement() try: diff --git a/golem/managers/agreement/plugins.py b/golem/managers/agreement/plugins.py index 11eed61c..426a0fa7 100644 --- a/golem/managers/agreement/plugins.py +++ b/golem/managers/agreement/plugins.py @@ -1,11 +1,11 @@ from random import random -from typing import Callable, Optional, Sequence, Tuple, Type, Union +from typing import Callable, Optional, Sequence, Tuple, Union from golem.managers.base import ManagerPluginException, ManagerScorePlugin, ProposalPluginResult from golem.payload.constraints import PropertyName from golem.resources import ProposalData -PropertyValueNumeric: Type[Union[int, float]] = Union[int, float] +PropertyValueNumeric = Union[int, float] BoundaryValues = Tuple[Tuple[float, PropertyValueNumeric], Tuple[float, PropertyValueNumeric]] diff --git a/golem/managers/agreement/scored_aot.py b/golem/managers/agreement/scored_aot.py index 92aa3978..9d9157a9 100644 --- a/golem/managers/agreement/scored_aot.py +++ b/golem/managers/agreement/scored_aot.py @@ -42,7 +42,7 @@ async def get_agreement(self) -> Agreement: # TODO: Support removing callback on resource close await self._event_bus.on_once( AgreementReleased, - self._terminate_agreement_if_released, + self._terminate_agreement, lambda event: event.resource.id == agreement.id, ) return agreement diff --git a/golem/managers/base.py b/golem/managers/base.py index 0f22d7eb..b8c8da4b 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -92,6 +92,7 @@ class WorkResult: class Work(ABC): _work_plugins: Optional[List["WorkManagerPlugin"]] + @abstractmethod def __call__(self, context: WorkContext) -> Awaitable[Optional[WorkResult]]: ... diff --git a/golem/managers/mixins.py b/golem/managers/mixins.py index 6ecdeb75..12b84ab2 100644 --- a/golem/managers/mixins.py +++ b/golem/managers/mixins.py @@ -33,8 +33,9 @@ async def stop(self) -> None: if not self.is_started(): raise ManagerException("Already stopped!") - self._background_loop_task.cancel() - self._background_loop_task = None + if self._background_loop_task is not None: + self._background_loop_task.cancel() + self._background_loop_task = None def is_started(self) -> bool: return self._background_loop_task is not None and not self._background_loop_task.done() diff --git a/golem/managers/network/single.py b/golem/managers/network/single.py index f89e7e11..b9cac1c2 100644 --- a/golem/managers/network/single.py +++ b/golem/managers/network/single.py @@ -52,5 +52,6 @@ async def get_provider_uri(self, provider_id: str, protocol: str = "http") -> st async def _add_provider_to_network(self, event: NewAgreement): await event.resource.get_data() provider_id = event.resource.data.offer.provider_id + assert provider_id is not None # TODO handle this case better logger.info(f"Adding provider {provider_id} to network") self._nodes[provider_id] = await self._network.create_node(provider_id) diff --git a/golem/managers/payment/pay_all.py b/golem/managers/payment/pay_all.py index 99b4ba49..68344af8 100644 --- a/golem/managers/payment/pay_all.py +++ b/golem/managers/payment/pay_all.py @@ -1,7 +1,7 @@ import asyncio import logging from decimal import Decimal -from typing import Optional +from typing import List, Optional from golem.managers.base import PaymentManager from golem.node import PAYMENT_DRIVER, PAYMENT_NETWORK, GolemNode @@ -38,7 +38,7 @@ def __init__( self._closed_agreements_count = 0 self._payed_invoices_count = 0 - self._event_handlers = [] + self._event_handlers: List = [] @trace_span() async def start(self): @@ -74,7 +74,8 @@ async def get_allocation(self) -> "Allocation": if self._allocation is None: await self._create_allocation() - return self._allocation + # TODO fix type + return self._allocation # type: ignore[return-value] @trace_span() async def wait_for_invoices(self): diff --git a/golem/managers/work/plugins.py b/golem/managers/work/plugins.py index 35187125..ba95cedf 100644 --- a/golem/managers/work/plugins.py +++ b/golem/managers/work/plugins.py @@ -18,7 +18,7 @@ def _work_plugin(work: Work): if not hasattr(work, WORK_PLUGIN_FIELD_NAME): work._work_plugins = [] - work._work_plugins.append(plugin) + work._work_plugins.append(plugin) # type: ignore [union-attr] return work diff --git a/golem/payload/constraints.py b/golem/payload/constraints.py index 3e50ba0f..92fb71f6 100644 --- a/golem/payload/constraints.py +++ b/golem/payload/constraints.py @@ -1,11 +1,11 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import Any, Literal, MutableSequence, Type, Union +from typing import Any, Literal, MutableSequence, Union from golem.payload.mixins import PropsConsSerializerMixin -PropertyName: Type[str] = str -PropertyValue: Type[Any] = Any +PropertyName = str +PropertyValue = Any class ConstraintException(Exception): diff --git a/golem/payload/properties.py b/golem/payload/properties.py index cedf56ae..b073161c 100644 --- a/golem/payload/properties.py +++ b/golem/payload/properties.py @@ -1,5 +1,5 @@ from copy import deepcopy -from typing import Any, Mapping +from typing import Any, Dict from golem.payload.mixins import PropsConsSerializerMixin @@ -18,7 +18,7 @@ def __init__(self, mapping=_missing, /) -> None: super().__init__(mapping_deep_copy) - def serialize(self) -> Mapping[str, Any]: + def serialize(self) -> Dict[str, Any]: """Serialize complex objects into format handled by Market API properties specification.""" return { key: self._serialize_property(value) for key, value in self.items() if value is not None diff --git a/golem/resources/agreement/agreement.py b/golem/resources/agreement/agreement.py index fb4564db..9a03747b 100644 --- a/golem/resources/agreement/agreement.py +++ b/golem/resources/agreement/agreement.py @@ -8,7 +8,7 @@ from golem.resources.activity import Activity from golem.resources.agreement.events import AgreementClosed, NewAgreement -from golem.resources.base import _NULL, Resource, TModel, api_call_wrapper +from golem.resources.base import _NULL, Resource, api_call_wrapper from golem.resources.invoice import Invoice if TYPE_CHECKING: @@ -29,7 +29,7 @@ class Agreement(Resource[RequestorApi, models.Agreement, "Proposal", Activity, _ await agreement.terminate() """ - def __init__(self, node: "GolemNode", id_: str, data: Optional[TModel] = None): + def __init__(self, node: "GolemNode", id_: str, data: Optional[models.Agreement] = None): super().__init__(node, id_, data) asyncio.create_task(node.event_bus.emit(NewAgreement(self))) diff --git a/golem/resources/allocation/allocation.py b/golem/resources/allocation/allocation.py index d709f387..92da7d0a 100644 --- a/golem/resources/allocation/allocation.py +++ b/golem/resources/allocation/allocation.py @@ -10,7 +10,7 @@ from golem.payload.properties import Properties from golem.resources.allocation.events import NewAllocation from golem.resources.allocation.exceptions import NoMatchingAccount -from golem.resources.base import _NULL, Resource, TModel, api_call_wrapper +from golem.resources.base import _NULL, Resource, api_call_wrapper from golem.resources.events import ResourceClosed if TYPE_CHECKING: @@ -23,7 +23,7 @@ class Allocation(Resource[RequestorApi, models.Allocation, _NULL, _NULL, _NULL]) Created with one of the :class:`Allocation`-returning methods of the :any:`GolemNode`. """ - def __init__(self, node: "GolemNode", id_: str, data: Optional[TModel] = None): + def __init__(self, node: "GolemNode", id_: str, data: Optional[models.Allocation] = None): super().__init__(node, id_, data) asyncio.create_task(node.event_bus.emit(NewAllocation(self))) diff --git a/golem/resources/debit_note/debit_note.py b/golem/resources/debit_note/debit_note.py index faf5d33b..03b016e1 100644 --- a/golem/resources/debit_note/debit_note.py +++ b/golem/resources/debit_note/debit_note.py @@ -5,7 +5,7 @@ from ya_payment import RequestorApi, models from golem.resources.allocation import Allocation -from golem.resources.base import _NULL, Resource, TModel, api_call_wrapper +from golem.resources.base import _NULL, Resource, api_call_wrapper from golem.resources.debit_note.events import NewDebitNote if TYPE_CHECKING: @@ -19,7 +19,7 @@ class DebitNote(Resource[RequestorApi, models.DebitNote, "Activity", _NULL, _NUL Usually created by a :any:`GolemNode` initialized with `collect_payment_events = True`. """ - def __init__(self, node: "GolemNode", id_: str, data: Optional[TModel] = None): + def __init__(self, node: "GolemNode", id_: str, data: Optional[models.DebitNote] = None): super().__init__(node, id_, data) asyncio.create_task(node.event_bus.emit(NewDebitNote(self))) diff --git a/golem/resources/demand/demand.py b/golem/resources/demand/demand.py index 41ceaccb..f2f9a33a 100644 --- a/golem/resources/demand/demand.py +++ b/golem/resources/demand/demand.py @@ -7,7 +7,7 @@ from ya_market import models as models from golem.payload import Constraints, Properties -from golem.resources.base import _NULL, Resource, ResourceNotFound, TModel, api_call_wrapper +from golem.resources.base import _NULL, Resource, ResourceNotFound, api_call_wrapper from golem.resources.demand.events import DemandClosed, NewDemand from golem.resources.proposal import Proposal from golem.utils.low import YagnaEventCollector @@ -20,8 +20,8 @@ class DemandData: properties: Properties constraints: Constraints - demand_id: str - requestor_id: str + demand_id: Optional[str] + requestor_id: Optional[str] timestamp: datetime @@ -31,7 +31,7 @@ class Demand(Resource[RequestorApi, models.Demand, _NULL, Proposal, _NULL], Yagn Created with one of the :class:`Demand`-returning methods of the :any:`GolemNode`. """ - def __init__(self, node: "GolemNode", id_: str, data: Optional[TModel] = None): + def __init__(self, node: "GolemNode", id_: str, data: Optional[models.Demand] = None): super().__init__(node, id_, data) asyncio.create_task(node.event_bus.emit(NewDemand(self))) diff --git a/golem/resources/exceptions.py b/golem/resources/exceptions.py index 4c1fe238..17c08f35 100644 --- a/golem/resources/exceptions.py +++ b/golem/resources/exceptions.py @@ -3,7 +3,7 @@ from golem.exceptions import GolemException if TYPE_CHECKING: - from golem.resources.base import TResource + from golem.resources.base import Resource class ResourceException(GolemException): @@ -26,13 +26,13 @@ class ResourceNotFound(ResourceException): """ - def __init__(self, resource: "TResource"): + def __init__(self, resource: Resource): self._resource = resource msg = f"{resource} doesn't exist" super().__init__(msg) @property - def resource(self) -> "TResource": + def resource(self) -> Resource: """Resource that caused the exception.""" return self._resource diff --git a/golem/resources/invoice/invoice.py b/golem/resources/invoice/invoice.py index 678aafd2..c5d9b665 100644 --- a/golem/resources/invoice/invoice.py +++ b/golem/resources/invoice/invoice.py @@ -5,7 +5,7 @@ from ya_payment import RequestorApi, models from golem.resources.allocation.allocation import Allocation -from golem.resources.base import _NULL, Resource, TModel, api_call_wrapper +from golem.resources.base import _NULL, Resource, api_call_wrapper from golem.resources.invoice.events import NewInvoice if TYPE_CHECKING: @@ -19,7 +19,7 @@ class Invoice(Resource[RequestorApi, models.Invoice, "Agreement", _NULL, _NULL]) Ususally created by a :any:`GolemNode` initialized with `collect_payment_events = True`. """ - def __init__(self, node: "GolemNode", id_: str, data: Optional[TModel] = None): + def __init__(self, node: "GolemNode", id_: str, data: Optional[models.Invoice] = None): super().__init__(node, id_, data) asyncio.create_task(node.event_bus.emit(NewInvoice(self))) diff --git a/golem/resources/proposal/proposal.py b/golem/resources/proposal/proposal.py index 242d0cb2..96183407 100644 --- a/golem/resources/proposal/proposal.py +++ b/golem/resources/proposal/proposal.py @@ -1,14 +1,14 @@ import asyncio from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, AsyncIterator, Literal, Optional, Type, Union +from typing import TYPE_CHECKING, AsyncIterator, Literal, Optional, Union from ya_market import RequestorApi from ya_market import models as models from golem.payload import Constraints, Properties from golem.resources.agreement import Agreement -from golem.resources.base import Resource, TModel, api_call_wrapper +from golem.resources.base import Resource, api_call_wrapper from golem.resources.proposal.events import NewProposal if TYPE_CHECKING: @@ -16,7 +16,7 @@ from golem.resources.demand import Demand -ProposalId: Type[str] = str +ProposalId = str # TODO: Use Enum ProposalState = Literal["Initial", "Draft", "Rejected", "Accepted", "Expired"] @@ -26,8 +26,8 @@ class ProposalData: properties: Properties constraints: Constraints - proposal_id: ProposalId - issuer_id: str + proposal_id: Optional[ProposalId] + issuer_id: Optional[str] state: ProposalState timestamp: datetime prev_proposal_id: Optional[str] @@ -60,7 +60,7 @@ class Proposal( _demand: Optional["Demand"] = None - def __init__(self, node: "GolemNode", id_: str, data: Optional[TModel] = None): + def __init__(self, node: "GolemNode", id_: str, data: Optional[models.Proposal] = None): super().__init__(node, id_, data) asyncio.create_task(node.event_bus.emit(NewProposal(self))) From 320557659a5114c545b28171a2d68ae0c5ab818d Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 19 Jul 2023 15:14:48 +0200 Subject: [PATCH 087/123] Fixed unit tests --- golem/resources/exceptions.py | 4 ++-- tests/unit/test_manager_agreement_plugins.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/golem/resources/exceptions.py b/golem/resources/exceptions.py index 17c08f35..7325cad2 100644 --- a/golem/resources/exceptions.py +++ b/golem/resources/exceptions.py @@ -26,13 +26,13 @@ class ResourceNotFound(ResourceException): """ - def __init__(self, resource: Resource): + def __init__(self, resource: "Resource"): self._resource = resource msg = f"{resource} doesn't exist" super().__init__(msg) @property - def resource(self) -> Resource: + def resource(self) -> "Resource": """Resource that caused the exception.""" return self._resource diff --git a/tests/unit/test_manager_agreement_plugins.py b/tests/unit/test_manager_agreement_plugins.py index f47131e7..2c48a3bb 100644 --- a/tests/unit/test_manager_agreement_plugins.py +++ b/tests/unit/test_manager_agreement_plugins.py @@ -32,12 +32,12 @@ def test_linear_score_plugin(kwargs, property_value, expected, mocker): result = scorer([proposal_data]) - assert result[proposal_id] == expected + assert result[0] == expected @pytest.mark.parametrize("expected_value", (0.876, 0.0, 0.2, 0.5, 1)) def test_random_score_plugin(mocker, expected_value): - mocker.patch("golem.managers.proposal.plugins.random", mocker.Mock(return_value=expected_value)) + mocker.patch("golem.managers.agreement.plugins.random", mocker.Mock(return_value=expected_value)) proposal_id = "foo" @@ -46,4 +46,4 @@ def test_random_score_plugin(mocker, expected_value): result = scorer([proposal_data]) - assert result[proposal_id] == expected_value + assert result[0] == expected_value From 5a9e75825c9ab86da6092e0e61402028d9dac046 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 19 Jul 2023 15:15:32 +0200 Subject: [PATCH 088/123] Fix formatting --- tests/unit/test_manager_agreement_plugins.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_manager_agreement_plugins.py b/tests/unit/test_manager_agreement_plugins.py index 2c48a3bb..8ca738cb 100644 --- a/tests/unit/test_manager_agreement_plugins.py +++ b/tests/unit/test_manager_agreement_plugins.py @@ -37,7 +37,9 @@ def test_linear_score_plugin(kwargs, property_value, expected, mocker): @pytest.mark.parametrize("expected_value", (0.876, 0.0, 0.2, 0.5, 1)) def test_random_score_plugin(mocker, expected_value): - mocker.patch("golem.managers.agreement.plugins.random", mocker.Mock(return_value=expected_value)) + mocker.patch( + "golem.managers.agreement.plugins.random", mocker.Mock(return_value=expected_value) + ) proposal_id = "foo" From 38750ef82c04c8883727c5372405fc2888cd76a0 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 20 Jul 2023 11:39:37 +0200 Subject: [PATCH 089/123] Fixed checks_typing --- golem/event_bus/in_memory/event_bus.py | 4 +-- golem/managers/agreement/plugins.py | 48 +++++++++++++++++--------- golem/managers/agreement/pricings.py | 5 ++- golem/managers/mixins.py | 4 +-- golem/managers/negotiation/plugins.py | 2 +- 5 files changed, 38 insertions(+), 25 deletions(-) diff --git a/golem/event_bus/in_memory/event_bus.py b/golem/event_bus/in_memory/event_bus.py index 88bce4d1..a5a81655 100644 --- a/golem/event_bus/in_memory/event_bus.py +++ b/golem/event_bus/in_memory/event_bus.py @@ -22,8 +22,8 @@ class _CallbackInfo: class InMemoryEventBus(EventBus[_CallbackHandler]): def __init__(self): - self._callbacks: DefaultDict[Type[TEvent], List[_CallbackInfo]] = defaultdict(list) - self._event_queue: asyncio.Queue[TEvent] = asyncio.Queue() + self._callbacks: DefaultDict[Type[Event], List[_CallbackInfo]] = defaultdict(list) + self._event_queue: asyncio.Queue[Event] = asyncio.Queue() self._process_event_queue_loop_task: Optional[asyncio.Task] = None async def start(self): diff --git a/golem/managers/agreement/plugins.py b/golem/managers/agreement/plugins.py index 426a0fa7..1f6784cd 100644 --- a/golem/managers/agreement/plugins.py +++ b/golem/managers/agreement/plugins.py @@ -1,5 +1,5 @@ from random import random -from typing import Callable, Optional, Sequence, Tuple, Union +from typing import Callable, List, Optional, Sequence, Tuple, Union from golem.managers.base import ManagerPluginException, ManagerScorePlugin, ProposalPluginResult from golem.payload.constraints import PropertyName @@ -99,8 +99,8 @@ class MapScore(ManagerScorePlugin): def __init__( self, callback: Callable[[ProposalData], Optional[float]], - normalize=False, - normalize_flip=False, + normalize: bool = False, + normalize_flip: bool = False, ) -> None: self._callback = callback self._normalize = normalize @@ -109,17 +109,31 @@ def __init__( def __call__(self, proposals_data: Sequence[ProposalData]) -> ProposalPluginResult: result = [self._callback(proposal_data) for proposal_data in proposals_data] - if self._normalize and result: - result_max = max(result) - result_min = min(result) - result_div = result_max - result_min - - if result_div == 0: - return result - - result = [(v - result_min) / result_div for v in result] - - if self._normalize_flip: - result = [1 - v for v in result] - - return result + if not self._normalize or result is None: + return result + + filtered = filter(None, result) + result_max = max(filtered) + result_min = min(filtered) + result_div = result_max - result_min + + if result_div == 0: + return result + + normalized_result: List[Optional[float]] = [] + for v in result: + if v is not None: + normalized_result.append((v - result_min) / result_div) + else: + normalized_result.append(v) + + if not self._normalize_flip: + return normalized_result + + flipped_result: List[Optional[float]] = [] + for v in normalized_result: + if v is not None: + flipped_result.append(1 - v) + else: + flipped_result.append(v) + return flipped_result diff --git a/golem/managers/agreement/pricings.py b/golem/managers/agreement/pricings.py index 954b2d8a..ebb51e9c 100644 --- a/golem/managers/agreement/pricings.py +++ b/golem/managers/agreement/pricings.py @@ -2,11 +2,10 @@ from datetime import timedelta from typing import Optional, Tuple -from golem.managers.base import PricingCallable from golem.resources import ProposalData -class LinearAverageCostPricing(PricingCallable): +class LinearAverageCostPricing: def __init__(self, average_cpu_load: float, average_duration: timedelta) -> None: self._average_cpu_load = average_cpu_load self._average_duration = average_duration @@ -40,7 +39,7 @@ def _get_linear_coeffs( return None - return coeffs + return tuple(float(c) for c in coeffs) # type: ignore[return-value] def _calculate_cost( self, price_duration_sec: float, price_cpu_sec: float, price_initial: float diff --git a/golem/managers/mixins.py b/golem/managers/mixins.py index 12b84ab2..602877b5 100644 --- a/golem/managers/mixins.py +++ b/golem/managers/mixins.py @@ -104,7 +104,7 @@ async def _do_scoring(self, proposals: Sequence[Proposal]): async def _run_plugins( self, proposals_data: Sequence[ProposalData] ) -> Sequence[Tuple[float, Sequence[float]]]: - proposal_scores = [] + proposal_scores: List[Tuple[float, Sequence[float]]] = [] for plugin in self._plugins: if isinstance(plugin, (list, tuple)): @@ -117,7 +117,7 @@ async def _run_plugins( if inspect.isawaitable(plugin_scores): plugin_scores = await plugin_scores - proposal_scores.append((weight, plugin_scores)) + proposal_scores.append((weight, plugin_scores)) # type: ignore[arg-type] return proposal_scores diff --git a/golem/managers/negotiation/plugins.py b/golem/managers/negotiation/plugins.py index 0ebe7071..d9af9022 100644 --- a/golem/managers/negotiation/plugins.py +++ b/golem/managers/negotiation/plugins.py @@ -73,5 +73,5 @@ async def __call__(self, demand_data: DemandData, proposal_data: ProposalData) - if cost is None and self.reject_on_unpricable: raise RejectProposal("Can't estimate costs!") - if self._cost <= cost: + if cost is not None and self._cost <= cost: raise RejectProposal(f"Exceeds estimated costs of `{self._cost}`!") From 9df163d648faf69180677de3ed05590dc9ea67e2 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 20 Jul 2023 11:52:36 +0200 Subject: [PATCH 090/123] Fix test_constraints_serialize --- tests/unit/test_payload_cons.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_payload_cons.py b/tests/unit/test_payload_cons.py index fb996b41..44040cb0 100644 --- a/tests/unit/test_payload_cons.py +++ b/tests/unit/test_payload_cons.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone from enum import Enum import pytest @@ -16,13 +16,13 @@ def test_constraints_serialize(): Constraint("foo", "=", "bar"), Constraint("int_field", "=", 123), Constraint("float_field", "=", 1.5), - Constraint("datetime_field", "=", datetime(2023, 1, 2)), + Constraint("datetime_field", "=", datetime(2023, 1, 2, tzinfo=timezone.utc)), Constraint("enum_field", "=", ExampleEnum.FOO), Constraint( "list_field", "=", [ - datetime(2023, 1, 2), + datetime(2023, 1, 2, tzinfo=timezone.utc), ExampleEnum.FOO, ], ), @@ -35,9 +35,9 @@ def test_constraints_serialize(): "(&(foo=bar)\n" "\t(int_field=123)\n" "\t(float_field=1.5)\n" - "\t(datetime_field=1672614000000)\n" + "\t(datetime_field=1672617600000)\n" "\t(enum_field=BAR)\n" - "\t(&(list_field=1672614000000)\n" + "\t(&(list_field=1672617600000)\n" "\t(list_field=BAR))\n" "\t(|(some.other.field=works!)))" ) From 33a2bdf0c04dc999264a2c50dc96bc4677bdc9ea Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 20 Jul 2023 12:02:36 +0200 Subject: [PATCH 091/123] Fixed list of all None in MapScore --- golem/managers/agreement/plugins.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/golem/managers/agreement/plugins.py b/golem/managers/agreement/plugins.py index 1f6784cd..4ba41405 100644 --- a/golem/managers/agreement/plugins.py +++ b/golem/managers/agreement/plugins.py @@ -112,7 +112,10 @@ def __call__(self, proposals_data: Sequence[ProposalData]) -> ProposalPluginResu if not self._normalize or result is None: return result - filtered = filter(None, result) + filtered = list(filter(None, result)) + if not filtered: + return result + result_max = max(filtered) result_min = min(filtered) result_div = result_max - result_min From 7c719d363668a319c0b624e5cf2fc37e6d8f6e03 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 20 Jul 2023 12:10:19 +0200 Subject: [PATCH 092/123] Add debug message for CI unit test --- golem/payload/mixins.py | 1 + 1 file changed, 1 insertion(+) diff --git a/golem/payload/mixins.py b/golem/payload/mixins.py index 612ee099..6d6a7fd4 100644 --- a/golem/payload/mixins.py +++ b/golem/payload/mixins.py @@ -17,6 +17,7 @@ def _serialize_value(cls, value: Any) -> Any: return type(value)(cls._serialize_value(v) for v in value) if isinstance(value, datetime.datetime): + print(f"\nSerializing {repr(value)} into {value.timestamp()}\n") return int(value.timestamp() * 1000) if isinstance(value, enum.Enum): From 912c8ce130b602f413c314bfefbd6438d758a092 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 20 Jul 2023 12:55:07 +0200 Subject: [PATCH 093/123] Fix for test_constraints_serialize --- golem/payload/mixins.py | 1 - tests/unit/test_payload_cons.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/golem/payload/mixins.py b/golem/payload/mixins.py index 6d6a7fd4..612ee099 100644 --- a/golem/payload/mixins.py +++ b/golem/payload/mixins.py @@ -17,7 +17,6 @@ def _serialize_value(cls, value: Any) -> Any: return type(value)(cls._serialize_value(v) for v in value) if isinstance(value, datetime.datetime): - print(f"\nSerializing {repr(value)} into {value.timestamp()}\n") return int(value.timestamp() * 1000) if isinstance(value, enum.Enum): diff --git a/tests/unit/test_payload_cons.py b/tests/unit/test_payload_cons.py index 44040cb0..ae34cc0b 100644 --- a/tests/unit/test_payload_cons.py +++ b/tests/unit/test_payload_cons.py @@ -1,4 +1,4 @@ -from datetime import datetime, timezone +from datetime import datetime from enum import Enum import pytest @@ -16,13 +16,13 @@ def test_constraints_serialize(): Constraint("foo", "=", "bar"), Constraint("int_field", "=", 123), Constraint("float_field", "=", 1.5), - Constraint("datetime_field", "=", datetime(2023, 1, 2, tzinfo=timezone.utc)), + Constraint("datetime_field", "=", datetime.fromtimestamp(1672617600)), Constraint("enum_field", "=", ExampleEnum.FOO), Constraint( "list_field", "=", [ - datetime(2023, 1, 2, tzinfo=timezone.utc), + datetime.fromtimestamp(1672617600), ExampleEnum.FOO, ], ), From 7a706c20124bb9534f11d601b45978971f90c573 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 20 Jul 2023 13:17:20 +0200 Subject: [PATCH 094/123] Fix PropsConsSerializerMixin datetime manipulations --- golem/payload/mixins.py | 2 +- tests/unit/test_payload_cons.py | 4 ++-- tests/unit/test_payload_props.py | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/golem/payload/mixins.py b/golem/payload/mixins.py index 612ee099..04b73f3d 100644 --- a/golem/payload/mixins.py +++ b/golem/payload/mixins.py @@ -17,7 +17,7 @@ def _serialize_value(cls, value: Any) -> Any: return type(value)(cls._serialize_value(v) for v in value) if isinstance(value, datetime.datetime): - return int(value.timestamp() * 1000) + return int(value.replace(tzinfo=datetime.timezone.utc).timestamp() * 1000) if isinstance(value, enum.Enum): return value.value diff --git a/tests/unit/test_payload_cons.py b/tests/unit/test_payload_cons.py index ae34cc0b..abaf08d5 100644 --- a/tests/unit/test_payload_cons.py +++ b/tests/unit/test_payload_cons.py @@ -16,13 +16,13 @@ def test_constraints_serialize(): Constraint("foo", "=", "bar"), Constraint("int_field", "=", 123), Constraint("float_field", "=", 1.5), - Constraint("datetime_field", "=", datetime.fromtimestamp(1672617600)), + Constraint("datetime_field", "=", datetime.utcfromtimestamp(1672617600)), Constraint("enum_field", "=", ExampleEnum.FOO), Constraint( "list_field", "=", [ - datetime.fromtimestamp(1672617600), + datetime.utcfromtimestamp(1672617600), ExampleEnum.FOO, ], ), diff --git a/tests/unit/test_payload_props.py b/tests/unit/test_payload_props.py index 6a8bbb9d..2c9121d8 100644 --- a/tests/unit/test_payload_props.py +++ b/tests/unit/test_payload_props.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone from enum import Enum from golem.payload import Properties @@ -37,10 +37,10 @@ def test_property_serialize(): "foo": "bar", "int_field": 123, "float_field": 1.5, - "datetime_field": datetime(2023, 1, 2), + "datetime_field": datetime(2023, 1, 2, tzinfo=timezone.utc), "enum_field": ExampleEnum.FOO, "list_field": [ - datetime(2023, 1, 2), + datetime(2023, 1, 2, tzinfo=timezone.utc), ExampleEnum.FOO, ], "nulled_field": None, @@ -53,10 +53,10 @@ def test_property_serialize(): "foo": "bar", "int_field": 123, "float_field": 1.5, - "datetime_field": 1672614000000, + "datetime_field": 1672617600000, "enum_field": "BAR", "list_field": [ - 1672614000000, + 1672617600000, "BAR", ], } From c244a9bc9dab42fc41f321c3a26ac3e52c7c3846 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 20 Jul 2023 13:23:07 +0200 Subject: [PATCH 095/123] Change how actions install poetry --- .github/actions/integration-tests-goth/action.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/actions/integration-tests-goth/action.yml b/.github/actions/integration-tests-goth/action.yml index ed799be4..032381ca 100644 --- a/.github/actions/integration-tests-goth/action.yml +++ b/.github/actions/integration-tests-goth/action.yml @@ -17,9 +17,7 @@ runs: with: python-version: ${{ inputs.PYTHON_VERSION }} - name: Install and configure Poetry - uses: snok/install-poetry@v1 - with: - version: 1.4.1 + run: python -m pip install -U pip setuptools poetry==1.3.2 - name: Install dependencies shell: bash From eac3afc6ce32886fe6010a230eba1947a556c1e0 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 20 Jul 2023 13:24:44 +0200 Subject: [PATCH 096/123] FIx action change --- .github/actions/integration-tests-goth/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/integration-tests-goth/action.yml b/.github/actions/integration-tests-goth/action.yml index 032381ca..e6c6d5ed 100644 --- a/.github/actions/integration-tests-goth/action.yml +++ b/.github/actions/integration-tests-goth/action.yml @@ -17,6 +17,7 @@ runs: with: python-version: ${{ inputs.PYTHON_VERSION }} - name: Install and configure Poetry + shell: bash run: python -m pip install -U pip setuptools poetry==1.3.2 - name: Install dependencies From 9e366fa8b030f9642171550e374eb9eca741ccda Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 20 Jul 2023 13:31:42 +0200 Subject: [PATCH 097/123] Bump poetry in integration tests action --- .github/actions/integration-tests-goth/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/integration-tests-goth/action.yml b/.github/actions/integration-tests-goth/action.yml index e6c6d5ed..16d2eebc 100644 --- a/.github/actions/integration-tests-goth/action.yml +++ b/.github/actions/integration-tests-goth/action.yml @@ -18,7 +18,7 @@ runs: python-version: ${{ inputs.PYTHON_VERSION }} - name: Install and configure Poetry shell: bash - run: python -m pip install -U pip setuptools poetry==1.3.2 + run: python -m pip install -U pip setuptools poetry==1.5.1 - name: Install dependencies shell: bash From 3e1cc8940e912c22997f892d441f0fd7139a2c12 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 20 Jul 2023 13:46:29 +0200 Subject: [PATCH 098/123] Fix ActivityPrepareReleaseMixin typing --- golem/managers/activity/mixins.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/golem/managers/activity/mixins.py b/golem/managers/activity/mixins.py index c2b6b4db..f107f581 100644 --- a/golem/managers/activity/mixins.py +++ b/golem/managers/activity/mixins.py @@ -1,6 +1,7 @@ import logging from typing import Awaitable, Callable, Optional +from golem.event_bus.base import EventBus from golem.managers.activity.defaults import default_on_activity_start, default_on_activity_stop from golem.managers.agreement.events import AgreementReleased from golem.managers.base import WorkContext @@ -11,6 +12,8 @@ class ActivityPrepareReleaseMixin: + _event_bus: EventBus + def __init__( self, on_activity_start: Optional[ @@ -52,5 +55,4 @@ async def _release_activity(self, activity: Activity) -> None: ) event = AgreementReleased(activity.parent) - # TODO how to give access to event bus without coupling with GolemNode? - await self._event_bus.emit(event) # type: ignore[attr-defined] + await self._event_bus.emit(event) From 8947ecde04615023b04271fbced456c37c77d3b3 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Mon, 24 Jul 2023 11:15:50 +0200 Subject: [PATCH 099/123] Debug frozen CI test --- tests/integration/test_1.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/integration/test_1.py b/tests/integration/test_1.py index dc6f92ad..e4e134fc 100644 --- a/tests/integration/test_1.py +++ b/tests/integration/test_1.py @@ -65,8 +65,13 @@ async def test_demand(golem): allocation = await golem.create_allocation(1) demand = await golem.create_demand(PAYLOAD, allocations=[allocation]) + print("test_demand: getting proposals") + async for proposal in demand.initial_proposals(): + print("test_demand: got proposal") break + + print("test_demand: getting proposals done") await demand.get_data() From adc01ee3248168566a778d6f1616ec81b5db26e7 Mon Sep 17 00:00:00 2001 From: approxit Date: Mon, 24 Jul 2023 12:12:17 +0200 Subject: [PATCH 100/123] Initial buffer implemtation --- .requirements.txt | 448 ++++++++++++++++++++++++++++++++ golem/utils/buffer/__init__.py | 0 golem/utils/buffer/base.py | 167 ++++++++++++ pyproject.toml | 2 +- tests/integration/test_1.py | 2 +- tests/unit/utils/test_buffer.py | 177 +++++++++++++ 6 files changed, 794 insertions(+), 2 deletions(-) create mode 100644 .requirements.txt create mode 100644 golem/utils/buffer/__init__.py create mode 100644 golem/utils/buffer/base.py create mode 100644 tests/unit/utils/test_buffer.py diff --git a/.requirements.txt b/.requirements.txt new file mode 100644 index 00000000..9fbb25d2 --- /dev/null +++ b/.requirements.txt @@ -0,0 +1,448 @@ +aiohttp==3.8.4 ; python_full_version >= "3.8.0" and python_version < "4.0" \ + --hash=sha256:03543dcf98a6619254b409be2d22b51f21ec66272be4ebda7b04e6412e4b2e14 \ + --hash=sha256:03baa76b730e4e15a45f81dfe29a8d910314143414e528737f8589ec60cf7391 \ + --hash=sha256:0a63f03189a6fa7c900226e3ef5ba4d3bd047e18f445e69adbd65af433add5a2 \ + --hash=sha256:10c8cefcff98fd9168cdd86c4da8b84baaa90bf2da2269c6161984e6737bf23e \ + --hash=sha256:147ae376f14b55f4f3c2b118b95be50a369b89b38a971e80a17c3fd623f280c9 \ + --hash=sha256:176a64b24c0935869d5bbc4c96e82f89f643bcdf08ec947701b9dbb3c956b7dd \ + --hash=sha256:17b79c2963db82086229012cff93ea55196ed31f6493bb1ccd2c62f1724324e4 \ + --hash=sha256:1a45865451439eb320784918617ba54b7a377e3501fb70402ab84d38c2cd891b \ + --hash=sha256:1b3ea7edd2d24538959c1c1abf97c744d879d4e541d38305f9bd7d9b10c9ec41 \ + --hash=sha256:22f6eab15b6db242499a16de87939a342f5a950ad0abaf1532038e2ce7d31567 \ + --hash=sha256:3032dcb1c35bc330134a5b8a5d4f68c1a87252dfc6e1262c65a7e30e62298275 \ + --hash=sha256:33587f26dcee66efb2fff3c177547bd0449ab7edf1b73a7f5dea1e38609a0c54 \ + --hash=sha256:34ce9f93a4a68d1272d26030655dd1b58ff727b3ed2a33d80ec433561b03d67a \ + --hash=sha256:3a80464982d41b1fbfe3154e440ba4904b71c1a53e9cd584098cd41efdb188ef \ + --hash=sha256:3b90467ebc3d9fa5b0f9b6489dfb2c304a1db7b9946fa92aa76a831b9d587e99 \ + --hash=sha256:3d89efa095ca7d442a6d0cbc755f9e08190ba40069b235c9886a8763b03785da \ + --hash=sha256:3d8ef1a630519a26d6760bc695842579cb09e373c5f227a21b67dc3eb16cfea4 \ + --hash=sha256:3f43255086fe25e36fd5ed8f2ee47477408a73ef00e804cb2b5cba4bf2ac7f5e \ + --hash=sha256:40653609b3bf50611356e6b6554e3a331f6879fa7116f3959b20e3528783e699 \ + --hash=sha256:41a86a69bb63bb2fc3dc9ad5ea9f10f1c9c8e282b471931be0268ddd09430b04 \ + --hash=sha256:493f5bc2f8307286b7799c6d899d388bbaa7dfa6c4caf4f97ef7521b9cb13719 \ + --hash=sha256:4a6cadebe132e90cefa77e45f2d2f1a4b2ce5c6b1bfc1656c1ddafcfe4ba8131 \ + --hash=sha256:4c745b109057e7e5f1848c689ee4fb3a016c8d4d92da52b312f8a509f83aa05e \ + --hash=sha256:4d347a172f866cd1d93126d9b239fcbe682acb39b48ee0873c73c933dd23bd0f \ + --hash=sha256:4dac314662f4e2aa5009977b652d9b8db7121b46c38f2073bfeed9f4049732cd \ + --hash=sha256:4ddaae3f3d32fc2cb4c53fab020b69a05c8ab1f02e0e59665c6f7a0d3a5be54f \ + --hash=sha256:5393fb786a9e23e4799fec788e7e735de18052f83682ce2dfcabaf1c00c2c08e \ + --hash=sha256:59f029a5f6e2d679296db7bee982bb3d20c088e52a2977e3175faf31d6fb75d1 \ + --hash=sha256:5a7bdf9e57126dc345b683c3632e8ba317c31d2a41acd5800c10640387d193ed \ + --hash=sha256:5b3f2e06a512e94722886c0827bee9807c86a9f698fac6b3aee841fab49bbfb4 \ + --hash=sha256:5ce45967538fb747370308d3145aa68a074bdecb4f3a300869590f725ced69c1 \ + --hash=sha256:5e14f25765a578a0a634d5f0cd1e2c3f53964553a00347998dfdf96b8137f777 \ + --hash=sha256:618c901dd3aad4ace71dfa0f5e82e88b46ef57e3239fc7027773cb6d4ed53531 \ + --hash=sha256:652b1bff4f15f6287550b4670546a2947f2a4575b6c6dff7760eafb22eacbf0b \ + --hash=sha256:6c08e8ed6fa3d477e501ec9db169bfac8140e830aa372d77e4a43084d8dd91ab \ + --hash=sha256:6ddb2a2026c3f6a68c3998a6c47ab6795e4127315d2e35a09997da21865757f8 \ + --hash=sha256:6e601588f2b502c93c30cd5a45bfc665faaf37bbe835b7cfd461753068232074 \ + --hash=sha256:6e74dd54f7239fcffe07913ff8b964e28b712f09846e20de78676ce2a3dc0bfc \ + --hash=sha256:7235604476a76ef249bd64cb8274ed24ccf6995c4a8b51a237005ee7a57e8643 \ + --hash=sha256:7ab43061a0c81198d88f39aaf90dae9a7744620978f7ef3e3708339b8ed2ef01 \ + --hash=sha256:7c7837fe8037e96b6dd5cfcf47263c1620a9d332a87ec06a6ca4564e56bd0f36 \ + --hash=sha256:80575ba9377c5171407a06d0196b2310b679dc752d02a1fcaa2bc20b235dbf24 \ + --hash=sha256:80a37fe8f7c1e6ce8f2d9c411676e4bc633a8462844e38f46156d07a7d401654 \ + --hash=sha256:8189c56eb0ddbb95bfadb8f60ea1b22fcfa659396ea36f6adcc521213cd7b44d \ + --hash=sha256:854f422ac44af92bfe172d8e73229c270dc09b96535e8a548f99c84f82dde241 \ + --hash=sha256:880e15bb6dad90549b43f796b391cfffd7af373f4646784795e20d92606b7a51 \ + --hash=sha256:8b631e26df63e52f7cce0cce6507b7a7f1bc9b0c501fcde69742130b32e8782f \ + --hash=sha256:8c29c77cc57e40f84acef9bfb904373a4e89a4e8b74e71aa8075c021ec9078c2 \ + --hash=sha256:91f6d540163f90bbaef9387e65f18f73ffd7c79f5225ac3d3f61df7b0d01ad15 \ + --hash=sha256:92c0cea74a2a81c4c76b62ea1cac163ecb20fb3ba3a75c909b9fa71b4ad493cf \ + --hash=sha256:9bcb89336efa095ea21b30f9e686763f2be4478f1b0a616969551982c4ee4c3b \ + --hash=sha256:a1f4689c9a1462f3df0a1f7e797791cd6b124ddbee2b570d34e7f38ade0e2c71 \ + --hash=sha256:a3fec6a4cb5551721cdd70473eb009d90935b4063acc5f40905d40ecfea23e05 \ + --hash=sha256:a5d794d1ae64e7753e405ba58e08fcfa73e3fad93ef9b7e31112ef3c9a0efb52 \ + --hash=sha256:a86d42d7cba1cec432d47ab13b6637bee393a10f664c425ea7b305d1301ca1a3 \ + --hash=sha256:adfbc22e87365a6e564c804c58fc44ff7727deea782d175c33602737b7feadb6 \ + --hash=sha256:aeb29c84bb53a84b1a81c6c09d24cf33bb8432cc5c39979021cc0f98c1292a1a \ + --hash=sha256:aede4df4eeb926c8fa70de46c340a1bc2c6079e1c40ccf7b0eae1313ffd33519 \ + --hash=sha256:b744c33b6f14ca26b7544e8d8aadff6b765a80ad6164fb1a430bbadd593dfb1a \ + --hash=sha256:b7a00a9ed8d6e725b55ef98b1b35c88013245f35f68b1b12c5cd4100dddac333 \ + --hash=sha256:bb96fa6b56bb536c42d6a4a87dfca570ff8e52de2d63cabebfd6fb67049c34b6 \ + --hash=sha256:bbcf1a76cf6f6dacf2c7f4d2ebd411438c275faa1dc0c68e46eb84eebd05dd7d \ + --hash=sha256:bca5f24726e2919de94f047739d0a4fc01372801a3672708260546aa2601bf57 \ + --hash=sha256:bf2e1a9162c1e441bf805a1fd166e249d574ca04e03b34f97e2928769e91ab5c \ + --hash=sha256:c4eb3b82ca349cf6fadcdc7abcc8b3a50ab74a62e9113ab7a8ebc268aad35bb9 \ + --hash=sha256:c6cc15d58053c76eacac5fa9152d7d84b8d67b3fde92709195cb984cfb3475ea \ + --hash=sha256:c6cd05ea06daca6ad6a4ca3ba7fe7dc5b5de063ff4daec6170ec0f9979f6c332 \ + --hash=sha256:c844fd628851c0bc309f3c801b3a3d58ce430b2ce5b359cd918a5a76d0b20cb5 \ + --hash=sha256:c9cb1565a7ad52e096a6988e2ee0397f72fe056dadf75d17fa6b5aebaea05622 \ + --hash=sha256:cab9401de3ea52b4b4c6971db5fb5c999bd4260898af972bf23de1c6b5dd9d71 \ + --hash=sha256:cd468460eefef601ece4428d3cf4562459157c0f6523db89365202c31b6daebb \ + --hash=sha256:d1e6a862b76f34395a985b3cd39a0d949ca80a70b6ebdea37d3ab39ceea6698a \ + --hash=sha256:d1f9282c5f2b5e241034a009779e7b2a1aa045f667ff521e7948ea9b56e0c5ff \ + --hash=sha256:d265f09a75a79a788237d7f9054f929ced2e69eb0bb79de3798c468d8a90f945 \ + --hash=sha256:db3fc6120bce9f446d13b1b834ea5b15341ca9ff3f335e4a951a6ead31105480 \ + --hash=sha256:dbf3a08a06b3f433013c143ebd72c15cac33d2914b8ea4bea7ac2c23578815d6 \ + --hash=sha256:de04b491d0e5007ee1b63a309956eaed959a49f5bb4e84b26c8f5d49de140fa9 \ + --hash=sha256:e4b09863aae0dc965c3ef36500d891a3ff495a2ea9ae9171e4519963c12ceefd \ + --hash=sha256:e595432ac259af2d4630008bf638873d69346372d38255774c0e286951e8b79f \ + --hash=sha256:e75b89ac3bd27d2d043b234aa7b734c38ba1b0e43f07787130a0ecac1e12228a \ + --hash=sha256:ea9eb976ffdd79d0e893869cfe179a8f60f152d42cb64622fca418cd9b18dc2a \ + --hash=sha256:eafb3e874816ebe2a92f5e155f17260034c8c341dad1df25672fb710627c6949 \ + --hash=sha256:ee3c36df21b5714d49fc4580247947aa64bcbe2939d1b77b4c8dcb8f6c9faecc \ + --hash=sha256:f352b62b45dff37b55ddd7b9c0c8672c4dd2eb9c0f9c11d395075a84e2c40f75 \ + --hash=sha256:fabb87dd8850ef0f7fe2b366d44b77d7e6fa2ea87861ab3844da99291e81e60f \ + --hash=sha256:fe11310ae1e4cd560035598c3f29d86cef39a83d244c7466f95c27ae04850f10 \ + --hash=sha256:fe7ba4a51f33ab275515f66b0a236bcde4fb5561498fe8f898d4e549b2e4509f +aiosignal==1.3.1 ; python_full_version >= "3.8.0" and python_version < "4.0" \ + --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ + --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 +arpeggio==2.0.0 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ + --hash=sha256:448e332deb0e9ccd04046f1c6c14529d197f41bc2fdb3931e43fc209042fbdd3 \ + --hash=sha256:d6b03839019bb8a68785f9292ee6a36b1954eb84b925b84a6b8a5e1e26d3ed3d +async-exit-stack==1.0.1 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ + --hash=sha256:24de1ad6d0ff27be97c89d6709fa49bf20db179eaf1f4d2e6e9b4409b80e747d \ + --hash=sha256:9b43b17683b3438f428ef3bbec20689f5abbb052aa4b564c643397330adfaa99 +async-timeout==4.0.2 ; python_full_version >= "3.8.0" and python_version < "4.0" \ + --hash=sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15 \ + --hash=sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c +attrs==23.1.0 ; python_full_version >= "3.8.0" and python_version < "4.0" \ + --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ + --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 +certifi==2020.12.5 ; python_full_version >= "3.8.0" and python_version < "4.0" \ + --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \ + --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 +charset-normalizer==3.1.0 ; python_full_version >= "3.8.0" and python_version < "4.0" \ + --hash=sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6 \ + --hash=sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1 \ + --hash=sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e \ + --hash=sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373 \ + --hash=sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62 \ + --hash=sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230 \ + --hash=sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be \ + --hash=sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c \ + --hash=sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0 \ + --hash=sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448 \ + --hash=sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f \ + --hash=sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649 \ + --hash=sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d \ + --hash=sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0 \ + --hash=sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706 \ + --hash=sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a \ + --hash=sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59 \ + --hash=sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23 \ + --hash=sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5 \ + --hash=sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb \ + --hash=sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e \ + --hash=sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e \ + --hash=sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c \ + --hash=sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28 \ + --hash=sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d \ + --hash=sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41 \ + --hash=sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974 \ + --hash=sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce \ + --hash=sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f \ + --hash=sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1 \ + --hash=sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d \ + --hash=sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8 \ + --hash=sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017 \ + --hash=sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31 \ + --hash=sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7 \ + --hash=sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8 \ + --hash=sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e \ + --hash=sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14 \ + --hash=sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd \ + --hash=sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d \ + --hash=sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795 \ + --hash=sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b \ + --hash=sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b \ + --hash=sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b \ + --hash=sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203 \ + --hash=sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f \ + --hash=sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19 \ + --hash=sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1 \ + --hash=sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a \ + --hash=sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac \ + --hash=sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9 \ + --hash=sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0 \ + --hash=sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137 \ + --hash=sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f \ + --hash=sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6 \ + --hash=sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5 \ + --hash=sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909 \ + --hash=sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f \ + --hash=sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0 \ + --hash=sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324 \ + --hash=sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755 \ + --hash=sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb \ + --hash=sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854 \ + --hash=sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c \ + --hash=sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60 \ + --hash=sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84 \ + --hash=sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0 \ + --hash=sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b \ + --hash=sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1 \ + --hash=sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531 \ + --hash=sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1 \ + --hash=sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11 \ + --hash=sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326 \ + --hash=sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df \ + --hash=sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab +click==8.1.3 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ + --hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \ + --hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48 +colorama==0.4.6 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" and platform_system == "Windows" \ + --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ + --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 +dnspython==2.3.0 ; python_full_version >= "3.8.0" and python_version < "4.0" \ + --hash=sha256:224e32b03eb46be70e12ef6d64e0be123a64e621ab4c0822ff6d450d52a540b9 \ + --hash=sha256:89141536394f909066cabd112e3e1a37e4e654db00a25308b0f130bc3152eb46 +frozenlist==1.3.3 ; python_full_version >= "3.8.0" and python_version < "4.0" \ + --hash=sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c \ + --hash=sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f \ + --hash=sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a \ + --hash=sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784 \ + --hash=sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27 \ + --hash=sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d \ + --hash=sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3 \ + --hash=sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678 \ + --hash=sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a \ + --hash=sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483 \ + --hash=sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8 \ + --hash=sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf \ + --hash=sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99 \ + --hash=sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c \ + --hash=sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48 \ + --hash=sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5 \ + --hash=sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56 \ + --hash=sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e \ + --hash=sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1 \ + --hash=sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401 \ + --hash=sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4 \ + --hash=sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e \ + --hash=sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649 \ + --hash=sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a \ + --hash=sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d \ + --hash=sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0 \ + --hash=sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6 \ + --hash=sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d \ + --hash=sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b \ + --hash=sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6 \ + --hash=sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf \ + --hash=sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef \ + --hash=sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7 \ + --hash=sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842 \ + --hash=sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba \ + --hash=sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420 \ + --hash=sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b \ + --hash=sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d \ + --hash=sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332 \ + --hash=sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936 \ + --hash=sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816 \ + --hash=sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91 \ + --hash=sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420 \ + --hash=sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448 \ + --hash=sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411 \ + --hash=sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4 \ + --hash=sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32 \ + --hash=sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b \ + --hash=sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0 \ + --hash=sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530 \ + --hash=sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669 \ + --hash=sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7 \ + --hash=sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1 \ + --hash=sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5 \ + --hash=sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce \ + --hash=sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4 \ + --hash=sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e \ + --hash=sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2 \ + --hash=sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d \ + --hash=sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9 \ + --hash=sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642 \ + --hash=sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0 \ + --hash=sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703 \ + --hash=sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb \ + --hash=sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1 \ + --hash=sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13 \ + --hash=sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab \ + --hash=sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38 \ + --hash=sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb \ + --hash=sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb \ + --hash=sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81 \ + --hash=sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8 \ + --hash=sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd \ + --hash=sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4 +idna==3.4 ; python_full_version >= "3.8.0" and python_version < "4.0" \ + --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ + --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 +jsonrpc-base==1.1.0 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ + --hash=sha256:7f374c57bfa1cb16d1f340d270bc0d9f1f5608fb1ac6c9ea15768c0e6ece48b7 +multidict==6.0.4 ; python_full_version >= "3.8.0" and python_version < "4.0" \ + --hash=sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9 \ + --hash=sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8 \ + --hash=sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03 \ + --hash=sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710 \ + --hash=sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161 \ + --hash=sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664 \ + --hash=sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569 \ + --hash=sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067 \ + --hash=sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313 \ + --hash=sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706 \ + --hash=sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2 \ + --hash=sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636 \ + --hash=sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49 \ + --hash=sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93 \ + --hash=sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603 \ + --hash=sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0 \ + --hash=sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60 \ + --hash=sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4 \ + --hash=sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e \ + --hash=sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1 \ + --hash=sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60 \ + --hash=sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951 \ + --hash=sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc \ + --hash=sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe \ + --hash=sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95 \ + --hash=sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d \ + --hash=sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8 \ + --hash=sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed \ + --hash=sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2 \ + --hash=sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775 \ + --hash=sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87 \ + --hash=sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c \ + --hash=sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2 \ + --hash=sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98 \ + --hash=sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3 \ + --hash=sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe \ + --hash=sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78 \ + --hash=sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660 \ + --hash=sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176 \ + --hash=sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e \ + --hash=sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988 \ + --hash=sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c \ + --hash=sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c \ + --hash=sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0 \ + --hash=sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449 \ + --hash=sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f \ + --hash=sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde \ + --hash=sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5 \ + --hash=sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d \ + --hash=sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac \ + --hash=sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a \ + --hash=sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9 \ + --hash=sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca \ + --hash=sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11 \ + --hash=sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35 \ + --hash=sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063 \ + --hash=sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b \ + --hash=sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982 \ + --hash=sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258 \ + --hash=sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1 \ + --hash=sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52 \ + --hash=sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480 \ + --hash=sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7 \ + --hash=sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461 \ + --hash=sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d \ + --hash=sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc \ + --hash=sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779 \ + --hash=sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a \ + --hash=sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547 \ + --hash=sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0 \ + --hash=sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171 \ + --hash=sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf \ + --hash=sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d \ + --hash=sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba +prettytable==3.7.0 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ + --hash=sha256:ef8334ee40b7ec721651fc4d37ecc7bb2ef55fde5098d994438f0dfdaa385c0c \ + --hash=sha256:f4aaf2ed6e6062a82fd2e6e5289bbbe705ec2788fe401a3a1f62a1cea55526d2 +python-dateutil==2.8.2 ; python_full_version >= "3.8.0" and python_version < "4.0" \ + --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ + --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 +semantic-version==2.10.0 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ + --hash=sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c \ + --hash=sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177 +setuptools==67.8.0 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ + --hash=sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f \ + --hash=sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102 +six==1.16.0 ; python_full_version >= "3.8.0" and python_version < "4.0" \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 +srvresolver==0.3.5 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ + --hash=sha256:0a8973d340486830ddd057895769cb672c7cd902ae61c21339875ed2a9d7ec0f \ + --hash=sha256:0cbb756d929b267e03dac98ea6d9a188d5ce6c8d30ca5553ed6737fb5ddd1c09 +textx==3.1.1 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ + --hash=sha256:1f2bd936309f5f9092567e3f5a0c513d8292d5852c63a72096fe5a57018d0f50 \ + --hash=sha256:e2fb7d090ea4a71f8bf2f0303770275c42a5f73854b4e7f6adfcdbe368fab0fa +wcwidth==0.2.6 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ + --hash=sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e \ + --hash=sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0 +ya-aioclient==0.6.4 ; python_full_version >= "3.8.0" and python_version < "4.0" \ + --hash=sha256:1d36c2544c2d7629a00a2b1f18b4535e836586cae99bfbf5714ed5ca3f9caa0c \ + --hash=sha256:e5e5926e3a7d834082b05aeb986d8fb2f7fa46a1bd1bca2f3f1c872e1ba479ae +yarl==1.9.2 ; python_full_version >= "3.8.0" and python_version < "4.0" \ + --hash=sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571 \ + --hash=sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3 \ + --hash=sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3 \ + --hash=sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c \ + --hash=sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7 \ + --hash=sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04 \ + --hash=sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191 \ + --hash=sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea \ + --hash=sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4 \ + --hash=sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4 \ + --hash=sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095 \ + --hash=sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e \ + --hash=sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74 \ + --hash=sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef \ + --hash=sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33 \ + --hash=sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde \ + --hash=sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45 \ + --hash=sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf \ + --hash=sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b \ + --hash=sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac \ + --hash=sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0 \ + --hash=sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528 \ + --hash=sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716 \ + --hash=sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb \ + --hash=sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18 \ + --hash=sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72 \ + --hash=sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6 \ + --hash=sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582 \ + --hash=sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5 \ + --hash=sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368 \ + --hash=sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc \ + --hash=sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9 \ + --hash=sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be \ + --hash=sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a \ + --hash=sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80 \ + --hash=sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8 \ + --hash=sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6 \ + --hash=sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417 \ + --hash=sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574 \ + --hash=sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59 \ + --hash=sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608 \ + --hash=sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82 \ + --hash=sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1 \ + --hash=sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3 \ + --hash=sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d \ + --hash=sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8 \ + --hash=sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc \ + --hash=sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac \ + --hash=sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8 \ + --hash=sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955 \ + --hash=sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0 \ + --hash=sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367 \ + --hash=sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb \ + --hash=sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a \ + --hash=sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623 \ + --hash=sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2 \ + --hash=sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6 \ + --hash=sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7 \ + --hash=sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4 \ + --hash=sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051 \ + --hash=sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938 \ + --hash=sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8 \ + --hash=sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9 \ + --hash=sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3 \ + --hash=sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5 \ + --hash=sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9 \ + --hash=sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333 \ + --hash=sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185 \ + --hash=sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3 \ + --hash=sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560 \ + --hash=sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b \ + --hash=sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7 \ + --hash=sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78 \ + --hash=sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7 diff --git a/golem/utils/buffer/__init__.py b/golem/utils/buffer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/golem/utils/buffer/base.py b/golem/utils/buffer/base.py new file mode 100644 index 00000000..a0e320bd --- /dev/null +++ b/golem/utils/buffer/base.py @@ -0,0 +1,167 @@ +import asyncio +import logging +from abc import ABC, abstractmethod +from datetime import timedelta +from typing import Awaitable, Callable, Generic, List, MutableSequence, Optional, Sequence, TypeVar + +from golem.utils.asyncio import create_task_with_logging +from golem.utils.logging import trace_span + +TItem = TypeVar("TItem") + +logger = logging.getLogger(__name__) + + +class Buffer(ABC, Generic[TItem]): + @abstractmethod + async def get_item(self) -> TItem: + ... + + +async def default_update_callback( + items: MutableSequence[TItem], items_to_process: Sequence[TItem] +) -> None: + items.extend(items_to_process) + + +class SequenceFilledBuffer(Buffer[TItem], Generic[TItem]): + def __init__( + self, + fill_callback: Callable[[], Awaitable[TItem]], + min_size: int, + max_size: int, + update_callback: Callable[ + [MutableSequence[TItem], Sequence[TItem]], Awaitable[None] + ] = default_update_callback, + update_interval: Optional[timedelta] = None, + ): + self._fill_callback = fill_callback + self._min_size = min_size + self._max_size = max_size + self._update_callback = update_callback + self._update_interval = update_interval + + self._items_to_process: List[TItem] = [] + self._items_to_process_lock = asyncio.Lock() + self._items: List[TItem] = [] + self._items_condition = asyncio.Condition() + self._items_requested_tasks: List[asyncio.Task] = [] + self._items_requests_pending_event = asyncio.Event() + self._items_requests_ready_event = asyncio.Event() + + self._background_loop_task: Optional[asyncio.Task] = None + + @trace_span() + async def start(self, *, fill=False) -> None: + if self.is_started(): + raise RuntimeError("Already started!") + + self._background_loop_task = create_task_with_logging(self._background_loop()) + + if fill: + self._handle_item_requests() + + @trace_span() + async def stop(self) -> None: + if not self.is_started(): + raise RuntimeError("Already stopped!") + + if self._background_loop_task is not None: + self._background_loop_task.cancel() + self._background_loop_task = None + + def is_started(self) -> bool: + return self._background_loop_task is not None and not self._background_loop_task.done() + + @trace_span(show_results=True) + async def get_item(self) -> TItem: + if not self.is_started(): + raise RuntimeError("Not started!") + + async with self._items_condition: + if not self._items: + logger.debug("No items to get, requesting fill") + self._handle_item_requests() + + logger.debug("Waiting for any item to pick...") + + def check(): + logger.debug("check %s", len(self._items)) + return 0 < len(self._items) + + await self._items_condition.wait_for(check) + + item = self._items.pop(0) + + self._handle_item_requests() + + return item + + @trace_span() + def _handle_item_requests(self) -> None: + # Check if we need to request any additional items + items_len = len(self._items) + if self._min_size <= items_len: + logger.debug( + "Items count `%d` are not below min_size `%d`, skipping", items_len, self._min_size + ) + return + + items_to_request = self._max_size - items_len - len(self._items_requested_tasks) + + self._items_requests_pending_event.set() + + def on_completetion(task): + self._items_requested_tasks.remove(task) + + if not self._items_requested_tasks: + logger.debug("All requested items received") + self._items_requests_pending_event.clear() + self._items_requests_ready_event.set() + + for _ in range(items_to_request): + task = create_task_with_logging(self._fill()) + task.add_done_callback(on_completetion) + self._items_requested_tasks.append(task) + + logger.debug("Requested %d items", items_to_request) + + async def _background_loop(self) -> None: + while True: + # check if any items are ready to process + # check if all requested items are ready to process or timeout axceeded + # run update_callback + + logger.debug("Waiting for any item requests...") + await self._items_requests_pending_event.wait() + + timeout = ( + None if self._update_interval is None else self._update_interval.total_seconds() + ) + logger.debug("Some items were requested, waitng on all with timeout `%s`", timeout) + + try: + await asyncio.wait_for(self._items_requests_ready_event.wait(), timeout=timeout) + self._items_requests_ready_event.clear() + logger.debug("Got all requested items") + except asyncio.TimeoutError: + logger.debug("Waiting for all requested items timed out, updating anyways...") + + async with self._items_to_process_lock: + items_to_process = self._items_to_process[:] + self._items_to_process.clear() + + async with self._items_condition: + await self._update_callback(self._items, items_to_process) + + logger.debug(f"Item collection updated {self._items}") + + self._items_condition.notify_all() + + async def _fill(self): + item = await self._fill_callback() + + logger.debug("Requested item `%s` received", item) + + async with self._items_to_process_lock: + self._items_to_process.append(item) diff --git a/pyproject.toml b/pyproject.toml index 81cf4a55..a64e5df0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -149,7 +149,7 @@ target-version = ['py38'] [tool.pytest.ini_options] asyncio_mode = "auto" -addopts = "--cov golem --cov-report html --cov-report term-missing -sv" +addopts = "--cov golem --cov-report html --cov-report term-missing --no-cov-on-fail -sv" testspaths = [ "tests", ] diff --git a/tests/integration/test_1.py b/tests/integration/test_1.py index e4e134fc..5129a3cb 100644 --- a/tests/integration/test_1.py +++ b/tests/integration/test_1.py @@ -70,7 +70,7 @@ async def test_demand(golem): async for proposal in demand.initial_proposals(): print("test_demand: got proposal") break - + print("test_demand: getting proposals done") await demand.get_data() diff --git a/tests/unit/utils/test_buffer.py b/tests/unit/utils/test_buffer.py new file mode 100644 index 00000000..63d32785 --- /dev/null +++ b/tests/unit/utils/test_buffer.py @@ -0,0 +1,177 @@ +import asyncio +import logging +from datetime import timedelta + +import pytest + +from golem.utils.buffer.base import SequenceFilledBuffer + + +@pytest.fixture +def fill_callback(mocker): + return mocker.AsyncMock() + + +@pytest.fixture +def buffer_class(): + return SequenceFilledBuffer + + +@pytest.fixture +def create_buffer(fill_callback, buffer_class): + def _create_buffer(*args, **kwargs): + return buffer_class( + fill_callback=fill_callback, + *args, + **kwargs, + ) + + return _create_buffer + + +async def test_buffer_start_stop(create_buffer): + buffer = create_buffer(min_size=0, max_size=0) + + assert not buffer.is_started() + + with pytest.raises(RuntimeError, match="Already stopped!"): + await buffer.stop() + + with pytest.raises(RuntimeError, match="Not started!"): + await buffer.get_item() + + await buffer.start() + + with pytest.raises(RuntimeError, match="Already started!"): + await buffer.start() + + assert buffer.is_started() + + await buffer.stop() + + assert not buffer.is_started() + + +async def test_buffer_will_block_on_empty(create_buffer): + buffer = create_buffer(min_size=0, max_size=0) + + await buffer.start() + + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(buffer.get_item(), 0.1) + + await buffer.stop() + + +@pytest.mark.parametrize( + "min_size, max_size, fill, await_count", + ( + (3, 10, False, 0), + (3, 10, True, 10), + (3, 5, True, 5), + ), +) +async def test_buffer_start_with_fill( + create_buffer, fill_callback, min_size, max_size, fill, await_count +): + buffer = create_buffer(min_size=min_size, max_size=max_size) + + await buffer.start(fill=fill) + + await asyncio.sleep(0.01) + + assert fill_callback.await_count == await_count + + await buffer.stop() + + +async def test_buffer_get_item_will_on_empty_will_trigger_fill(create_buffer, fill_callback): + buffer = create_buffer(min_size=3, max_size=10) + + await buffer.start() + + await asyncio.sleep(0.01) + + assert fill_callback.await_count == 0 + + item = await asyncio.wait_for(buffer.get_item(), 0.05) + + assert item == fill_callback.mock_calls[0] + + assert fill_callback.await_count == 10 + + await buffer.stop() + + +async def test_buffer_get_item_will_trigger_fill_on_below_min_size(create_buffer, fill_callback): + buffer = create_buffer(min_size=3, max_size=10) + + await buffer.start(fill=True) + + await asyncio.sleep(0.01) + + assert fill_callback.await_count == 10 + + item = await asyncio.wait_for(buffer.get_item(), 0.05) + + assert item == fill_callback.mock_calls[0] + + assert fill_callback.await_count == 10 + + done, _ = await asyncio.wait([buffer.get_item() for _ in range(6)], timeout=0.05) + + assert [d.result() for d in done] == fill_callback.mock_calls[1:7] + + assert fill_callback.await_count == 10 + + item = await asyncio.wait_for(buffer.get_item(), 0.05) + + assert item == fill_callback.mock_calls[8] + + assert fill_callback.await_count == 18 + + await buffer.stop() + + +async def _test_buffer_fill_can_add_requests_while_other_requests_are_running( + buffer_class, mocker, caplog +): + caplog.set_level(logging.DEBUG, logger="golem.utils.buffer.base") + queue: asyncio.Queue[int] = asyncio.Queue() + + for i in range(6): + await queue.put(i) + + async def fill_callback(): + item = await queue.get() + queue.task_done() + return item + + mocked_fill_callback = mocker.AsyncMock(wraps=fill_callback) + + buffer = buffer_class( + fill_callback=mocked_fill_callback, + min_size=3, + max_size=6, + update_interval=timedelta(seconds=0.05), + ) + + await buffer.start(fill=True) + + await asyncio.sleep(0.1) + + assert mocked_fill_callback.await_count == 6 + + done, _ = await asyncio.wait( + [asyncio.create_task(buffer.get_item()) for _ in range(3)], timeout=0.1 + ) + + assert [d.result() for d in done] == list(range(3)) + + assert mocked_fill_callback.await_count == 6 + assert queue.qsize() == 0 + + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(buffer.get_item(), 0.1) + + await buffer.stop() From 5a5bd94bb0989ffc680c116a5a328c0222010a44 Mon Sep 17 00:00:00 2001 From: approxit Date: Mon, 24 Jul 2023 17:04:21 +0200 Subject: [PATCH 101/123] Additional Buffer tests --- .gitignore | 2 +- .requirements.txt | 448 ---------------------- golem/managers/demand/auto.py | 2 +- golem/utils/{buffer/base.py => buffer.py} | 33 +- golem/utils/buffer/__init__.py | 0 tests/unit/utils/test_buffer.py | 41 +- 6 files changed, 55 insertions(+), 471 deletions(-) delete mode 100644 .requirements.txt rename golem/utils/{buffer/base.py => buffer.py} (89%) delete mode 100644 golem/utils/buffer/__init__.py diff --git a/.gitignore b/.gitignore index 64ca5e0a..3701131b 100644 --- a/.gitignore +++ b/.gitignore @@ -165,6 +165,6 @@ build/ .envs/ # licheck artifacts -.requrements.txt +.requirements.txt temp/ diff --git a/.requirements.txt b/.requirements.txt deleted file mode 100644 index 9fbb25d2..00000000 --- a/.requirements.txt +++ /dev/null @@ -1,448 +0,0 @@ -aiohttp==3.8.4 ; python_full_version >= "3.8.0" and python_version < "4.0" \ - --hash=sha256:03543dcf98a6619254b409be2d22b51f21ec66272be4ebda7b04e6412e4b2e14 \ - --hash=sha256:03baa76b730e4e15a45f81dfe29a8d910314143414e528737f8589ec60cf7391 \ - --hash=sha256:0a63f03189a6fa7c900226e3ef5ba4d3bd047e18f445e69adbd65af433add5a2 \ - --hash=sha256:10c8cefcff98fd9168cdd86c4da8b84baaa90bf2da2269c6161984e6737bf23e \ - --hash=sha256:147ae376f14b55f4f3c2b118b95be50a369b89b38a971e80a17c3fd623f280c9 \ - --hash=sha256:176a64b24c0935869d5bbc4c96e82f89f643bcdf08ec947701b9dbb3c956b7dd \ - --hash=sha256:17b79c2963db82086229012cff93ea55196ed31f6493bb1ccd2c62f1724324e4 \ - --hash=sha256:1a45865451439eb320784918617ba54b7a377e3501fb70402ab84d38c2cd891b \ - --hash=sha256:1b3ea7edd2d24538959c1c1abf97c744d879d4e541d38305f9bd7d9b10c9ec41 \ - --hash=sha256:22f6eab15b6db242499a16de87939a342f5a950ad0abaf1532038e2ce7d31567 \ - --hash=sha256:3032dcb1c35bc330134a5b8a5d4f68c1a87252dfc6e1262c65a7e30e62298275 \ - --hash=sha256:33587f26dcee66efb2fff3c177547bd0449ab7edf1b73a7f5dea1e38609a0c54 \ - --hash=sha256:34ce9f93a4a68d1272d26030655dd1b58ff727b3ed2a33d80ec433561b03d67a \ - --hash=sha256:3a80464982d41b1fbfe3154e440ba4904b71c1a53e9cd584098cd41efdb188ef \ - --hash=sha256:3b90467ebc3d9fa5b0f9b6489dfb2c304a1db7b9946fa92aa76a831b9d587e99 \ - --hash=sha256:3d89efa095ca7d442a6d0cbc755f9e08190ba40069b235c9886a8763b03785da \ - --hash=sha256:3d8ef1a630519a26d6760bc695842579cb09e373c5f227a21b67dc3eb16cfea4 \ - --hash=sha256:3f43255086fe25e36fd5ed8f2ee47477408a73ef00e804cb2b5cba4bf2ac7f5e \ - --hash=sha256:40653609b3bf50611356e6b6554e3a331f6879fa7116f3959b20e3528783e699 \ - --hash=sha256:41a86a69bb63bb2fc3dc9ad5ea9f10f1c9c8e282b471931be0268ddd09430b04 \ - --hash=sha256:493f5bc2f8307286b7799c6d899d388bbaa7dfa6c4caf4f97ef7521b9cb13719 \ - --hash=sha256:4a6cadebe132e90cefa77e45f2d2f1a4b2ce5c6b1bfc1656c1ddafcfe4ba8131 \ - --hash=sha256:4c745b109057e7e5f1848c689ee4fb3a016c8d4d92da52b312f8a509f83aa05e \ - --hash=sha256:4d347a172f866cd1d93126d9b239fcbe682acb39b48ee0873c73c933dd23bd0f \ - --hash=sha256:4dac314662f4e2aa5009977b652d9b8db7121b46c38f2073bfeed9f4049732cd \ - --hash=sha256:4ddaae3f3d32fc2cb4c53fab020b69a05c8ab1f02e0e59665c6f7a0d3a5be54f \ - --hash=sha256:5393fb786a9e23e4799fec788e7e735de18052f83682ce2dfcabaf1c00c2c08e \ - --hash=sha256:59f029a5f6e2d679296db7bee982bb3d20c088e52a2977e3175faf31d6fb75d1 \ - --hash=sha256:5a7bdf9e57126dc345b683c3632e8ba317c31d2a41acd5800c10640387d193ed \ - --hash=sha256:5b3f2e06a512e94722886c0827bee9807c86a9f698fac6b3aee841fab49bbfb4 \ - --hash=sha256:5ce45967538fb747370308d3145aa68a074bdecb4f3a300869590f725ced69c1 \ - --hash=sha256:5e14f25765a578a0a634d5f0cd1e2c3f53964553a00347998dfdf96b8137f777 \ - --hash=sha256:618c901dd3aad4ace71dfa0f5e82e88b46ef57e3239fc7027773cb6d4ed53531 \ - --hash=sha256:652b1bff4f15f6287550b4670546a2947f2a4575b6c6dff7760eafb22eacbf0b \ - --hash=sha256:6c08e8ed6fa3d477e501ec9db169bfac8140e830aa372d77e4a43084d8dd91ab \ - --hash=sha256:6ddb2a2026c3f6a68c3998a6c47ab6795e4127315d2e35a09997da21865757f8 \ - --hash=sha256:6e601588f2b502c93c30cd5a45bfc665faaf37bbe835b7cfd461753068232074 \ - --hash=sha256:6e74dd54f7239fcffe07913ff8b964e28b712f09846e20de78676ce2a3dc0bfc \ - --hash=sha256:7235604476a76ef249bd64cb8274ed24ccf6995c4a8b51a237005ee7a57e8643 \ - --hash=sha256:7ab43061a0c81198d88f39aaf90dae9a7744620978f7ef3e3708339b8ed2ef01 \ - --hash=sha256:7c7837fe8037e96b6dd5cfcf47263c1620a9d332a87ec06a6ca4564e56bd0f36 \ - --hash=sha256:80575ba9377c5171407a06d0196b2310b679dc752d02a1fcaa2bc20b235dbf24 \ - --hash=sha256:80a37fe8f7c1e6ce8f2d9c411676e4bc633a8462844e38f46156d07a7d401654 \ - --hash=sha256:8189c56eb0ddbb95bfadb8f60ea1b22fcfa659396ea36f6adcc521213cd7b44d \ - --hash=sha256:854f422ac44af92bfe172d8e73229c270dc09b96535e8a548f99c84f82dde241 \ - --hash=sha256:880e15bb6dad90549b43f796b391cfffd7af373f4646784795e20d92606b7a51 \ - --hash=sha256:8b631e26df63e52f7cce0cce6507b7a7f1bc9b0c501fcde69742130b32e8782f \ - --hash=sha256:8c29c77cc57e40f84acef9bfb904373a4e89a4e8b74e71aa8075c021ec9078c2 \ - --hash=sha256:91f6d540163f90bbaef9387e65f18f73ffd7c79f5225ac3d3f61df7b0d01ad15 \ - --hash=sha256:92c0cea74a2a81c4c76b62ea1cac163ecb20fb3ba3a75c909b9fa71b4ad493cf \ - --hash=sha256:9bcb89336efa095ea21b30f9e686763f2be4478f1b0a616969551982c4ee4c3b \ - --hash=sha256:a1f4689c9a1462f3df0a1f7e797791cd6b124ddbee2b570d34e7f38ade0e2c71 \ - --hash=sha256:a3fec6a4cb5551721cdd70473eb009d90935b4063acc5f40905d40ecfea23e05 \ - --hash=sha256:a5d794d1ae64e7753e405ba58e08fcfa73e3fad93ef9b7e31112ef3c9a0efb52 \ - --hash=sha256:a86d42d7cba1cec432d47ab13b6637bee393a10f664c425ea7b305d1301ca1a3 \ - --hash=sha256:adfbc22e87365a6e564c804c58fc44ff7727deea782d175c33602737b7feadb6 \ - --hash=sha256:aeb29c84bb53a84b1a81c6c09d24cf33bb8432cc5c39979021cc0f98c1292a1a \ - --hash=sha256:aede4df4eeb926c8fa70de46c340a1bc2c6079e1c40ccf7b0eae1313ffd33519 \ - --hash=sha256:b744c33b6f14ca26b7544e8d8aadff6b765a80ad6164fb1a430bbadd593dfb1a \ - --hash=sha256:b7a00a9ed8d6e725b55ef98b1b35c88013245f35f68b1b12c5cd4100dddac333 \ - --hash=sha256:bb96fa6b56bb536c42d6a4a87dfca570ff8e52de2d63cabebfd6fb67049c34b6 \ - --hash=sha256:bbcf1a76cf6f6dacf2c7f4d2ebd411438c275faa1dc0c68e46eb84eebd05dd7d \ - --hash=sha256:bca5f24726e2919de94f047739d0a4fc01372801a3672708260546aa2601bf57 \ - --hash=sha256:bf2e1a9162c1e441bf805a1fd166e249d574ca04e03b34f97e2928769e91ab5c \ - --hash=sha256:c4eb3b82ca349cf6fadcdc7abcc8b3a50ab74a62e9113ab7a8ebc268aad35bb9 \ - --hash=sha256:c6cc15d58053c76eacac5fa9152d7d84b8d67b3fde92709195cb984cfb3475ea \ - --hash=sha256:c6cd05ea06daca6ad6a4ca3ba7fe7dc5b5de063ff4daec6170ec0f9979f6c332 \ - --hash=sha256:c844fd628851c0bc309f3c801b3a3d58ce430b2ce5b359cd918a5a76d0b20cb5 \ - --hash=sha256:c9cb1565a7ad52e096a6988e2ee0397f72fe056dadf75d17fa6b5aebaea05622 \ - --hash=sha256:cab9401de3ea52b4b4c6971db5fb5c999bd4260898af972bf23de1c6b5dd9d71 \ - --hash=sha256:cd468460eefef601ece4428d3cf4562459157c0f6523db89365202c31b6daebb \ - --hash=sha256:d1e6a862b76f34395a985b3cd39a0d949ca80a70b6ebdea37d3ab39ceea6698a \ - --hash=sha256:d1f9282c5f2b5e241034a009779e7b2a1aa045f667ff521e7948ea9b56e0c5ff \ - --hash=sha256:d265f09a75a79a788237d7f9054f929ced2e69eb0bb79de3798c468d8a90f945 \ - --hash=sha256:db3fc6120bce9f446d13b1b834ea5b15341ca9ff3f335e4a951a6ead31105480 \ - --hash=sha256:dbf3a08a06b3f433013c143ebd72c15cac33d2914b8ea4bea7ac2c23578815d6 \ - --hash=sha256:de04b491d0e5007ee1b63a309956eaed959a49f5bb4e84b26c8f5d49de140fa9 \ - --hash=sha256:e4b09863aae0dc965c3ef36500d891a3ff495a2ea9ae9171e4519963c12ceefd \ - --hash=sha256:e595432ac259af2d4630008bf638873d69346372d38255774c0e286951e8b79f \ - --hash=sha256:e75b89ac3bd27d2d043b234aa7b734c38ba1b0e43f07787130a0ecac1e12228a \ - --hash=sha256:ea9eb976ffdd79d0e893869cfe179a8f60f152d42cb64622fca418cd9b18dc2a \ - --hash=sha256:eafb3e874816ebe2a92f5e155f17260034c8c341dad1df25672fb710627c6949 \ - --hash=sha256:ee3c36df21b5714d49fc4580247947aa64bcbe2939d1b77b4c8dcb8f6c9faecc \ - --hash=sha256:f352b62b45dff37b55ddd7b9c0c8672c4dd2eb9c0f9c11d395075a84e2c40f75 \ - --hash=sha256:fabb87dd8850ef0f7fe2b366d44b77d7e6fa2ea87861ab3844da99291e81e60f \ - --hash=sha256:fe11310ae1e4cd560035598c3f29d86cef39a83d244c7466f95c27ae04850f10 \ - --hash=sha256:fe7ba4a51f33ab275515f66b0a236bcde4fb5561498fe8f898d4e549b2e4509f -aiosignal==1.3.1 ; python_full_version >= "3.8.0" and python_version < "4.0" \ - --hash=sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc \ - --hash=sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17 -arpeggio==2.0.0 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ - --hash=sha256:448e332deb0e9ccd04046f1c6c14529d197f41bc2fdb3931e43fc209042fbdd3 \ - --hash=sha256:d6b03839019bb8a68785f9292ee6a36b1954eb84b925b84a6b8a5e1e26d3ed3d -async-exit-stack==1.0.1 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ - --hash=sha256:24de1ad6d0ff27be97c89d6709fa49bf20db179eaf1f4d2e6e9b4409b80e747d \ - --hash=sha256:9b43b17683b3438f428ef3bbec20689f5abbb052aa4b564c643397330adfaa99 -async-timeout==4.0.2 ; python_full_version >= "3.8.0" and python_version < "4.0" \ - --hash=sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15 \ - --hash=sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c -attrs==23.1.0 ; python_full_version >= "3.8.0" and python_version < "4.0" \ - --hash=sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04 \ - --hash=sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015 -certifi==2020.12.5 ; python_full_version >= "3.8.0" and python_version < "4.0" \ - --hash=sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c \ - --hash=sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830 -charset-normalizer==3.1.0 ; python_full_version >= "3.8.0" and python_version < "4.0" \ - --hash=sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6 \ - --hash=sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1 \ - --hash=sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e \ - --hash=sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373 \ - --hash=sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62 \ - --hash=sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230 \ - --hash=sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be \ - --hash=sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c \ - --hash=sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0 \ - --hash=sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448 \ - --hash=sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f \ - --hash=sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649 \ - --hash=sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d \ - --hash=sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0 \ - --hash=sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706 \ - --hash=sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a \ - --hash=sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59 \ - --hash=sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23 \ - --hash=sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5 \ - --hash=sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb \ - --hash=sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e \ - --hash=sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e \ - --hash=sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c \ - --hash=sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28 \ - --hash=sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d \ - --hash=sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41 \ - --hash=sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974 \ - --hash=sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce \ - --hash=sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f \ - --hash=sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1 \ - --hash=sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d \ - --hash=sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8 \ - --hash=sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017 \ - --hash=sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31 \ - --hash=sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7 \ - --hash=sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8 \ - --hash=sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e \ - --hash=sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14 \ - --hash=sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd \ - --hash=sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d \ - --hash=sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795 \ - --hash=sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b \ - --hash=sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b \ - --hash=sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b \ - --hash=sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203 \ - --hash=sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f \ - --hash=sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19 \ - --hash=sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1 \ - --hash=sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a \ - --hash=sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac \ - --hash=sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9 \ - --hash=sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0 \ - --hash=sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137 \ - --hash=sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f \ - --hash=sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6 \ - --hash=sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5 \ - --hash=sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909 \ - --hash=sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f \ - --hash=sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0 \ - --hash=sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324 \ - --hash=sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755 \ - --hash=sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb \ - --hash=sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854 \ - --hash=sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c \ - --hash=sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60 \ - --hash=sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84 \ - --hash=sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0 \ - --hash=sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b \ - --hash=sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1 \ - --hash=sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531 \ - --hash=sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1 \ - --hash=sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11 \ - --hash=sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326 \ - --hash=sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df \ - --hash=sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab -click==8.1.3 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ - --hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \ - --hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48 -colorama==0.4.6 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" and platform_system == "Windows" \ - --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ - --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6 -dnspython==2.3.0 ; python_full_version >= "3.8.0" and python_version < "4.0" \ - --hash=sha256:224e32b03eb46be70e12ef6d64e0be123a64e621ab4c0822ff6d450d52a540b9 \ - --hash=sha256:89141536394f909066cabd112e3e1a37e4e654db00a25308b0f130bc3152eb46 -frozenlist==1.3.3 ; python_full_version >= "3.8.0" and python_version < "4.0" \ - --hash=sha256:008a054b75d77c995ea26629ab3a0c0d7281341f2fa7e1e85fa6153ae29ae99c \ - --hash=sha256:02c9ac843e3390826a265e331105efeab489ffaf4dd86384595ee8ce6d35ae7f \ - --hash=sha256:034a5c08d36649591be1cbb10e09da9f531034acfe29275fc5454a3b101ce41a \ - --hash=sha256:05cdb16d09a0832eedf770cb7bd1fe57d8cf4eaf5aced29c4e41e3f20b30a784 \ - --hash=sha256:0693c609e9742c66ba4870bcee1ad5ff35462d5ffec18710b4ac89337ff16e27 \ - --hash=sha256:0771aed7f596c7d73444c847a1c16288937ef988dc04fb9f7be4b2aa91db609d \ - --hash=sha256:0af2e7c87d35b38732e810befb9d797a99279cbb85374d42ea61c1e9d23094b3 \ - --hash=sha256:14143ae966a6229350021384870458e4777d1eae4c28d1a7aa47f24d030e6678 \ - --hash=sha256:180c00c66bde6146a860cbb81b54ee0df350d2daf13ca85b275123bbf85de18a \ - --hash=sha256:1841e200fdafc3d51f974d9d377c079a0694a8f06de2e67b48150328d66d5483 \ - --hash=sha256:23d16d9f477bb55b6154654e0e74557040575d9d19fe78a161bd33d7d76808e8 \ - --hash=sha256:2b07ae0c1edaa0a36339ec6cce700f51b14a3fc6545fdd32930d2c83917332cf \ - --hash=sha256:2c926450857408e42f0bbc295e84395722ce74bae69a3b2aa2a65fe22cb14b99 \ - --hash=sha256:2e24900aa13212e75e5b366cb9065e78bbf3893d4baab6052d1aca10d46d944c \ - --hash=sha256:303e04d422e9b911a09ad499b0368dc551e8c3cd15293c99160c7f1f07b59a48 \ - --hash=sha256:352bd4c8c72d508778cf05ab491f6ef36149f4d0cb3c56b1b4302852255d05d5 \ - --hash=sha256:3843f84a6c465a36559161e6c59dce2f2ac10943040c2fd021cfb70d58c4ad56 \ - --hash=sha256:394c9c242113bfb4b9aa36e2b80a05ffa163a30691c7b5a29eba82e937895d5e \ - --hash=sha256:3bbdf44855ed8f0fbcd102ef05ec3012d6a4fd7c7562403f76ce6a52aeffb2b1 \ - --hash=sha256:40de71985e9042ca00b7953c4f41eabc3dc514a2d1ff534027f091bc74416401 \ - --hash=sha256:41fe21dc74ad3a779c3d73a2786bdf622ea81234bdd4faf90b8b03cad0c2c0b4 \ - --hash=sha256:47df36a9fe24054b950bbc2db630d508cca3aa27ed0566c0baf661225e52c18e \ - --hash=sha256:4ea42116ceb6bb16dbb7d526e242cb6747b08b7710d9782aa3d6732bd8d27649 \ - --hash=sha256:58bcc55721e8a90b88332d6cd441261ebb22342e238296bb330968952fbb3a6a \ - --hash=sha256:5c11e43016b9024240212d2a65043b70ed8dfd3b52678a1271972702d990ac6d \ - --hash=sha256:5cf820485f1b4c91e0417ea0afd41ce5cf5965011b3c22c400f6d144296ccbc0 \ - --hash=sha256:5d8860749e813a6f65bad8285a0520607c9500caa23fea6ee407e63debcdbef6 \ - --hash=sha256:6327eb8e419f7d9c38f333cde41b9ae348bec26d840927332f17e887a8dcb70d \ - --hash=sha256:65a5e4d3aa679610ac6e3569e865425b23b372277f89b5ef06cf2cdaf1ebf22b \ - --hash=sha256:66080ec69883597e4d026f2f71a231a1ee9887835902dbe6b6467d5a89216cf6 \ - --hash=sha256:783263a4eaad7c49983fe4b2e7b53fa9770c136c270d2d4bbb6d2192bf4d9caf \ - --hash=sha256:7f44e24fa70f6fbc74aeec3e971f60a14dde85da364aa87f15d1be94ae75aeef \ - --hash=sha256:7fdfc24dcfce5b48109867c13b4cb15e4660e7bd7661741a391f821f23dfdca7 \ - --hash=sha256:810860bb4bdce7557bc0febb84bbd88198b9dbc2022d8eebe5b3590b2ad6c842 \ - --hash=sha256:841ea19b43d438a80b4de62ac6ab21cfe6827bb8a9dc62b896acc88eaf9cecba \ - --hash=sha256:84610c1502b2461255b4c9b7d5e9c48052601a8957cd0aea6ec7a7a1e1fb9420 \ - --hash=sha256:899c5e1928eec13fd6f6d8dc51be23f0d09c5281e40d9cf4273d188d9feeaf9b \ - --hash=sha256:8bae29d60768bfa8fb92244b74502b18fae55a80eac13c88eb0b496d4268fd2d \ - --hash=sha256:8df3de3a9ab8325f94f646609a66cbeeede263910c5c0de0101079ad541af332 \ - --hash=sha256:8fa3c6e3305aa1146b59a09b32b2e04074945ffcfb2f0931836d103a2c38f936 \ - --hash=sha256:924620eef691990dfb56dc4709f280f40baee568c794b5c1885800c3ecc69816 \ - --hash=sha256:9309869032abb23d196cb4e4db574232abe8b8be1339026f489eeb34a4acfd91 \ - --hash=sha256:9545a33965d0d377b0bc823dcabf26980e77f1b6a7caa368a365a9497fb09420 \ - --hash=sha256:9ac5995f2b408017b0be26d4a1d7c61bce106ff3d9e3324374d66b5964325448 \ - --hash=sha256:9bbbcedd75acdfecf2159663b87f1bb5cfc80e7cd99f7ddd9d66eb98b14a8411 \ - --hash=sha256:a4ae8135b11652b08a8baf07631d3ebfe65a4c87909dbef5fa0cdde440444ee4 \ - --hash=sha256:a6394d7dadd3cfe3f4b3b186e54d5d8504d44f2d58dcc89d693698e8b7132b32 \ - --hash=sha256:a97b4fe50b5890d36300820abd305694cb865ddb7885049587a5678215782a6b \ - --hash=sha256:ae4dc05c465a08a866b7a1baf360747078b362e6a6dbeb0c57f234db0ef88ae0 \ - --hash=sha256:b1c63e8d377d039ac769cd0926558bb7068a1f7abb0f003e3717ee003ad85530 \ - --hash=sha256:b1e2c1185858d7e10ff045c496bbf90ae752c28b365fef2c09cf0fa309291669 \ - --hash=sha256:b4395e2f8d83fbe0c627b2b696acce67868793d7d9750e90e39592b3626691b7 \ - --hash=sha256:b756072364347cb6aa5b60f9bc18e94b2f79632de3b0190253ad770c5df17db1 \ - --hash=sha256:ba64dc2b3b7b158c6660d49cdb1d872d1d0bf4e42043ad8d5006099479a194e5 \ - --hash=sha256:bed331fe18f58d844d39ceb398b77d6ac0b010d571cba8267c2e7165806b00ce \ - --hash=sha256:c188512b43542b1e91cadc3c6c915a82a5eb95929134faf7fd109f14f9892ce4 \ - --hash=sha256:c21b9aa40e08e4f63a2f92ff3748e6b6c84d717d033c7b3438dd3123ee18f70e \ - --hash=sha256:ca713d4af15bae6e5d79b15c10c8522859a9a89d3b361a50b817c98c2fb402a2 \ - --hash=sha256:cd4210baef299717db0a600d7a3cac81d46ef0e007f88c9335db79f8979c0d3d \ - --hash=sha256:cfe33efc9cb900a4c46f91a5ceba26d6df370ffddd9ca386eb1d4f0ad97b9ea9 \ - --hash=sha256:d5cd3ab21acbdb414bb6c31958d7b06b85eeb40f66463c264a9b343a4e238642 \ - --hash=sha256:dfbac4c2dfcc082fcf8d942d1e49b6aa0766c19d3358bd86e2000bf0fa4a9cf0 \ - --hash=sha256:e235688f42b36be2b6b06fc37ac2126a73b75fb8d6bc66dd632aa35286238703 \ - --hash=sha256:eb82dbba47a8318e75f679690190c10a5e1f447fbf9df41cbc4c3afd726d88cb \ - --hash=sha256:ebb86518203e12e96af765ee89034a1dbb0c3c65052d1b0c19bbbd6af8a145e1 \ - --hash=sha256:ee78feb9d293c323b59a6f2dd441b63339a30edf35abcb51187d2fc26e696d13 \ - --hash=sha256:eedab4c310c0299961ac285591acd53dc6723a1ebd90a57207c71f6e0c2153ab \ - --hash=sha256:efa568b885bca461f7c7b9e032655c0c143d305bf01c30caf6db2854a4532b38 \ - --hash=sha256:efce6ae830831ab6a22b9b4091d411698145cb9b8fc869e1397ccf4b4b6455cb \ - --hash=sha256:f163d2fd041c630fed01bc48d28c3ed4a3b003c00acd396900e11ee5316b56bb \ - --hash=sha256:f20380df709d91525e4bee04746ba612a4df0972c1b8f8e1e8af997e678c7b81 \ - --hash=sha256:f30f1928162e189091cf4d9da2eac617bfe78ef907a761614ff577ef4edfb3c8 \ - --hash=sha256:f470c92737afa7d4c3aacc001e335062d582053d4dbe73cda126f2d7031068dd \ - --hash=sha256:ff8bf625fe85e119553b5383ba0fb6aa3d0ec2ae980295aaefa552374926b3f4 -idna==3.4 ; python_full_version >= "3.8.0" and python_version < "4.0" \ - --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4 \ - --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 -jsonrpc-base==1.1.0 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ - --hash=sha256:7f374c57bfa1cb16d1f340d270bc0d9f1f5608fb1ac6c9ea15768c0e6ece48b7 -multidict==6.0.4 ; python_full_version >= "3.8.0" and python_version < "4.0" \ - --hash=sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9 \ - --hash=sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8 \ - --hash=sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03 \ - --hash=sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710 \ - --hash=sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161 \ - --hash=sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664 \ - --hash=sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569 \ - --hash=sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067 \ - --hash=sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313 \ - --hash=sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706 \ - --hash=sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2 \ - --hash=sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636 \ - --hash=sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49 \ - --hash=sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93 \ - --hash=sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603 \ - --hash=sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0 \ - --hash=sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60 \ - --hash=sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4 \ - --hash=sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e \ - --hash=sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1 \ - --hash=sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60 \ - --hash=sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951 \ - --hash=sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc \ - --hash=sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe \ - --hash=sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95 \ - --hash=sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d \ - --hash=sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8 \ - --hash=sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed \ - --hash=sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2 \ - --hash=sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775 \ - --hash=sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87 \ - --hash=sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c \ - --hash=sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2 \ - --hash=sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98 \ - --hash=sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3 \ - --hash=sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe \ - --hash=sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78 \ - --hash=sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660 \ - --hash=sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176 \ - --hash=sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e \ - --hash=sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988 \ - --hash=sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c \ - --hash=sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c \ - --hash=sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0 \ - --hash=sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449 \ - --hash=sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f \ - --hash=sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde \ - --hash=sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5 \ - --hash=sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d \ - --hash=sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac \ - --hash=sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a \ - --hash=sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9 \ - --hash=sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca \ - --hash=sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11 \ - --hash=sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35 \ - --hash=sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063 \ - --hash=sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b \ - --hash=sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982 \ - --hash=sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258 \ - --hash=sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1 \ - --hash=sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52 \ - --hash=sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480 \ - --hash=sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7 \ - --hash=sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461 \ - --hash=sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d \ - --hash=sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc \ - --hash=sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779 \ - --hash=sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a \ - --hash=sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547 \ - --hash=sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0 \ - --hash=sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171 \ - --hash=sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf \ - --hash=sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d \ - --hash=sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba -prettytable==3.7.0 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ - --hash=sha256:ef8334ee40b7ec721651fc4d37ecc7bb2ef55fde5098d994438f0dfdaa385c0c \ - --hash=sha256:f4aaf2ed6e6062a82fd2e6e5289bbbe705ec2788fe401a3a1f62a1cea55526d2 -python-dateutil==2.8.2 ; python_full_version >= "3.8.0" and python_version < "4.0" \ - --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ - --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 -semantic-version==2.10.0 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ - --hash=sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c \ - --hash=sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177 -setuptools==67.8.0 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ - --hash=sha256:5df61bf30bb10c6f756eb19e7c9f3b473051f48db77fddbe06ff2ca307df9a6f \ - --hash=sha256:62642358adc77ffa87233bc4d2354c4b2682d214048f500964dbe760ccedf102 -six==1.16.0 ; python_full_version >= "3.8.0" and python_version < "4.0" \ - --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ - --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 -srvresolver==0.3.5 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ - --hash=sha256:0a8973d340486830ddd057895769cb672c7cd902ae61c21339875ed2a9d7ec0f \ - --hash=sha256:0cbb756d929b267e03dac98ea6d9a188d5ce6c8d30ca5553ed6737fb5ddd1c09 -textx==3.1.1 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ - --hash=sha256:1f2bd936309f5f9092567e3f5a0c513d8292d5852c63a72096fe5a57018d0f50 \ - --hash=sha256:e2fb7d090ea4a71f8bf2f0303770275c42a5f73854b4e7f6adfcdbe368fab0fa -wcwidth==0.2.6 ; python_full_version >= "3.8.0" and python_full_version < "4.0.0" \ - --hash=sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e \ - --hash=sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0 -ya-aioclient==0.6.4 ; python_full_version >= "3.8.0" and python_version < "4.0" \ - --hash=sha256:1d36c2544c2d7629a00a2b1f18b4535e836586cae99bfbf5714ed5ca3f9caa0c \ - --hash=sha256:e5e5926e3a7d834082b05aeb986d8fb2f7fa46a1bd1bca2f3f1c872e1ba479ae -yarl==1.9.2 ; python_full_version >= "3.8.0" and python_version < "4.0" \ - --hash=sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571 \ - --hash=sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3 \ - --hash=sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3 \ - --hash=sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c \ - --hash=sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7 \ - --hash=sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04 \ - --hash=sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191 \ - --hash=sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea \ - --hash=sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4 \ - --hash=sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4 \ - --hash=sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095 \ - --hash=sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e \ - --hash=sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74 \ - --hash=sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef \ - --hash=sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33 \ - --hash=sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde \ - --hash=sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45 \ - --hash=sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf \ - --hash=sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b \ - --hash=sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac \ - --hash=sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0 \ - --hash=sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528 \ - --hash=sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716 \ - --hash=sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb \ - --hash=sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18 \ - --hash=sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72 \ - --hash=sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6 \ - --hash=sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582 \ - --hash=sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5 \ - --hash=sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368 \ - --hash=sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc \ - --hash=sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9 \ - --hash=sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be \ - --hash=sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a \ - --hash=sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80 \ - --hash=sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8 \ - --hash=sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6 \ - --hash=sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417 \ - --hash=sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574 \ - --hash=sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59 \ - --hash=sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608 \ - --hash=sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82 \ - --hash=sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1 \ - --hash=sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3 \ - --hash=sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d \ - --hash=sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8 \ - --hash=sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc \ - --hash=sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac \ - --hash=sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8 \ - --hash=sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955 \ - --hash=sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0 \ - --hash=sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367 \ - --hash=sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb \ - --hash=sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a \ - --hash=sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623 \ - --hash=sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2 \ - --hash=sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6 \ - --hash=sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7 \ - --hash=sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4 \ - --hash=sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051 \ - --hash=sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938 \ - --hash=sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8 \ - --hash=sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9 \ - --hash=sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3 \ - --hash=sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5 \ - --hash=sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9 \ - --hash=sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333 \ - --hash=sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185 \ - --hash=sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3 \ - --hash=sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560 \ - --hash=sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b \ - --hash=sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7 \ - --hash=sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78 \ - --hash=sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7 diff --git a/golem/managers/demand/auto.py b/golem/managers/demand/auto.py index c04ceb0b..aa4f0ca2 100644 --- a/golem/managers/demand/auto.py +++ b/golem/managers/demand/auto.py @@ -27,7 +27,7 @@ def __init__( super().__init__(*args, **kwargs) - @trace_span() + @trace_span(show_results=True) async def get_initial_proposal(self) -> Proposal: return await self.get_scored_proposal() diff --git a/golem/utils/buffer/base.py b/golem/utils/buffer.py similarity index 89% rename from golem/utils/buffer/base.py rename to golem/utils/buffer.py index a0e320bd..291677ae 100644 --- a/golem/utils/buffer/base.py +++ b/golem/utils/buffer.py @@ -17,6 +17,14 @@ class Buffer(ABC, Generic[TItem]): async def get_item(self) -> TItem: ... + @abstractmethod + async def start(self, *, fill=False) -> None: + ... + + @abstractmethod + async def stop(self) -> None: + ... + async def default_update_callback( items: MutableSequence[TItem], items_to_process: Sequence[TItem] @@ -24,7 +32,7 @@ async def default_update_callback( items.extend(items_to_process) -class SequenceFilledBuffer(Buffer[TItem], Generic[TItem]): +class ConcurrentlyFilledBuffer(Buffer[TItem], Generic[TItem]): def __init__( self, fill_callback: Callable[[], Awaitable[TItem]], @@ -68,8 +76,23 @@ async def stop(self) -> None: if self._background_loop_task is not None: self._background_loop_task.cancel() + + try: + await self._background_loop_task + except asyncio.CancelledError: + pass + self._background_loop_task = None + tasks_to_cancel = self._items_requested_tasks[:] + for task in tasks_to_cancel: + task.cancel() + + try: + await task + except asyncio.CancelledError: + pass + def is_started(self) -> bool: return self._background_loop_task is not None and not self._background_loop_task.done() @@ -111,7 +134,7 @@ def _handle_item_requests(self) -> None: self._items_requests_pending_event.set() - def on_completetion(task): + def on_completion(task): self._items_requested_tasks.remove(task) if not self._items_requested_tasks: @@ -121,17 +144,13 @@ def on_completetion(task): for _ in range(items_to_request): task = create_task_with_logging(self._fill()) - task.add_done_callback(on_completetion) + task.add_done_callback(on_completion) self._items_requested_tasks.append(task) logger.debug("Requested %d items", items_to_request) async def _background_loop(self) -> None: while True: - # check if any items are ready to process - # check if all requested items are ready to process or timeout axceeded - # run update_callback - logger.debug("Waiting for any item requests...") await self._items_requests_pending_event.wait() diff --git a/golem/utils/buffer/__init__.py b/golem/utils/buffer/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/unit/utils/test_buffer.py b/tests/unit/utils/test_buffer.py index 63d32785..69fd820e 100644 --- a/tests/unit/utils/test_buffer.py +++ b/tests/unit/utils/test_buffer.py @@ -1,10 +1,9 @@ import asyncio -import logging from datetime import timedelta import pytest -from golem.utils.buffer.base import SequenceFilledBuffer +from golem.utils.buffer import ConcurrentlyFilledBuffer @pytest.fixture @@ -14,7 +13,7 @@ def fill_callback(mocker): @pytest.fixture def buffer_class(): - return SequenceFilledBuffer + return ConcurrentlyFilledBuffer @pytest.fixture @@ -133,13 +132,12 @@ async def test_buffer_get_item_will_trigger_fill_on_below_min_size(create_buffer await buffer.stop() -async def _test_buffer_fill_can_add_requests_while_other_requests_are_running( +async def test_buffer_fill_can_add_requests_while_other_requests_are_running( buffer_class, mocker, caplog ): - caplog.set_level(logging.DEBUG, logger="golem.utils.buffer.base") queue: asyncio.Queue[int] = asyncio.Queue() - for i in range(6): + for i in range(3): await queue.put(i) async def fill_callback(): @@ -162,16 +160,31 @@ async def fill_callback(): assert mocked_fill_callback.await_count == 6 - done, _ = await asyncio.wait( - [asyncio.create_task(buffer.get_item()) for _ in range(3)], timeout=0.1 - ) + item = await asyncio.wait_for(buffer.get_item(), 0.05) - assert [d.result() for d in done] == list(range(3)) + assert item == 0 - assert mocked_fill_callback.await_count == 6 - assert queue.qsize() == 0 + assert mocked_fill_callback.await_count == 7 - with pytest.raises(asyncio.TimeoutError): - await asyncio.wait_for(buffer.get_item(), 0.1) + await buffer.stop() + + assert len(asyncio.all_tasks()) == 1, "Other async tasks are running!" + + +# TODO: Expiration needs to be implemented +async def _test_buffer_can_expire_elements(create_buffer, fill_callback): + buffer = create_buffer(min_size=3, max_size=6, remove_after=timedelta(seconds=0.5)) + + await buffer.start(fill=True) + + await asyncio.sleep(0.1) + + assert fill_callback.await_count == 6 + + await asyncio.sleep(0.5) + + assert fill_callback.await_count == 12 await buffer.stop() + + assert len(asyncio.all_tasks()) == 1, "Other async tasks are running!" From d91c45b92d46b189725bf0680f25cfea2394922f Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 25 Jul 2023 09:13:28 +0200 Subject: [PATCH 102/123] Revert "Debug frozen CI test" This reverts commit 8947ecde04615023b04271fbced456c37c77d3b3. --- tests/integration/test_1.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/integration/test_1.py b/tests/integration/test_1.py index 5129a3cb..dc6f92ad 100644 --- a/tests/integration/test_1.py +++ b/tests/integration/test_1.py @@ -65,14 +65,9 @@ async def test_demand(golem): allocation = await golem.create_allocation(1) demand = await golem.create_demand(PAYLOAD, allocations=[allocation]) - print("test_demand: getting proposals") - async for proposal in demand.initial_proposals(): - print("test_demand: got proposal") break - print("test_demand: getting proposals done") - await demand.get_data() await demand.unsubscribe() From ff266e14dfb80e32a735d902c6b182858fa36e84 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 25 Jul 2023 10:21:51 +0200 Subject: [PATCH 103/123] Add demand refresh; add buffer to demand manager --- golem/managers/agreement/scored_aot.py | 2 +- golem/managers/demand/auto.py | 69 ++++++++++++++++++++++---- golem/managers/mixins.py | 8 +-- golem/utils/logging.py | 2 +- 4 files changed, 65 insertions(+), 16 deletions(-) diff --git a/golem/managers/agreement/scored_aot.py b/golem/managers/agreement/scored_aot.py index 9d9157a9..a9e88a26 100644 --- a/golem/managers/agreement/scored_aot.py +++ b/golem/managers/agreement/scored_aot.py @@ -51,7 +51,7 @@ async def _background_loop(self) -> None: while True: proposal = await self._get_draft_proposal() - await self.manage_scoring(proposal) + await self.manage_scoring([proposal]) @trace_span(show_arguments=True) async def _terminate_agreement(self, event: AgreementReleased) -> None: diff --git a/golem/managers/demand/auto.py b/golem/managers/demand/auto.py index aa4f0ca2..e0b34aa9 100644 --- a/golem/managers/demand/auto.py +++ b/golem/managers/demand/auto.py @@ -1,12 +1,16 @@ +import asyncio import logging -from typing import Awaitable, Callable +from datetime import datetime, timedelta +from typing import Awaitable, Callable, List, MutableSequence, Sequence, Tuple from golem.managers.base import DemandManager from golem.managers.mixins import BackgroundLoopMixin, WeightProposalScoringPluginsMixin from golem.node import GolemNode from golem.payload import Payload -from golem.resources import Allocation, Proposal +from golem.resources import Allocation, Demand, Proposal from golem.resources.demand.demand_builder import DemandBuilder +from golem.utils.asyncio import create_task_with_logging +from golem.utils.buffer import ConcurrentlyFilledBuffer from golem.utils.logging import trace_span logger = logging.getLogger(__name__) @@ -25,27 +29,68 @@ def __init__( self._get_allocation = get_allocation self._payload = payload + self._initial_proposals: asyncio.Queue = asyncio.Queue() + self._buffer = ConcurrentlyFilledBuffer( + fill_callback=self._initial_proposals.get, + min_size=1, + max_size=1000, + update_callback=self._update_buffer_callback, + update_interval=timedelta(seconds=1), + ) + + self._demands: List[Tuple[Demand, asyncio.Task]] = [] super().__init__(*args, **kwargs) @trace_span(show_results=True) async def get_initial_proposal(self) -> Proposal: - return await self.get_scored_proposal() + return await self._buffer.get_item() @trace_span() async def _background_loop(self) -> None: + await self._buffer.start() + await self._create_and_subscribe_demand() + try: + while True: + if datetime.utcfromtimestamp( + self._demands[-1][0].data.properties["golem.srv.comp.expiration"] / 1000 + ) < datetime.utcnow() + timedelta(minutes=29, seconds=50): + self._stop_consuming_initial_proposals() + await self._create_and_subscribe_demand() + await asyncio.sleep(0.5) + finally: + self._stop_consuming_initial_proposals() + await self._buffer.stop() + await self._unsubscribe_demands() + + async def _update_buffer_callback( + self, items: MutableSequence[Proposal], items_to_process: Sequence[Proposal] + ) -> None: + items.extend(items_to_process) + items = [proposal for _, proposal in await self.do_scoring(items)] + + @trace_span() + async def _create_and_subscribe_demand(self): allocation = await self._get_allocation() demand_builder = await self._prepare_demand_builder(allocation) - demand = await demand_builder.create_demand(self._golem) demand.start_collecting_events() + await demand.get_data() + self._demands.append( + (demand, create_task_with_logging(self._consume_initial_proposals(demand))) + ) - logger.debug(f"`{demand}` posted on market with `{demand_builder}`") - + @trace_span() + async def _consume_initial_proposals(self, demand: Demand): try: - async for initial_proposal in demand.initial_proposals(): - await self.manage_scoring(initial_proposal) - finally: - await demand.unsubscribe() + async for initial in demand.initial_proposals(): + logger.debug(f"New initial proposal {initial}") + self._initial_proposals.put_nowait(initial) + except asyncio.CancelledError: + ... + + @trace_span() + def _stop_consuming_initial_proposals(self) -> List[bool]: + return [d[1].cancel() for d in self._demands] @trace_span() async def _prepare_demand_builder(self, allocation: Allocation) -> DemandBuilder: @@ -59,3 +104,7 @@ async def _prepare_demand_builder(self, allocation: Allocation) -> DemandBuilder await demand_builder.add(self._payload) return demand_builder + + @trace_span() + async def _unsubscribe_demands(self): + await asyncio.gather(*[demand.unsubscribe() for demand, _ in self._demands]) diff --git a/golem/managers/mixins.py b/golem/managers/mixins.py index 602877b5..7e8b5fe1 100644 --- a/golem/managers/mixins.py +++ b/golem/managers/mixins.py @@ -71,12 +71,12 @@ def __init__( super().__init__(*args, **kwargs) @trace_span(show_arguments=True) - async def manage_scoring(self, proposal: Proposal) -> None: + async def manage_scoring(self, proposals: List[Proposal]) -> None: async with self._scored_proposals_condition: all_proposals = list(sp[1] for sp in self._scored_proposals) - all_proposals.append(proposal) + all_proposals.extend(proposals) - self._scored_proposals = await self._do_scoring(all_proposals) + self._scored_proposals = await self.do_scoring(all_proposals) self._scored_proposals_condition.notify_all() @@ -91,7 +91,7 @@ async def get_scored_proposal(self): return proposal - async def _do_scoring(self, proposals: Sequence[Proposal]): + async def do_scoring(self, proposals: Sequence[Proposal]) -> List[Tuple[float, Proposal]]: proposals_data = await self._get_proposals_data_from_proposals(proposals) proposal_scores = await self._run_plugins(proposals_data) diff --git a/golem/utils/logging.py b/golem/utils/logging.py index fff7a07c..bbddbc75 100644 --- a/golem/utils/logging.py +++ b/golem/utils/logging.py @@ -57,7 +57,7 @@ "level": "INFO", }, "golem.managers.activity": { - "level": "DEBUG", + "level": "INFO", }, "golem.managers.work": { "level": "INFO", From 31958a8596d07c087553ec6a20873d04c92be27d Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 25 Jul 2023 12:09:19 +0200 Subject: [PATCH 104/123] Improve Demand manager refresh --- golem/managers/activity/pool.py | 8 ++++---- golem/managers/demand/auto.py | 34 +++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/golem/managers/activity/pool.py b/golem/managers/activity/pool.py index 68906ec6..a1b9e115 100644 --- a/golem/managers/activity/pool.py +++ b/golem/managers/activity/pool.py @@ -62,23 +62,23 @@ async def _background_loop(self): @trace_span() async def _release_activity_and_pop_from_pool(self): activity = await self._pool.get() + self._pool.task_done() await self._release_activity(activity) - logger.info(f"Activity `{activity}` removed from the pool") @trace_span() async def _prepare_activity_and_put_in_pool(self): agreement = await self._get_agreement() activity = await self._prepare_activity(agreement) await self._pool.put(activity) - logger.info(f"Activity `{activity}` added to the pool") @asynccontextmanager async def _get_activity_from_pool(self): activity = await self._pool.get() - logger.info(f"Activity `{activity}` taken from the pool") + self._pool.task_done() + logger.debug(f"Activity `{activity}` taken from the pool") yield activity self._pool.put_nowait(activity) - logger.info(f"Activity `{activity}` back in the pool") + logger.debug(f"Activity `{activity}` back in the pool") @trace_span(show_arguments=True, show_results=True) async def do_work(self, work: Work) -> WorkResult: diff --git a/golem/managers/demand/auto.py b/golem/managers/demand/auto.py index e0b34aa9..18acc63f 100644 --- a/golem/managers/demand/auto.py +++ b/golem/managers/demand/auto.py @@ -29,9 +29,9 @@ def __init__( self._get_allocation = get_allocation self._payload = payload - self._initial_proposals: asyncio.Queue = asyncio.Queue() + self._initial_proposals: asyncio.Queue[Proposal] = asyncio.Queue() self._buffer = ConcurrentlyFilledBuffer( - fill_callback=self._initial_proposals.get, + fill_callback=self._get_initial_proposal, min_size=1, max_size=1000, update_callback=self._update_buffer_callback, @@ -45,28 +45,42 @@ def __init__( async def get_initial_proposal(self) -> Proposal: return await self._buffer.get_item() + async def _get_initial_proposal(self) -> Proposal: + proposal = await self._initial_proposals.get() + self._initial_proposals.task_done() + return proposal + @trace_span() async def _background_loop(self) -> None: await self._buffer.start() await self._create_and_subscribe_demand() try: while True: - if datetime.utcfromtimestamp( - self._demands[-1][0].data.properties["golem.srv.comp.expiration"] / 1000 - ) < datetime.utcnow() + timedelta(minutes=29, seconds=50): - self._stop_consuming_initial_proposals() - await self._create_and_subscribe_demand() - await asyncio.sleep(0.5) + await self._wait_for_demand_to_expire() + self._stop_consuming_initial_proposals() + await self._create_and_subscribe_demand() finally: self._stop_consuming_initial_proposals() await self._buffer.stop() await self._unsubscribe_demands() + async def _wait_for_demand_to_expire(self): + remaining: timedelta = ( + datetime.utcfromtimestamp( + self._demands[-1][0].data.properties["golem.srv.comp.expiration"] / 1000 + ) + - datetime.utcnow() + ) + await asyncio.sleep(remaining.seconds) + async def _update_buffer_callback( self, items: MutableSequence[Proposal], items_to_process: Sequence[Proposal] ) -> None: - items.extend(items_to_process) - items = [proposal for _, proposal in await self.do_scoring(items)] + scored_proposals = [ + proposal for _, proposal in await self.do_scoring(items + items_to_process) + ] + items.clear() + items.extend(scored_proposals) @trace_span() async def _create_and_subscribe_demand(self): From 80135bdcfdbae4dbdc67d2e577e84b94369b0c39 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 25 Jul 2023 12:43:48 +0200 Subject: [PATCH 105/123] Add buffer to agreement manager --- golem/managers/agreement/scored_aot.py | 40 +++++++++++++++++++------- golem/managers/mixins.py | 24 ---------------- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/golem/managers/agreement/scored_aot.py b/golem/managers/agreement/scored_aot.py index a9e88a26..bdf8becc 100644 --- a/golem/managers/agreement/scored_aot.py +++ b/golem/managers/agreement/scored_aot.py @@ -1,19 +1,19 @@ import logging -from typing import Awaitable, Callable +from datetime import timedelta +from typing import Awaitable, Callable, MutableSequence, Sequence from golem.managers.agreement.events import AgreementReleased from golem.managers.base import AgreementManager -from golem.managers.mixins import BackgroundLoopMixin, WeightProposalScoringPluginsMixin +from golem.managers.mixins import WeightProposalScoringPluginsMixin from golem.node import GolemNode from golem.resources import Agreement, Proposal +from golem.utils.buffer import ConcurrentlyFilledBuffer from golem.utils.logging import trace_span logger = logging.getLogger(__name__) -class ScoredAheadOfTimeAgreementManager( - BackgroundLoopMixin, WeightProposalScoringPluginsMixin, AgreementManager -): +class ScoredAheadOfTimeAgreementManager(WeightProposalScoringPluginsMixin, AgreementManager): def __init__( self, golem: GolemNode, @@ -24,12 +24,20 @@ def __init__( self._get_draft_proposal = get_draft_proposal self._event_bus = golem.event_bus + self._buffer = ConcurrentlyFilledBuffer( + fill_callback=self._get_draft_proposal, + min_size=5, + max_size=10, + update_callback=self._update_buffer_callback, + update_interval=timedelta(seconds=2), + ) + super().__init__(*args, **kwargs) @trace_span(show_arguments=True) async def get_agreement(self) -> Agreement: while True: - proposal = await self.get_scored_proposal() + proposal = await self._buffer.get_item() try: agreement = await proposal.create_agreement() await agreement.confirm() @@ -47,11 +55,23 @@ async def get_agreement(self) -> Agreement: ) return agreement - async def _background_loop(self) -> None: - while True: - proposal = await self._get_draft_proposal() + @trace_span() + async def start(self) -> None: + await self._buffer.start() - await self.manage_scoring([proposal]) + @trace_span() + async def stop(self) -> None: + await self._buffer.stop() + + @trace_span(show_arguments=True) + async def _update_buffer_callback( + self, items: MutableSequence[Proposal], items_to_process: Sequence[Proposal] + ) -> None: + scored_proposals = [ + proposal for _, proposal in await self.do_scoring(items + items_to_process) + ] + items.clear() + items.extend(scored_proposals) @trace_span(show_arguments=True) async def _terminate_agreement(self, event: AgreementReleased) -> None: diff --git a/golem/managers/mixins.py b/golem/managers/mixins.py index 7e8b5fe1..c1594c3e 100644 --- a/golem/managers/mixins.py +++ b/golem/managers/mixins.py @@ -65,32 +65,8 @@ def __init__( ) -> None: self._demand_offer_parser = demand_offer_parser or TextXPayloadSyntaxParser() - self._scored_proposals: List[Tuple[float, Proposal]] = [] - self._scored_proposals_condition = asyncio.Condition() - super().__init__(*args, **kwargs) - @trace_span(show_arguments=True) - async def manage_scoring(self, proposals: List[Proposal]) -> None: - async with self._scored_proposals_condition: - all_proposals = list(sp[1] for sp in self._scored_proposals) - all_proposals.extend(proposals) - - self._scored_proposals = await self.do_scoring(all_proposals) - - self._scored_proposals_condition.notify_all() - - @trace_span() - async def get_scored_proposal(self): - async with self._scored_proposals_condition: - await self._scored_proposals_condition.wait_for(lambda: 0 < len(self._scored_proposals)) - - score, proposal = self._scored_proposals.pop(0) - - logger.info(f"Proposal `{proposal}` picked with score `{score}`") - - return proposal - async def do_scoring(self, proposals: Sequence[Proposal]) -> List[Tuple[float, Proposal]]: proposals_data = await self._get_proposals_data_from_proposals(proposals) proposal_scores = await self._run_plugins(proposals_data) From c214e744055a59d91a397462b86f96d81e9b3170 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 25 Jul 2023 13:01:18 +0200 Subject: [PATCH 106/123] Improve buffer implementation --- examples/managers/blender/blender.py | 5 ++++- examples/managers/ssh.py | 4 +++- golem/managers/agreement/scored_aot.py | 9 +++++---- golem/managers/demand/auto.py | 7 ++++--- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/examples/managers/blender/blender.py b/examples/managers/blender/blender.py index cc594460..e6d6f0cb 100644 --- a/examples/managers/blender/blender.py +++ b/examples/managers/blender/blender.py @@ -51,7 +51,10 @@ async def run_on_golem( plugins=market_plugins, ) agreement_manager = ScoredAheadOfTimeAgreementManager( - golem, negotiation_manager.get_draft_proposal, plugins=scoring_plugins + golem, + negotiation_manager.get_draft_proposal, + plugins=scoring_plugins, + buffer_size=(3, 10), ) activity_manager = ActivityPoolManager( golem, agreement_manager.get_agreement, size=threads, on_activity_start=init_func diff --git a/examples/managers/ssh.py b/examples/managers/ssh.py index 888702e0..cfdadd67 100644 --- a/examples/managers/ssh.py +++ b/examples/managers/ssh.py @@ -78,7 +78,9 @@ async def main(): ) negotiation_manager = SequentialNegotiationManager(golem, demand_manager.get_initial_proposal) agreement_manager = ScoredAheadOfTimeAgreementManager( - golem, negotiation_manager.get_draft_proposal + golem, + negotiation_manager.get_draft_proposal, + buffer_size=(1, 2), ) activity_manager = SingleUseActivityManager( golem, diff --git a/golem/managers/agreement/scored_aot.py b/golem/managers/agreement/scored_aot.py index bdf8becc..5e4fe29e 100644 --- a/golem/managers/agreement/scored_aot.py +++ b/golem/managers/agreement/scored_aot.py @@ -1,6 +1,6 @@ import logging from datetime import timedelta -from typing import Awaitable, Callable, MutableSequence, Sequence +from typing import Awaitable, Callable, MutableSequence, Sequence, Tuple from golem.managers.agreement.events import AgreementReleased from golem.managers.base import AgreementManager @@ -18,6 +18,7 @@ def __init__( self, golem: GolemNode, get_draft_proposal: Callable[[], Awaitable[Proposal]], + buffer_size: Tuple[int, int] = (1, 1), *args, **kwargs, ): @@ -26,8 +27,8 @@ def __init__( self._buffer = ConcurrentlyFilledBuffer( fill_callback=self._get_draft_proposal, - min_size=5, - max_size=10, + min_size=buffer_size[0], + max_size=buffer_size[1], update_callback=self._update_buffer_callback, update_interval=timedelta(seconds=2), ) @@ -68,7 +69,7 @@ async def _update_buffer_callback( self, items: MutableSequence[Proposal], items_to_process: Sequence[Proposal] ) -> None: scored_proposals = [ - proposal for _, proposal in await self.do_scoring(items + items_to_process) + proposal for _, proposal in await self.do_scoring([*items, *items_to_process]) ] items.clear() items.extend(scored_proposals) diff --git a/golem/managers/demand/auto.py b/golem/managers/demand/auto.py index 18acc63f..12e8e4c6 100644 --- a/golem/managers/demand/auto.py +++ b/golem/managers/demand/auto.py @@ -22,6 +22,7 @@ def __init__( golem: GolemNode, get_allocation: Callable[[], Awaitable[Allocation]], payload: Payload, + buffer_size: Tuple[int, int] = (1, 1000), *args, **kwargs, ) -> None: @@ -32,8 +33,8 @@ def __init__( self._initial_proposals: asyncio.Queue[Proposal] = asyncio.Queue() self._buffer = ConcurrentlyFilledBuffer( fill_callback=self._get_initial_proposal, - min_size=1, - max_size=1000, + min_size=buffer_size[0], + max_size=buffer_size[1], update_callback=self._update_buffer_callback, update_interval=timedelta(seconds=1), ) @@ -77,7 +78,7 @@ async def _update_buffer_callback( self, items: MutableSequence[Proposal], items_to_process: Sequence[Proposal] ) -> None: scored_proposals = [ - proposal for _, proposal in await self.do_scoring(items + items_to_process) + proposal for _, proposal in await self.do_scoring([*items, *items_to_process]) ] items.clear() items.extend(scored_proposals) From 15b8f033d890ad0739b080ac5f8dbaa37ba7e8fe Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 25 Jul 2023 13:06:04 +0200 Subject: [PATCH 107/123] Fix buffer with min_size 0 --- golem/managers/agreement/scored_aot.py | 2 +- golem/utils/buffer.py | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/golem/managers/agreement/scored_aot.py b/golem/managers/agreement/scored_aot.py index 5e4fe29e..f8044cac 100644 --- a/golem/managers/agreement/scored_aot.py +++ b/golem/managers/agreement/scored_aot.py @@ -18,7 +18,7 @@ def __init__( self, golem: GolemNode, get_draft_proposal: Callable[[], Awaitable[Proposal]], - buffer_size: Tuple[int, int] = (1, 1), + buffer_size: Tuple[int, int] = (0, 1), *args, **kwargs, ): diff --git a/golem/utils/buffer.py b/golem/utils/buffer.py index 291677ae..ce9375fa 100644 --- a/golem/utils/buffer.py +++ b/golem/utils/buffer.py @@ -116,19 +116,16 @@ def check(): item = self._items.pop(0) - self._handle_item_requests() + # Check if we need to request any additional items + items_len = len(self._items) + if items_len < self._min_size: + self._handle_item_requests() return item @trace_span() def _handle_item_requests(self) -> None: - # Check if we need to request any additional items items_len = len(self._items) - if self._min_size <= items_len: - logger.debug( - "Items count `%d` are not below min_size `%d`, skipping", items_len, self._min_size - ) - return items_to_request = self._max_size - items_len - len(self._items_requested_tasks) From 951a4962afd7228f2ca923925a662a74363b8758 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 25 Jul 2023 13:30:39 +0200 Subject: [PATCH 108/123] Improve managers module imports --- examples/managers/basic_composition.py | 36 +++++++------ examples/managers/blender/blender.py | 25 +++++---- examples/managers/ssh.py | 19 ++++--- golem/managers/__init__.py | 53 ++++++++++++++++++++ golem/managers/activity/__init__.py | 2 + golem/managers/agreement/__init__.py | 6 +++ golem/managers/agreement/scored_aot.py | 2 +- golem/managers/demand/__init__.py | 3 ++ golem/managers/demand/auto.py | 2 +- golem/managers/negotiation/__init__.py | 12 ++++- golem/managers/network/__init__.py | 3 ++ golem/managers/work/__init__.py | 2 + golem/payload/__init__.py | 2 + tests/unit/test_manager_agreement_plugins.py | 2 +- 14 files changed, 130 insertions(+), 39 deletions(-) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index 43a9788a..8d1d25a4 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -4,26 +4,30 @@ from random import randint, random from typing import List -from golem.managers.activity.single_use import SingleUseActivityManager -from golem.managers.agreement.plugins import MapScore, PropertyValueLerpScore, RandomScore -from golem.managers.agreement.pricings import LinearAverageCostPricing -from golem.managers.agreement.scored_aot import ScoredAheadOfTimeAgreementManager -from golem.managers.base import RejectProposal, WorkContext, WorkResult -from golem.managers.demand.auto import AutoDemandManager -from golem.managers.negotiation import SequentialNegotiationManager -from golem.managers.negotiation.plugins import ( +from golem.managers import ( AddChosenPaymentPlatform, + AutoDemandManager, BlacklistProviderId, + LinearAverageCostPricing, + MapScore, + PayAllPaymentManager, + PropertyValueLerpScore, + RandomScore, RejectIfCostsExceeds, + RejectProposal, + ScoredAheadOfTimeAgreementManager, + SequentialNegotiationManager, + SequentialWorkManager, + SingleUseActivityManager, + WorkContext, + WorkResult, + redundancy_cancel_others_on_first_done, + retry, + work_plugin, ) -from golem.managers.payment.pay_all import PayAllPaymentManager -from golem.managers.work.plugins import redundancy_cancel_others_on_first_done, retry, work_plugin -from golem.managers.work.sequential import SequentialWorkManager from golem.node import GolemNode -from golem.payload import RepositoryVmPayload -from golem.payload.defaults import INF_MEM -from golem.resources.demand.demand import DemandData -from golem.resources.proposal.proposal import ProposalData +from golem.payload import RepositoryVmPayload, defaults +from golem.resources import DemandData, ProposalData from golem.utils.logging import DEFAULT_LOGGING BLACKLISTED_PROVIDERS = [ @@ -106,7 +110,7 @@ async def main(): negotiation_manager.get_draft_proposal, plugins=[ MapScore(linear_average_cost, normalize=True, normalize_flip=True), - [0.5, PropertyValueLerpScore(INF_MEM, zero_at=1, one_at=8)], + [0.5, PropertyValueLerpScore(defaults.INF_MEM, zero_at=1, one_at=8)], [0.1, RandomScore()], [0.0, lambda proposals_data: [random() for _ in range(len(proposals_data))]], [0.0, MapScore(lambda proposal_data: random())], diff --git a/examples/managers/blender/blender.py b/examples/managers/blender/blender.py index e6d6f0cb..8a3326e7 100644 --- a/examples/managers/blender/blender.py +++ b/examples/managers/blender/blender.py @@ -5,17 +5,20 @@ from pathlib import Path from typing import List -from golem.managers.activity.pool import ActivityPoolManager -from golem.managers.agreement.plugins import MapScore -from golem.managers.agreement.pricings import LinearAverageCostPricing -from golem.managers.agreement.scored_aot import ScoredAheadOfTimeAgreementManager -from golem.managers.base import WorkContext, WorkResult -from golem.managers.demand.auto import AutoDemandManager -from golem.managers.negotiation import SequentialNegotiationManager -from golem.managers.negotiation.plugins import AddChosenPaymentPlatform -from golem.managers.payment.pay_all import PayAllPaymentManager -from golem.managers.work.asynchronous import AsynchronousWorkManager -from golem.managers.work.plugins import retry +from golem.managers import ( + ActivityPoolManager, + AddChosenPaymentPlatform, + AsynchronousWorkManager, + AutoDemandManager, + LinearAverageCostPricing, + MapScore, + PayAllPaymentManager, + ScoredAheadOfTimeAgreementManager, + SequentialNegotiationManager, + WorkContext, + WorkResult, + retry, +) from golem.node import GolemNode from golem.payload import RepositoryVmPayload from golem.utils.logging import DEFAULT_LOGGING diff --git a/examples/managers/ssh.py b/examples/managers/ssh.py index cfdadd67..9ad8573c 100644 --- a/examples/managers/ssh.py +++ b/examples/managers/ssh.py @@ -4,14 +4,17 @@ import string from uuid import uuid4 -from golem.managers.activity.single_use import SingleUseActivityManager -from golem.managers.agreement.scored_aot import ScoredAheadOfTimeAgreementManager -from golem.managers.base import WorkContext, WorkResult -from golem.managers.demand.auto import AutoDemandManager -from golem.managers.negotiation import SequentialNegotiationManager -from golem.managers.network.single import SingleNetworkManager -from golem.managers.payment.pay_all import PayAllPaymentManager -from golem.managers.work.sequential import SequentialWorkManager +from golem.managers import ( + AutoDemandManager, + PayAllPaymentManager, + ScoredAheadOfTimeAgreementManager, + SequentialNegotiationManager, + SequentialWorkManager, + SingleNetworkManager, + SingleUseActivityManager, + WorkContext, + WorkResult, +) from golem.node import GolemNode from golem.payload import RepositoryVmPayload from golem.utils.logging import DEFAULT_LOGGING diff --git a/golem/managers/__init__.py b/golem/managers/__init__.py index e69de29b..74af3f98 100644 --- a/golem/managers/__init__.py +++ b/golem/managers/__init__.py @@ -0,0 +1,53 @@ +from golem.managers.activity import ActivityPoolManager, SingleUseActivityManager +from golem.managers.agreement import ( + LinearAverageCostPricing, + MapScore, + PropertyValueLerpScore, + RandomScore, + ScoredAheadOfTimeAgreementManager, +) +from golem.managers.base import RejectProposal, WorkContext, WorkResult +from golem.managers.demand import AutoDemandManager +from golem.managers.negotiation import ( + AddChosenPaymentPlatform, + BlacklistProviderId, + RejectIfCostsExceeds, + SequentialNegotiationManager, +) +from golem.managers.network import SingleNetworkManager +from golem.managers.payment import PayAllPaymentManager +from golem.managers.work import ( + SequentialWorkManager, + redundancy_cancel_others_on_first_done, + retry, + work_plugin, +) +from golem.managers.work.asynchronous import AsynchronousWorkManager + +__all__ = ( + "SingleUseActivityManager", + "ActivityPoolManager", + "default_on_activity_start", + "default_on_activity_stop", + "AgreementReleased", + "ScoredAheadOfTimeAgreementManager", + "MapScore", + "PropertyValueLerpScore", + "RandomScore", + "LinearAverageCostPricing", + "RejectProposal", + "WorkContext", + "WorkResult", + "AutoDemandManager", + "PayAllPaymentManager", + "SequentialNegotiationManager", + "AddChosenPaymentPlatform", + "BlacklistProviderId", + "RejectIfCostsExceeds", + "AsynchronousWorkManager", + "SequentialWorkManager", + "work_plugin", + "redundancy_cancel_others_on_first_done", + "retry", + "SingleNetworkManager", +) diff --git a/golem/managers/activity/__init__.py b/golem/managers/activity/__init__.py index 45b0e3aa..2a7a8976 100644 --- a/golem/managers/activity/__init__.py +++ b/golem/managers/activity/__init__.py @@ -1,7 +1,9 @@ from golem.managers.activity.defaults import default_on_activity_start, default_on_activity_stop +from golem.managers.activity.pool import ActivityPoolManager from golem.managers.activity.single_use import SingleUseActivityManager __all__ = ( + "ActivityPoolManager", "SingleUseActivityManager", "default_on_activity_start", "default_on_activity_stop", diff --git a/golem/managers/agreement/__init__.py b/golem/managers/agreement/__init__.py index 9437cbd1..c61f2a0c 100644 --- a/golem/managers/agreement/__init__.py +++ b/golem/managers/agreement/__init__.py @@ -1,7 +1,13 @@ from golem.managers.agreement.events import AgreementReleased +from golem.managers.agreement.plugins import MapScore, PropertyValueLerpScore, RandomScore +from golem.managers.agreement.pricings import LinearAverageCostPricing from golem.managers.agreement.scored_aot import ScoredAheadOfTimeAgreementManager __all__ = ( "AgreementReleased", "ScoredAheadOfTimeAgreementManager", + "MapScore", + "PropertyValueLerpScore", + "RandomScore", + "LinearAverageCostPricing", ) diff --git a/golem/managers/agreement/scored_aot.py b/golem/managers/agreement/scored_aot.py index f8044cac..b160ac3d 100644 --- a/golem/managers/agreement/scored_aot.py +++ b/golem/managers/agreement/scored_aot.py @@ -58,7 +58,7 @@ async def get_agreement(self) -> Agreement: @trace_span() async def start(self) -> None: - await self._buffer.start() + await self._buffer.start(fill=True) @trace_span() async def stop(self) -> None: diff --git a/golem/managers/demand/__init__.py b/golem/managers/demand/__init__.py index e69de29b..b79750a0 100644 --- a/golem/managers/demand/__init__.py +++ b/golem/managers/demand/__init__.py @@ -0,0 +1,3 @@ +from golem.managers.demand.auto import AutoDemandManager + +__all__ = ("AutoDemandManager",) diff --git a/golem/managers/demand/auto.py b/golem/managers/demand/auto.py index 12e8e4c6..d06a7aff 100644 --- a/golem/managers/demand/auto.py +++ b/golem/managers/demand/auto.py @@ -53,7 +53,7 @@ async def _get_initial_proposal(self) -> Proposal: @trace_span() async def _background_loop(self) -> None: - await self._buffer.start() + await self._buffer.start(fill=True) await self._create_and_subscribe_demand() try: while True: diff --git a/golem/managers/negotiation/__init__.py b/golem/managers/negotiation/__init__.py index d45d6578..a78bb935 100644 --- a/golem/managers/negotiation/__init__.py +++ b/golem/managers/negotiation/__init__.py @@ -1,3 +1,13 @@ +from golem.managers.negotiation.plugins import ( + AddChosenPaymentPlatform, + BlacklistProviderId, + RejectIfCostsExceeds, +) from golem.managers.negotiation.sequential import SequentialNegotiationManager -__all__ = ("SequentialNegotiationManager",) +__all__ = ( + "SequentialNegotiationManager", + "AddChosenPaymentPlatform", + "BlacklistProviderId", + "RejectIfCostsExceeds", +) diff --git a/golem/managers/network/__init__.py b/golem/managers/network/__init__.py index e69de29b..22c6ed42 100644 --- a/golem/managers/network/__init__.py +++ b/golem/managers/network/__init__.py @@ -0,0 +1,3 @@ +from golem.managers.network.single import SingleNetworkManager + +__all__ = ("SingleNetworkManager",) diff --git a/golem/managers/work/__init__.py b/golem/managers/work/__init__.py index 8913103d..3ebe7a4d 100644 --- a/golem/managers/work/__init__.py +++ b/golem/managers/work/__init__.py @@ -1,7 +1,9 @@ +from golem.managers.work.asynchronous import AsynchronousWorkManager from golem.managers.work.plugins import redundancy_cancel_others_on_first_done, retry, work_plugin from golem.managers.work.sequential import SequentialWorkManager __all__ = ( + "AsynchronousWorkManager", "SequentialWorkManager", "work_plugin", "redundancy_cancel_others_on_first_done", diff --git a/golem/payload/__init__.py b/golem/payload/__init__.py index 25e5afbc..a13395ea 100644 --- a/golem/payload/__init__.py +++ b/golem/payload/__init__.py @@ -1,3 +1,4 @@ +from golem.payload import defaults from golem.payload.base import Payload, constraint, prop from golem.payload.constraints import ( Constraint, @@ -31,4 +32,5 @@ "PayloadSyntaxParser", "PropertyName", "PropertyValue", + "defaults", ) diff --git a/tests/unit/test_manager_agreement_plugins.py b/tests/unit/test_manager_agreement_plugins.py index 8ca738cb..92794482 100644 --- a/tests/unit/test_manager_agreement_plugins.py +++ b/tests/unit/test_manager_agreement_plugins.py @@ -1,6 +1,6 @@ import pytest -from golem.managers.agreement.plugins import PropertyValueLerpScore, RandomScore +from golem.managers import PropertyValueLerpScore, RandomScore @pytest.mark.parametrize( From 7a2457ed90fa892f139e5ccbbfe8d52b2415328b Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 25 Jul 2023 15:51:14 +0200 Subject: [PATCH 109/123] Add missing imports; enamed pipeline test module --- golem/managers/activity/__init__.py | 6 ++++-- golem/managers/work/__init__.py | 2 ++ tests/unit/{test_mid.py => test_pipeline.py} | 0 3 files changed, 6 insertions(+), 2 deletions(-) rename tests/unit/{test_mid.py => test_pipeline.py} (100%) diff --git a/golem/managers/activity/__init__.py b/golem/managers/activity/__init__.py index 2a7a8976..39f7a284 100644 --- a/golem/managers/activity/__init__.py +++ b/golem/managers/activity/__init__.py @@ -1,10 +1,12 @@ from golem.managers.activity.defaults import default_on_activity_start, default_on_activity_stop +from golem.managers.activity.mixins import ActivityPrepareReleaseMixin from golem.managers.activity.pool import ActivityPoolManager from golem.managers.activity.single_use import SingleUseActivityManager __all__ = ( - "ActivityPoolManager", - "SingleUseActivityManager", "default_on_activity_start", "default_on_activity_stop", + "ActivityPrepareReleaseMixin", + "ActivityPoolManager", + "SingleUseActivityManager", ) diff --git a/golem/managers/work/__init__.py b/golem/managers/work/__init__.py index 3ebe7a4d..437c17a5 100644 --- a/golem/managers/work/__init__.py +++ b/golem/managers/work/__init__.py @@ -1,9 +1,11 @@ from golem.managers.work.asynchronous import AsynchronousWorkManager +from golem.managers.work.mixins import WorkManagerPluginsMixin from golem.managers.work.plugins import redundancy_cancel_others_on_first_done, retry, work_plugin from golem.managers.work.sequential import SequentialWorkManager __all__ = ( "AsynchronousWorkManager", + "WorkManagerPluginsMixin", "SequentialWorkManager", "work_plugin", "redundancy_cancel_others_on_first_done", diff --git a/tests/unit/test_mid.py b/tests/unit/test_pipeline.py similarity index 100% rename from tests/unit/test_mid.py rename to tests/unit/test_pipeline.py From 8523334b69eace86e629a2898c343aec041bfd17 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 25 Jul 2023 15:57:42 +0200 Subject: [PATCH 110/123] Add missing import to managers module --- golem/managers/__init__.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/golem/managers/__init__.py b/golem/managers/__init__.py index 74af3f98..117461a4 100644 --- a/golem/managers/__init__.py +++ b/golem/managers/__init__.py @@ -8,6 +8,7 @@ ) from golem.managers.base import RejectProposal, WorkContext, WorkResult from golem.managers.demand import AutoDemandManager +from golem.managers.mixins import BackgroundLoopMixin from golem.managers.negotiation import ( AddChosenPaymentPlatform, BlacklistProviderId, @@ -25,29 +26,27 @@ from golem.managers.work.asynchronous import AsynchronousWorkManager __all__ = ( - "SingleUseActivityManager", "ActivityPoolManager", - "default_on_activity_start", - "default_on_activity_stop", - "AgreementReleased", - "ScoredAheadOfTimeAgreementManager", + "SingleUseActivityManager", + "LinearAverageCostPricing", "MapScore", "PropertyValueLerpScore", "RandomScore", - "LinearAverageCostPricing", + "ScoredAheadOfTimeAgreementManager", "RejectProposal", "WorkContext", "WorkResult", "AutoDemandManager", - "PayAllPaymentManager", - "SequentialNegotiationManager", + "BackgroundLoopMixin", "AddChosenPaymentPlatform", "BlacklistProviderId", "RejectIfCostsExceeds", - "AsynchronousWorkManager", + "SequentialNegotiationManager", + "SingleNetworkManager", + "PayAllPaymentManager", "SequentialWorkManager", - "work_plugin", "redundancy_cancel_others_on_first_done", "retry", - "SingleNetworkManager", + "work_plugin", + "AsynchronousWorkManager", ) From 1e812b609664a1007293401c32c235eddad25b55 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Tue, 25 Jul 2023 16:13:56 +0200 Subject: [PATCH 111/123] Add BackgroundLoopMixin happ path test --- golem/managers/__init__.py | 3 ++- tests/unit/test_managers_mixins.py | 33 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_managers_mixins.py diff --git a/golem/managers/__init__.py b/golem/managers/__init__.py index 117461a4..8cd8d83c 100644 --- a/golem/managers/__init__.py +++ b/golem/managers/__init__.py @@ -6,7 +6,7 @@ RandomScore, ScoredAheadOfTimeAgreementManager, ) -from golem.managers.base import RejectProposal, WorkContext, WorkResult +from golem.managers.base import Manager, RejectProposal, WorkContext, WorkResult from golem.managers.demand import AutoDemandManager from golem.managers.mixins import BackgroundLoopMixin from golem.managers.negotiation import ( @@ -33,6 +33,7 @@ "PropertyValueLerpScore", "RandomScore", "ScoredAheadOfTimeAgreementManager", + "Manager", "RejectProposal", "WorkContext", "WorkResult", diff --git a/tests/unit/test_managers_mixins.py b/tests/unit/test_managers_mixins.py new file mode 100644 index 00000000..895189c2 --- /dev/null +++ b/tests/unit/test_managers_mixins.py @@ -0,0 +1,33 @@ +import asyncio +import random +from typing import Optional + +from golem.managers import BackgroundLoopMixin, Manager + + +class FooBarBackgroundLoopManager(BackgroundLoopMixin, Manager): + def __init__(self, foo: int, *args, **kwargs) -> None: + self.foo: int = foo + self.bar: Optional[int] = None + + super().__init__(*args, **kwargs) + + async def _background_loop(self) -> None: + self.bar = self.foo + while True: + # await to switch out of the loop + await asyncio.sleep(1) + + +async def test_background_loop_mixin_ok(): + given_bar = random.randint(0, 10) + manager = FooBarBackgroundLoopManager(given_bar) + assert not manager.is_started() + assert manager.bar is None + async with manager: + # await to switch to `FooBarBackgroundLoopManager._background_loop` + await asyncio.sleep(0.1) + assert manager.is_started() + assert manager.bar == given_bar + assert not manager.is_started() + assert manager.bar == given_bar From 8f1a6c9e97d701aa8d1134e957be2e410f94923e Mon Sep 17 00:00:00 2001 From: shadeofblue Date: Tue, 25 Jul 2023 19:28:35 +0200 Subject: [PATCH 112/123] switch the goth network to rinkeby --- tests/integration/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/run-tests.sh b/tests/integration/run-tests.sh index 00f07ac6..3805d931 100755 --- a/tests/integration/run-tests.sh +++ b/tests/integration/run-tests.sh @@ -1,4 +1,4 @@ #!/bin/bash source /tmp/goth_interactive.env -pytest -sv tests/integration +YAGNA_PAYMENT_NETWORK=rinkeby pytest -sv tests/integration From 786a808089b9d09eef6fea787ca0a022485fb81a Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 26 Jul 2023 08:58:57 +0200 Subject: [PATCH 113/123] Fix unit tests on 3.11 --- tests/unit/utils/test_buffer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/utils/test_buffer.py b/tests/unit/utils/test_buffer.py index 69fd820e..b770c42a 100644 --- a/tests/unit/utils/test_buffer.py +++ b/tests/unit/utils/test_buffer.py @@ -117,7 +117,9 @@ async def test_buffer_get_item_will_trigger_fill_on_below_min_size(create_buffer assert fill_callback.await_count == 10 - done, _ = await asyncio.wait([buffer.get_item() for _ in range(6)], timeout=0.05) + done, _ = await asyncio.wait( + [asyncio.ensure_future(buffer.get_item()) for _ in range(6)], timeout=0.05 + ) assert [d.result() for d in done] == fill_callback.mock_calls[1:7] From 2f2a9f0e3f1e1b8f1907967786a1b80d6b674412 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Wed, 26 Jul 2023 11:57:23 +0200 Subject: [PATCH 114/123] Add WeightProposalScoringPluginsMixin unit tests --- golem/managers/__init__.py | 6 +- golem/managers/mixins.py | 2 +- golem/managers/work/plugins.py | 2 +- tests/unit/conftest.py | 129 +++++++++++++++++++++++++++++ tests/unit/test_managers_mixins.py | 111 ++++++++++++++++++++++++- 5 files changed, 244 insertions(+), 6 deletions(-) create mode 100644 tests/unit/conftest.py diff --git a/golem/managers/__init__.py b/golem/managers/__init__.py index 8cd8d83c..f9ba9ba3 100644 --- a/golem/managers/__init__.py +++ b/golem/managers/__init__.py @@ -6,9 +6,9 @@ RandomScore, ScoredAheadOfTimeAgreementManager, ) -from golem.managers.base import Manager, RejectProposal, WorkContext, WorkResult +from golem.managers.base import Manager, ManagerScorePlugin, RejectProposal, WorkContext, WorkResult from golem.managers.demand import AutoDemandManager -from golem.managers.mixins import BackgroundLoopMixin +from golem.managers.mixins import BackgroundLoopMixin, WeightProposalScoringPluginsMixin from golem.managers.negotiation import ( AddChosenPaymentPlatform, BlacklistProviderId, @@ -34,11 +34,13 @@ "RandomScore", "ScoredAheadOfTimeAgreementManager", "Manager", + "ManagerScorePlugin", "RejectProposal", "WorkContext", "WorkResult", "AutoDemandManager", "BackgroundLoopMixin", + "WeightProposalScoringPluginsMixin", "AddChosenPaymentPlatform", "BlacklistProviderId", "RejectIfCostsExceeds", diff --git a/golem/managers/mixins.py b/golem/managers/mixins.py index c1594c3e..c2ea8ebe 100644 --- a/golem/managers/mixins.py +++ b/golem/managers/mixins.py @@ -131,7 +131,7 @@ def _transpose_plugin_scores( return [ (plugin_weight, plugin_scores[proposal_index]) for plugin_weight, plugin_scores in plugin_scores - if plugin_scores[proposal_index] is None + if plugin_scores[proposal_index] is not None ] # FIXME: This should be already provided by low level diff --git a/golem/managers/work/plugins.py b/golem/managers/work/plugins.py index ba95cedf..70658c20 100644 --- a/golem/managers/work/plugins.py +++ b/golem/managers/work/plugins.py @@ -63,7 +63,7 @@ def redundancy_cancel_others_on_first_done(size: int): def _redundancy(do_work: DoWorkCallable): @wraps(do_work) async def wrapper(work: Work) -> WorkResult: - tasks = [do_work(work) for _ in range(size)] + tasks = [asyncio.ensure_future(do_work(work)) for _ in range(size)] tasks_done, tasks_pending = await asyncio.wait( tasks, return_when=asyncio.FIRST_COMPLETED diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 00000000..9ea4164a --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1,129 @@ +import datetime +from typing import Dict + +import pytest +from ya_market.models.proposal import Proposal as yaProposal + + +@pytest.fixture +def yagna_proposal(): + def _yagna_proposal(properties: Dict = {}): + return yaProposal( + proposal_id="R-8bee6cba3e99059ee2a9a355c7f9d502f2ee836cecd00d98ebac87cc88b9fc4f", + issuer_id="0xf97cdecb935fc970ed1bd3da6eacc7a325e582dc", + state="Initial", + timestamp=datetime.datetime.utcnow(), + constraints="(&\n" + " (golem.srv.comp.expiration>4843951180715)\n" + " (golem.node.debug.subnet=public)\n" + ")", + properties={ + "golem.activity.caps.transfer.protocol": ["http", "https", "gftp"], + "golem.com.payment.debit-notes.accept-timeout?": 240, + "golem.com.pricing.model": "linear", + "golem.com.pricing.model.linear.coeffs": [5e-05, 0.0001, 0.0], + "golem.com.scheme": "payu", + "golem.com.scheme.payu.debit-note.interval-sec?": 120, + "golem.com.scheme.payu.payment-timeout-sec?": 120, + "golem.com.usage.vector": ["golem.usage.duration_sec", "golem.usage.cpu_sec"], + "golem.inf.cpu.architecture": "x86_64", + "golem.inf.cpu.brand": "Intel(R) Core(TM) i7-8700 CPU @ " "3.20GHz", + "golem.inf.cpu.capabilities": [ + "sse3", + "pclmulqdq", + "dtes64", + "monitor", + "dscpl", + "vmx", + "smx", + "eist", + "tm2", + "ssse3", + "fma", + "cmpxchg16b", + "pdcm", + "pcid", + "sse41", + "sse42", + "x2apic", + "movbe", + "popcnt", + "tsc_deadline", + "aesni", + "xsave", + "osxsave", + "avx", + "f16c", + "rdrand", + "fpu", + "vme", + "de", + "pse", + "tsc", + "msr", + "pae", + "mce", + "cx8", + "apic", + "sep", + "mtrr", + "pge", + "mca", + "cmov", + "pat", + "pse36", + "clfsh", + "ds", + "acpi", + "mmx", + "fxsr", + "sse", + "sse2", + "ss", + "htt", + "tm", + "pbe", + "fsgsbase", + "adjust_msr", + "bmi1", + "hle", + "avx2", + "smep", + "bmi2", + "rep_movsb_stosb", + "invpcid", + "rtm", + "deprecate_fpu_cs_ds", + "mpx", + "rdseed", + "adx", + "smap", + "clflushopt", + "processor_trace", + "sgx", + "sgx_lc", + ], + "golem.inf.cpu.cores": 6, + "golem.inf.cpu.model": "Stepping 10 Family 6 Model 302", + "golem.inf.cpu.threads": 1, + "golem.inf.cpu.vendor": "GenuineIntel", + "golem.inf.mem.gib": 4.0, + "golem.inf.storage.gib": 40.0, + "golem.node.debug.subnet": "public", + "golem.node.id.name": "golem2005_2.h", + "golem.node.net.is-public": False, + "golem.runtime.capabilities": [ + "inet", + "vpn", + "manifest-support", + "start-entrypoint", + ], + "golem.runtime.name": "vm", + "golem.runtime.version": "0.3.0", + "golem.srv.caps.multi-activity": True, + "golem.srv.caps.payload-manifest": True, + **properties, + }, + ) + + return _yagna_proposal diff --git a/tests/unit/test_managers_mixins.py b/tests/unit/test_managers_mixins.py index 895189c2..bf1552af 100644 --- a/tests/unit/test_managers_mixins.py +++ b/tests/unit/test_managers_mixins.py @@ -1,8 +1,21 @@ import asyncio import random -from typing import Optional +from datetime import timedelta +from typing import Dict, Iterable, Optional, Sequence, Tuple +from unittest.mock import AsyncMock -from golem.managers import BackgroundLoopMixin, Manager +import pytest + +from golem.managers import ( + BackgroundLoopMixin, + LinearAverageCostPricing, + Manager, + ManagerScorePlugin, + MapScore, + PropertyValueLerpScore, + WeightProposalScoringPluginsMixin, +) +from golem.payload import defaults class FooBarBackgroundLoopManager(BackgroundLoopMixin, Manager): @@ -31,3 +44,97 @@ async def test_background_loop_mixin_ok(): assert manager.bar == given_bar assert not manager.is_started() assert manager.bar == given_bar + + +class FooBarWeightProposalScoringPluginsManager(WeightProposalScoringPluginsMixin, Manager): + ... + + +@pytest.mark.parametrize( + "given_plugins, properties, expected_weights", + ( + ( + [], + ({} for _ in range(5)), + [0.0, 0.0, 0.0, 0.0, 0.0], + ), + ( + ( + ( + 0.5, + PropertyValueLerpScore(defaults.INF_MEM, zero_at=1, one_at=5), + ), + ), + ({defaults.INF_MEM: gib} for gib in range(7)), + [1.0, 1.0, 0.75, 0.5, 0.25, 0.0, 0.0], + ), + ( + ( + ( + 1.0, + MapScore( + LinearAverageCostPricing( + average_cpu_load=1, average_duration=timedelta(seconds=60) + ), + normalize=True, + normalize_flip=True, + ), + ), + ), + ( + { + "golem.com.pricing.model": "linear", + "golem.com.pricing.model.linear.coeffs": coeffs, + } + for coeffs in ([5e-05, 0.0001, 0.0], [5e-05, 0.0003, 0.0], [5e-05, 0.0002, 0.0]) + ), + [1.0, 0.5, 0.0], + ), + ( + ( + ( + 1.0, + PropertyValueLerpScore(defaults.INF_MEM, zero_at=1, one_at=5), + ), + ( + 1.0, + MapScore( + LinearAverageCostPricing( + average_cpu_load=1, average_duration=timedelta(seconds=60) + ), + normalize=True, + normalize_flip=True, + ), + ), + ), + ( + { + defaults.INF_MEM: gib, + "golem.com.pricing.model": "linear", + "golem.com.pricing.model.linear.coeffs": coeffs, + } + for gib, coeffs in ( + (4, [5e-05, 0.0001, 0.0]), + (3, [5e-05, 0.0003, 0.0]), + (2, [5e-05, 0.0002, 0.0]), + ) + ), + [0.875, 0.375, 0.25], + ), + ), +) +async def test_weight_proposal_scoring_plugins_mixin_ok( + yagna_proposal, + given_plugins: Sequence[Tuple[float, ManagerScorePlugin]], + properties: Iterable[Dict], + expected_weights: Sequence[float], +): + given_proposals = [] + for props in properties: + proposal = AsyncMock() + proposal.get_data.return_value = yagna_proposal(properties=props) + given_proposals.append(proposal) + + manager = FooBarWeightProposalScoringPluginsManager(plugins=given_plugins) + received_proposals = await manager.do_scoring(given_proposals) + assert expected_weights == [weight for weight, _ in received_proposals] From 46e876765b6a79ef9cb9135788646e60e06722ac Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 27 Jul 2023 11:55:51 +0200 Subject: [PATCH 115/123] Add test_work_manager_plugins_manager_mixin_ok --- golem/managers/__init__.py | 28 +++++++++++++++- golem/managers/base.py | 9 +---- golem/managers/work/plugins.py | 7 ++-- tests/unit/test_managers_mixins.py | 54 ++++++++++++++++++++++++++++++ tests/unit/utils/test_buffer.py | 2 +- 5 files changed, 87 insertions(+), 13 deletions(-) diff --git a/golem/managers/__init__.py b/golem/managers/__init__.py index f9ba9ba3..bd84cee2 100644 --- a/golem/managers/__init__.py +++ b/golem/managers/__init__.py @@ -6,7 +6,22 @@ RandomScore, ScoredAheadOfTimeAgreementManager, ) -from golem.managers.base import Manager, ManagerScorePlugin, RejectProposal, WorkContext, WorkResult +from golem.managers.base import ( + ActivityManager, + AgreementManager, + DemandManager, + DoWorkCallable, + Manager, + ManagerScorePlugin, + NegotiationManager, + NetworkManager, + PaymentManager, + RejectProposal, + Work, + WorkContext, + WorkManager, + WorkResult, +) from golem.managers.demand import AutoDemandManager from golem.managers.mixins import BackgroundLoopMixin, WeightProposalScoringPluginsMixin from golem.managers.negotiation import ( @@ -19,6 +34,7 @@ from golem.managers.payment import PayAllPaymentManager from golem.managers.work import ( SequentialWorkManager, + WorkManagerPluginsMixin, redundancy_cancel_others_on_first_done, retry, work_plugin, @@ -33,11 +49,20 @@ "PropertyValueLerpScore", "RandomScore", "ScoredAheadOfTimeAgreementManager", + "DoWorkCallable", "Manager", "ManagerScorePlugin", "RejectProposal", + "Work", + "WorkManager", "WorkContext", "WorkResult", + "NetworkManager", + "PaymentManager", + "DemandManager", + "NegotiationManager", + "AgreementManager", + "ActivityManager", "AutoDemandManager", "BackgroundLoopMixin", "WeightProposalScoringPluginsMixin", @@ -48,6 +73,7 @@ "SingleNetworkManager", "PayAllPaymentManager", "SequentialWorkManager", + "WorkManagerPluginsMixin", "redundancy_cancel_others_on_first_done", "retry", "work_plugin", diff --git a/golem/managers/base.py b/golem/managers/base.py index b8c8da4b..e5598cac 100644 --- a/golem/managers/base.py +++ b/golem/managers/base.py @@ -88,14 +88,7 @@ class WorkResult: WORK_PLUGIN_FIELD_NAME = "_work_plugins" - -class Work(ABC): - _work_plugins: Optional[List["WorkManagerPlugin"]] - - @abstractmethod - def __call__(self, context: WorkContext) -> Awaitable[Optional[WorkResult]]: - ... - +Work = Callable[[WorkContext], Awaitable[Optional[WorkResult]]] DoWorkCallable = Callable[[Work], Awaitable[WorkResult]] diff --git a/golem/managers/work/plugins.py b/golem/managers/work/plugins.py index 70658c20..5612b9bb 100644 --- a/golem/managers/work/plugins.py +++ b/golem/managers/work/plugins.py @@ -1,6 +1,7 @@ import asyncio import logging from functools import wraps +from typing import List from golem.managers.base import ( WORK_PLUGIN_FIELD_NAME, @@ -16,9 +17,9 @@ def work_plugin(plugin: WorkManagerPlugin): def _work_plugin(work: Work): if not hasattr(work, WORK_PLUGIN_FIELD_NAME): - work._work_plugins = [] + setattr(work, WORK_PLUGIN_FIELD_NAME, []) - work._work_plugins.append(plugin) # type: ignore [union-attr] + getattr(work, WORK_PLUGIN_FIELD_NAME).append(plugin) return work @@ -63,7 +64,7 @@ def redundancy_cancel_others_on_first_done(size: int): def _redundancy(do_work: DoWorkCallable): @wraps(do_work) async def wrapper(work: Work) -> WorkResult: - tasks = [asyncio.ensure_future(do_work(work)) for _ in range(size)] + tasks: List[asyncio.Task] = [asyncio.ensure_future(do_work(work)) for _ in range(size)] tasks_done, tasks_pending = await asyncio.wait( tasks, return_when=asyncio.FIRST_COMPLETED diff --git a/tests/unit/test_managers_mixins.py b/tests/unit/test_managers_mixins.py index bf1552af..73e05c23 100644 --- a/tests/unit/test_managers_mixins.py +++ b/tests/unit/test_managers_mixins.py @@ -8,12 +8,18 @@ from golem.managers import ( BackgroundLoopMixin, + DoWorkCallable, LinearAverageCostPricing, Manager, ManagerScorePlugin, MapScore, PropertyValueLerpScore, WeightProposalScoringPluginsMixin, + Work, + WorkContext, + WorkManager, + WorkManagerPluginsMixin, + WorkResult, ) from golem.payload import defaults @@ -138,3 +144,51 @@ async def test_weight_proposal_scoring_plugins_mixin_ok( manager = FooBarWeightProposalScoringPluginsManager(plugins=given_plugins) received_proposals = await manager.do_scoring(given_proposals) assert expected_weights == [weight for weight, _ in received_proposals] + + +class FooBarWorkManagerPluginsManager(WorkManagerPluginsMixin, WorkManager): + def __init__(self, do_work: DoWorkCallable, *args, **kwargs): + self._do_work = do_work + super().__init__(*args, **kwargs) + + async def do_work(self, work: Work) -> WorkResult: + return await self._do_work_with_plugins(self._do_work, work) + + +@pytest.mark.parametrize( + "expected_work_result, expected_called_count", + ( + ("ZERO", None), + ("ONE", 1), + ("TWO", 2), + ("TEN", 10), + ), +) +async def test_work_manager_plugins_manager_mixin_ok( + expected_work_result: str, expected_called_count: Optional[int] +): + async def _do_work_func(work: Work) -> WorkResult: + work_result = await work(AsyncMock()) + if not isinstance(work_result, WorkResult): + work_result = WorkResult(result=work_result) + return work_result + + async def _work(context: WorkContext) -> Optional[WorkResult]: + return WorkResult(result=expected_work_result) + + def _plugin(do_work: DoWorkCallable) -> DoWorkCallable: + async def wrapper(work: Work) -> WorkResult: + work_result = await do_work(work) + work_result.extras["called_count"] = work_result.extras.get("called_count", 0) + 1 + return work_result + + return wrapper + + work_plugins = [_plugin for _ in range(expected_called_count or 0)] + + manager = FooBarWorkManagerPluginsManager(do_work=_do_work_func, plugins=work_plugins) + + result = await manager.do_work(_work) + + assert result.result == expected_work_result + assert result.extras.get("called_count") == expected_called_count diff --git a/tests/unit/utils/test_buffer.py b/tests/unit/utils/test_buffer.py index b770c42a..38697942 100644 --- a/tests/unit/utils/test_buffer.py +++ b/tests/unit/utils/test_buffer.py @@ -118,7 +118,7 @@ async def test_buffer_get_item_will_trigger_fill_on_below_min_size(create_buffer assert fill_callback.await_count == 10 done, _ = await asyncio.wait( - [asyncio.ensure_future(buffer.get_item()) for _ in range(6)], timeout=0.05 + [asyncio.create_task(buffer.get_item()) for _ in range(6)], timeout=0.05 ) assert [d.result() for d in done] == fill_callback.mock_calls[1:7] From c99ca2feecd06deb38b9c6384c8ddf3c9edbc866 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 27 Jul 2023 14:46:24 +0200 Subject: [PATCH 116/123] Add SshHandler to managers ssh example --- examples/managers/ssh.py | 73 ++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/examples/managers/ssh.py b/examples/managers/ssh.py index 9ad8573c..0ae420ca 100644 --- a/examples/managers/ssh.py +++ b/examples/managers/ssh.py @@ -13,53 +13,57 @@ SingleNetworkManager, SingleUseActivityManager, WorkContext, - WorkResult, ) from golem.node import GolemNode from golem.payload import RepositoryVmPayload from golem.utils.logging import DEFAULT_LOGGING -def on_activity_start(get_network_deploy_args): - async def _on_activity_start(context: WorkContext): +class SshHandler: + def __init__(self, app_key: str, network_manager: SingleNetworkManager) -> None: + self._app_key = app_key + self._network_manager = network_manager + self.started = False + + async def on_activity_start(self, context: WorkContext): deploy_args = { - "net": [await get_network_deploy_args(context._activity.parent.parent.data.issuer_id)] + "net": [ + await self._network_manager.get_deploy_args( + context._activity.parent.parent.data.issuer_id + ) + ] } batch = await context.create_batch() batch.deploy(deploy_args) batch.start() await batch() - return _on_activity_start - - -def work(app_key, get_provider_uri): - async def _work(context: WorkContext) -> str: - password = "".join(random.choice(string.ascii_letters + string.digits) for _ in range(8)) - batch = await context.create_batch() - batch.run("syslogd") - batch.run("ssh-keygen -A") - batch.run(f'echo -e "{password}\n{password}" | passwd') - batch.run("/usr/sbin/sshd") - batch_result = await batch() - result = "" - for event in batch_result: - result += f"{event.stdout}" - + async def work(self, context: WorkContext) -> None: + password = await self._run_ssh_server(context) print( "Connect with:\n" " ssh -o ProxyCommand='websocat asyncstdio: " - f"{await get_provider_uri(context._activity.parent.parent.data.issuer_id, 'ws')}" + f"{await self._network_manager.get_provider_uri(context._activity.parent.parent.data.issuer_id, 'ws')}" f" --binary " - f'-H=Authorization:"Bearer {app_key}"\' root@{uuid4().hex} ' + f'-H=Authorization:"Bearer {self._app_key}"\' root@{uuid4().hex} ' ) print(f"PASSWORD: {password}") + self.started = True + try: + while True: + await asyncio.sleep(1) + except asyncio.CancelledError: + ... - for _ in range(3): - await asyncio.sleep(1) - return result - - return _work + async def _run_ssh_server(self, context: WorkContext) -> str: + password = "".join(random.choice(string.ascii_letters + string.digits) for _ in range(8)) + batch = await context.create_batch() + batch.run("syslogd") + batch.run("ssh-keygen -A") + batch.run(f'echo -e "{password}\n{password}" | passwd') + batch.run("/usr/sbin/sshd") + await batch() + return password async def main(): @@ -85,18 +89,21 @@ async def main(): negotiation_manager.get_draft_proposal, buffer_size=(1, 2), ) + + ssh_handler = SshHandler(golem._api_config.app_key, network_manager=network_manager) + activity_manager = SingleUseActivityManager( golem, agreement_manager.get_agreement, - on_activity_start=on_activity_start(network_manager.get_deploy_args), + on_activity_start=ssh_handler.on_activity_start, ) work_manager = SequentialWorkManager(golem, activity_manager.do_work) - # TODO use different managers so it allows to finish work func without destroying activity async with golem, network_manager, payment_manager, demand_manager, negotiation_manager, agreement_manager: # noqa: E501 line too long - result: WorkResult = await work_manager.do_work( - work(golem._api_config.app_key, network_manager.get_provider_uri) - ) - print(f"\nWORK MANAGER RESULTS:{result.result}\n") + task = asyncio.create_task(work_manager.do_work(ssh_handler.work)) + while not ssh_handler.started: + await asyncio.sleep(0.1) + _, pending = await asyncio.wait((task,), timeout=20) + [p.cancel() for p in pending] if __name__ == "__main__": From d528f6b9885bd926b848764a852ffb236a0670ea Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 27 Jul 2023 15:10:18 +0200 Subject: [PATCH 117/123] Add QueueWorkManager and use it in blender example --- examples/managers/blender/blender.py | 6 ++-- golem/managers/__init__.py | 2 ++ golem/managers/work/__init__.py | 2 ++ golem/managers/work/queue.py | 44 ++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 golem/managers/work/queue.py diff --git a/examples/managers/blender/blender.py b/examples/managers/blender/blender.py index 8a3326e7..897b534b 100644 --- a/examples/managers/blender/blender.py +++ b/examples/managers/blender/blender.py @@ -8,11 +8,11 @@ from golem.managers import ( ActivityPoolManager, AddChosenPaymentPlatform, - AsynchronousWorkManager, AutoDemandManager, LinearAverageCostPricing, MapScore, PayAllPaymentManager, + QueueWorkManager, ScoredAheadOfTimeAgreementManager, SequentialNegotiationManager, WorkContext, @@ -62,7 +62,9 @@ async def run_on_golem( activity_manager = ActivityPoolManager( golem, agreement_manager.get_agreement, size=threads, on_activity_start=init_func ) - work_manager = AsynchronousWorkManager(golem, activity_manager.do_work, plugins=task_plugins) + work_manager = QueueWorkManager( + golem, activity_manager.do_work, size=threads, plugins=task_plugins + ) async with golem, payment_manager, demand_manager, negotiation_manager, agreement_manager, activity_manager: # noqa: E501 line too long results: List[WorkResult] = await work_manager.do_work_list(task_list) diff --git a/golem/managers/__init__.py b/golem/managers/__init__.py index bd84cee2..1e6153eb 100644 --- a/golem/managers/__init__.py +++ b/golem/managers/__init__.py @@ -33,6 +33,7 @@ from golem.managers.network import SingleNetworkManager from golem.managers.payment import PayAllPaymentManager from golem.managers.work import ( + QueueWorkManager, SequentialWorkManager, WorkManagerPluginsMixin, redundancy_cancel_others_on_first_done, @@ -73,6 +74,7 @@ "SingleNetworkManager", "PayAllPaymentManager", "SequentialWorkManager", + "QueueWorkManager", "WorkManagerPluginsMixin", "redundancy_cancel_others_on_first_done", "retry", diff --git a/golem/managers/work/__init__.py b/golem/managers/work/__init__.py index 437c17a5..b8f2740d 100644 --- a/golem/managers/work/__init__.py +++ b/golem/managers/work/__init__.py @@ -1,10 +1,12 @@ from golem.managers.work.asynchronous import AsynchronousWorkManager from golem.managers.work.mixins import WorkManagerPluginsMixin from golem.managers.work.plugins import redundancy_cancel_others_on_first_done, retry, work_plugin +from golem.managers.work.queue import QueueWorkManager from golem.managers.work.sequential import SequentialWorkManager __all__ = ( "AsynchronousWorkManager", + "QueueWorkManager", "WorkManagerPluginsMixin", "SequentialWorkManager", "work_plugin", diff --git a/golem/managers/work/queue.py b/golem/managers/work/queue.py new file mode 100644 index 00000000..e79a1b42 --- /dev/null +++ b/golem/managers/work/queue.py @@ -0,0 +1,44 @@ +import asyncio +import logging +from typing import List + +from golem.managers.base import DoWorkCallable, Work, WorkManager, WorkResult +from golem.managers.work.mixins import WorkManagerPluginsMixin +from golem.node import GolemNode +from golem.utils.asyncio import create_task_with_logging +from golem.utils.logging import trace_span + +logger = logging.getLogger(__name__) + + +class QueueWorkManager(WorkManagerPluginsMixin, WorkManager): + def __init__(self, golem: GolemNode, do_work: DoWorkCallable, size: int, *args, **kwargs): + self._do_work = do_work + self._size = size + + self._queue = asyncio.Queue() + self._results = [] + + super().__init__(*args, **kwargs) + + @trace_span(show_arguments=True, show_results=True) + async def do_work(self, work: Work) -> WorkResult: + result = await self._do_work_with_plugins(self._do_work, work) + logger.info(f"Work `{work}` completed") + return result + + @trace_span(show_arguments=True, show_results=True) + async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: + workers = [create_task_with_logging(self.worker()) for _ in range(self._size)] + [self._queue.put_nowait(work) for work in work_list] + await self._queue.join() + [w.cancel() for w in workers] + await asyncio.gather(*workers, return_exceptions=True) + return self._results + + async def worker(self): + while True: + work = await self._queue.get() + result = await self.do_work(work) + self._queue.task_done() + self._results.append(result) From ecd510ff93bc591abf1222d66c9ac9e63e56f31b Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Thu, 27 Jul 2023 15:14:59 +0200 Subject: [PATCH 118/123] Simplify basic composition managers example --- examples/managers/basic_composition.py | 68 ++------------------------ 1 file changed, 5 insertions(+), 63 deletions(-) diff --git a/examples/managers/basic_composition.py b/examples/managers/basic_composition.py index 8d1d25a4..eed32dec 100644 --- a/examples/managers/basic_composition.py +++ b/examples/managers/basic_composition.py @@ -1,47 +1,23 @@ import asyncio import logging.config -from datetime import timedelta -from random import randint, random +from random import randint from typing import List from golem.managers import ( AddChosenPaymentPlatform, AutoDemandManager, - BlacklistProviderId, - LinearAverageCostPricing, - MapScore, PayAllPaymentManager, - PropertyValueLerpScore, - RandomScore, - RejectIfCostsExceeds, - RejectProposal, ScoredAheadOfTimeAgreementManager, SequentialNegotiationManager, SequentialWorkManager, SingleUseActivityManager, WorkContext, WorkResult, - redundancy_cancel_others_on_first_done, - retry, - work_plugin, ) from golem.node import GolemNode -from golem.payload import RepositoryVmPayload, defaults -from golem.resources import DemandData, ProposalData +from golem.payload import RepositoryVmPayload from golem.utils.logging import DEFAULT_LOGGING -BLACKLISTED_PROVIDERS = [ - "0x3b0f605fcb0690458064c10346af0c5f6b7202a5", - "0x7ad8ce2f95f69be197d136e308303d2395e68379", - "0x40f401ead13eabe677324bf50605c68caabb22c7", -] - - -async def blacklist_func(demand_data: DemandData, proposal_data: ProposalData) -> None: - provider_id = proposal_data.issuer_id - if provider_id in BLACKLISTED_PROVIDERS: - raise RejectProposal(f"Provider ID `{provider_id}` is blacklisted by the requestor") - async def commands_work_example(context: WorkContext) -> str: r = await context.run("echo 'hello golem'") @@ -52,7 +28,6 @@ async def commands_work_example(context: WorkContext) -> str: return result -@work_plugin(redundancy_cancel_others_on_first_done(size=2)) async def batch_work_example(context: WorkContext): if randint(0, 1): raise Exception("Random fail") @@ -77,53 +52,20 @@ async def main(): golem = GolemNode() - linear_average_cost = LinearAverageCostPricing( - average_cpu_load=0.2, average_duration=timedelta(seconds=5) - ) - payment_manager = PayAllPaymentManager(golem, budget=1.0) - demand_manager = AutoDemandManager( - golem, - payment_manager.get_allocation, - payload, - ) + demand_manager = AutoDemandManager(golem, payment_manager.get_allocation, payload) negotiation_manager = SequentialNegotiationManager( golem, demand_manager.get_initial_proposal, plugins=[ AddChosenPaymentPlatform(), - # class based plugin - BlacklistProviderId(BLACKLISTED_PROVIDERS), - RejectIfCostsExceeds(1, linear_average_cost), - # func plugin - blacklist_func, - # lambda plugin - lambda _, proposal_data: proposal_data.issuer_id not in BLACKLISTED_PROVIDERS, - # lambda plugin with reject reason - lambda _, proposal_data: RejectProposal(f"Blacklisting {proposal_data.issuer_id}") - if proposal_data.issuer_id in BLACKLISTED_PROVIDERS - else None, ], ) agreement_manager = ScoredAheadOfTimeAgreementManager( - golem, - negotiation_manager.get_draft_proposal, - plugins=[ - MapScore(linear_average_cost, normalize=True, normalize_flip=True), - [0.5, PropertyValueLerpScore(defaults.INF_MEM, zero_at=1, one_at=8)], - [0.1, RandomScore()], - [0.0, lambda proposals_data: [random() for _ in range(len(proposals_data))]], - [0.0, MapScore(lambda proposal_data: random())], - ], + golem, negotiation_manager.get_draft_proposal ) activity_manager = SingleUseActivityManager(golem, agreement_manager.get_agreement) - work_manager = SequentialWorkManager( - golem, - activity_manager.do_work, - plugins=[ - retry(tries=5), - ], - ) + work_manager = SequentialWorkManager(golem, activity_manager.do_work) async with golem: async with payment_manager, demand_manager, negotiation_manager, agreement_manager: From 969e3d2da31e2f12426e81c1693abd6ae99536ce Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Fri, 28 Jul 2023 09:53:50 +0200 Subject: [PATCH 119/123] Fix liniting --- examples/managers/ssh.py | 3 ++- golem/managers/work/queue.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/managers/ssh.py b/examples/managers/ssh.py index 0ae420ca..e5fd813c 100644 --- a/examples/managers/ssh.py +++ b/examples/managers/ssh.py @@ -40,10 +40,11 @@ async def on_activity_start(self, context: WorkContext): async def work(self, context: WorkContext) -> None: password = await self._run_ssh_server(context) + provider_id = context._activity.parent.parent.data.issuer_id print( "Connect with:\n" " ssh -o ProxyCommand='websocat asyncstdio: " - f"{await self._network_manager.get_provider_uri(context._activity.parent.parent.data.issuer_id, 'ws')}" + f"{await self._network_manager.get_provider_uri(provider_id, 'ws')}" f" --binary " f'-H=Authorization:"Bearer {self._app_key}"\' root@{uuid4().hex} ' ) diff --git a/golem/managers/work/queue.py b/golem/managers/work/queue.py index e79a1b42..99239eee 100644 --- a/golem/managers/work/queue.py +++ b/golem/managers/work/queue.py @@ -16,8 +16,8 @@ def __init__(self, golem: GolemNode, do_work: DoWorkCallable, size: int, *args, self._do_work = do_work self._size = size - self._queue = asyncio.Queue() - self._results = [] + self._queue: asyncio.Queue[Work] = asyncio.Queue() + self._results: List[WorkResult] = [] super().__init__(*args, **kwargs) @@ -30,7 +30,8 @@ async def do_work(self, work: Work) -> WorkResult: @trace_span(show_arguments=True, show_results=True) async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: workers = [create_task_with_logging(self.worker()) for _ in range(self._size)] - [self._queue.put_nowait(work) for work in work_list] + for work in work_list: + self._queue.put_nowait(work) await self._queue.join() [w.cancel() for w in workers] await asyncio.gather(*workers, return_exceptions=True) From 1d568ce6ce93cf3dc1034155077b3d5bfed498eb Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Fri, 28 Jul 2023 12:30:40 +0200 Subject: [PATCH 120/123] Restore plugins managers example --- examples/managers/plugins.py | 119 +++++++++++++++++++++++ golem/managers/activity/pool.py | 30 +++--- golem/managers/activity/single_use.py | 8 +- golem/managers/agreement/scored_aot.py | 2 +- golem/managers/negotiation/sequential.py | 2 +- golem/utils/logging.py | 3 + 6 files changed, 147 insertions(+), 17 deletions(-) create mode 100644 examples/managers/plugins.py diff --git a/examples/managers/plugins.py b/examples/managers/plugins.py new file mode 100644 index 00000000..6c3b25d6 --- /dev/null +++ b/examples/managers/plugins.py @@ -0,0 +1,119 @@ +import asyncio +import logging.config +from datetime import timedelta +from random import randint, random + +from golem.managers import ( + ActivityPoolManager, + AddChosenPaymentPlatform, + AutoDemandManager, + BlacklistProviderId, + LinearAverageCostPricing, + MapScore, + PayAllPaymentManager, + PropertyValueLerpScore, + RandomScore, + RejectIfCostsExceeds, + RejectProposal, + ScoredAheadOfTimeAgreementManager, + SequentialNegotiationManager, + SequentialWorkManager, + WorkContext, + WorkResult, + redundancy_cancel_others_on_first_done, + retry, + work_plugin, +) +from golem.node import GolemNode +from golem.payload import RepositoryVmPayload, defaults +from golem.resources import DemandData, ProposalData +from golem.utils.logging import DEFAULT_LOGGING + +BLACKLISTED_PROVIDERS = [ + "0x3b0f605fcb0690458064c10346af0c5f6b7202a5", + "0x7ad8ce2f95f69be197d136e308303d2395e68379", + "0x40f401ead13eabe677324bf50605c68caabb22c7", +] + + +async def blacklist_func(demand_data: DemandData, proposal_data: ProposalData) -> None: + provider_id = proposal_data.issuer_id + if provider_id in BLACKLISTED_PROVIDERS: + raise RejectProposal(f"Provider ID `{provider_id}` is blacklisted by the requestor") + + +@work_plugin(redundancy_cancel_others_on_first_done(size=2)) +async def commands_work_example(context: WorkContext) -> str: + if randint(0, 1): + raise Exception("Random fail") + r = await context.run("echo 'hello golem'") + await r.wait() + result = "" + for event in r.events: + result += event.stdout + return result + + +async def main(): + logging.config.dictConfig(DEFAULT_LOGGING) + payload = RepositoryVmPayload("9a3b5d67b0b27746283cb5f287c13eab1beaa12d92a9f536b747c7ae") + + golem = GolemNode() + + linear_average_cost = LinearAverageCostPricing( + average_cpu_load=0.2, average_duration=timedelta(seconds=5) + ) + + payment_manager = PayAllPaymentManager(golem, budget=1.0) + demand_manager = AutoDemandManager( + golem, + payment_manager.get_allocation, + payload, + ) + negotiation_manager = SequentialNegotiationManager( + golem, + demand_manager.get_initial_proposal, + plugins=[ + AddChosenPaymentPlatform(), + # class based plugin + BlacklistProviderId(BLACKLISTED_PROVIDERS), + RejectIfCostsExceeds(1, linear_average_cost), + # func plugin + blacklist_func, + # lambda plugin + lambda _, proposal_data: proposal_data.issuer_id not in BLACKLISTED_PROVIDERS, + # lambda plugin with reject reason + lambda _, proposal_data: RejectProposal(f"Blacklisting {proposal_data.issuer_id}") + if proposal_data.issuer_id in BLACKLISTED_PROVIDERS + else None, + ], + ) + agreement_manager = ScoredAheadOfTimeAgreementManager( + golem, + negotiation_manager.get_draft_proposal, + plugins=[ + MapScore(linear_average_cost, normalize=True, normalize_flip=True), + [0.5, PropertyValueLerpScore(defaults.INF_MEM, zero_at=1, one_at=8)], + [0.1, RandomScore()], + [0.0, lambda proposals_data: [random() for _ in range(len(proposals_data))]], + [0.0, MapScore(lambda proposal_data: random())], + ], + ) + activity_manager = ActivityPoolManager(golem, agreement_manager.get_agreement, size=3) + work_manager = SequentialWorkManager( + golem, + activity_manager.do_work, + plugins=[ + retry(tries=5), + ], + ) + + async with golem: + async with payment_manager, demand_manager, negotiation_manager, agreement_manager, activity_manager: # noqa: E501 line too long + await asyncio.sleep(10) + result: WorkResult = await work_manager.do_work(commands_work_example) + print(f"\nWORK MANAGER RESULT:{result}\n") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/golem/managers/activity/pool.py b/golem/managers/activity/pool.py index a1b9e115..91a3008a 100644 --- a/golem/managers/activity/pool.py +++ b/golem/managers/activity/pool.py @@ -26,43 +26,46 @@ def __init__( self._event_bus = golem.event_bus self._pool_target_size = size + self._pool_current_size = 0 self._pool: asyncio.Queue[Activity] = asyncio.Queue() super().__init__(*args, **kwargs) async def _background_loop(self): - pool_current_size = 0 try: while True: - if pool_current_size > self._pool_target_size: + if self._pool_current_size > self._pool_target_size: # TODO check tasks results and add fallback await asyncio.gather( *[ self._release_activity_and_pop_from_pool() - for _ in range(pool_current_size - self._pool_target_size) + for _ in range(self._pool_current_size - self._pool_target_size) ] ) - pool_current_size -= pool_current_size - self._pool_target_size - elif pool_current_size < self._pool_target_size: + elif self._pool_current_size < self._pool_target_size: # TODO check tasks results and add fallback await asyncio.gather( *[ self._prepare_activity_and_put_in_pool() - for _ in range(self._pool_target_size - pool_current_size) + for _ in range(self._pool_target_size - self._pool_current_size) ] ) - pool_current_size += self._pool_target_size - pool_current_size # TODO: Use events instead of sleep await asyncio.sleep(0.01) finally: - logger.info(f"Releasing all {pool_current_size} activity from the pool") + logger.info(f"Releasing all {self._pool_current_size} activity from the pool") + # TODO cancel release adn prepare tasks await asyncio.gather( - *[self._release_activity_and_pop_from_pool() for _ in range(pool_current_size)] + *[ + self._release_activity_and_pop_from_pool() + for _ in range(self._pool_current_size) + ] ) @trace_span() async def _release_activity_and_pop_from_pool(self): activity = await self._pool.get() self._pool.task_done() + self._pool_current_size -= 1 await self._release_activity(activity) @trace_span() @@ -70,15 +73,18 @@ async def _prepare_activity_and_put_in_pool(self): agreement = await self._get_agreement() activity = await self._prepare_activity(agreement) await self._pool.put(activity) + self._pool_current_size += 1 @asynccontextmanager async def _get_activity_from_pool(self): activity = await self._pool.get() self._pool.task_done() logger.debug(f"Activity `{activity}` taken from the pool") - yield activity - self._pool.put_nowait(activity) - logger.debug(f"Activity `{activity}` back in the pool") + try: + yield activity + finally: + await self._pool.put(activity) + logger.debug(f"Activity `{activity}` back in the pool") @trace_span(show_arguments=True, show_results=True) async def do_work(self, work: Work) -> WorkResult: diff --git a/golem/managers/activity/single_use.py b/golem/managers/activity/single_use.py index 572187a6..bb49dbe1 100644 --- a/golem/managers/activity/single_use.py +++ b/golem/managers/activity/single_use.py @@ -27,9 +27,11 @@ async def _prepare_single_use_activity(self) -> AsyncGenerator[Activity, None]: try: activity = await self._prepare_activity(agreement) logger.info(f"Activity `{activity}` created") - yield activity - await self._release_activity(activity) - break + try: + yield activity + finally: + await self._release_activity(activity) + break except Exception: logger.exception("Creating activity failed, but will be retried with new agreement") diff --git a/golem/managers/agreement/scored_aot.py b/golem/managers/agreement/scored_aot.py index b160ac3d..8ae39d99 100644 --- a/golem/managers/agreement/scored_aot.py +++ b/golem/managers/agreement/scored_aot.py @@ -18,7 +18,7 @@ def __init__( self, golem: GolemNode, get_draft_proposal: Callable[[], Awaitable[Proposal]], - buffer_size: Tuple[int, int] = (0, 1), + buffer_size: Tuple[int, int] = (1, 3), *args, **kwargs, ): diff --git a/golem/managers/negotiation/sequential.py b/golem/managers/negotiation/sequential.py index 384cae1b..88ae17de 100644 --- a/golem/managers/negotiation/sequential.py +++ b/golem/managers/negotiation/sequential.py @@ -42,7 +42,7 @@ async def get_draft_proposal(self) -> Proposal: @trace_span() async def _background_loop(self) -> None: - while True: # TODO add buffer + while True: proposal = await self._get_initial_proposal() demand_data = await self._get_demand_data_from_proposal(proposal) diff --git a/golem/utils/logging.py b/golem/utils/logging.py index bbddbc75..49cb7de6 100644 --- a/golem/utils/logging.py +++ b/golem/utils/logging.py @@ -35,6 +35,9 @@ "golem": { "level": "INFO", }, + "golem.utils": { + "level": "INFO", + }, "golem.managers": { "level": "INFO", }, From 3b35ea719e605b563b261c155c484a7de3f77d6f Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Fri, 28 Jul 2023 13:44:39 +0200 Subject: [PATCH 121/123] CR changes --- .gitignore | 2 -- golem/event_bus/in_memory/event_bus.py | 50 +++++--------------------- 2 files changed, 9 insertions(+), 43 deletions(-) diff --git a/.gitignore b/.gitignore index 3701131b..007173a5 100644 --- a/.gitignore +++ b/.gitignore @@ -166,5 +166,3 @@ build/ # licheck artifacts .requirements.txt - -temp/ diff --git a/golem/event_bus/in_memory/event_bus.py b/golem/event_bus/in_memory/event_bus.py index a5a81655..5a6c4c13 100644 --- a/golem/event_bus/in_memory/event_bus.py +++ b/golem/event_bus/in_memory/event_bus.py @@ -6,6 +6,7 @@ from golem.event_bus.base import Event, EventBus, EventBusError, TEvent from golem.utils.asyncio import create_task_with_logging +from golem.utils.logging import trace_span logger = logging.getLogger(__name__) @@ -26,9 +27,8 @@ def __init__(self): self._event_queue: asyncio.Queue[Event] = asyncio.Queue() self._process_event_queue_loop_task: Optional[asyncio.Task] = None + @trace_span() async def start(self): - logger.debug("Starting event bus...") - if self.is_started(): message = "Event bus is already started!" logger.debug(f"Starting event bus failed with `{message}`") @@ -38,11 +38,8 @@ async def start(self): self._process_event_queue_loop() ) - logger.debug("Starting event bus done") - + @trace_span() async def stop(self): - logger.debug("Stopping event bus...") - if not self.is_started(): message = "Event bus is not started!" logger.debug(f"Stopping event bus failed with `{message}`") @@ -54,25 +51,20 @@ async def stop(self): self._process_event_queue_loop_task.cancel() self._process_event_queue_loop_task = None - logger.debug("Stopping event bus done") - + @trace_span(show_results=True) def is_started(self) -> bool: return ( self._process_event_queue_loop_task is not None and not self._process_event_queue_loop_task.done() ) + @trace_span(show_arguments=True) async def on( self, event_type: Type[TEvent], callback: Callable[[TEvent], Awaitable[None]], filter_func: Optional[Callable[[TEvent], bool]] = None, ) -> _CallbackHandler: - logger.debug( - f"Adding callback handler for `{event_type}` with callback `{callback}`" - f" and filter `{filter_func}`..." - ) - callback_info = _CallbackInfo( callback=callback, filter_func=filter_func, @@ -83,21 +75,15 @@ async def on( callback_handler = (event_type, callback_info) - logger.debug(f"Adding callback handler done with `{id(callback_handler)}`") - return callback_handler + @trace_span(show_arguments=True) async def on_once( self, event_type: Type[TEvent], callback: Callable[[TEvent], Awaitable[None]], filter_func: Optional[Callable[[TEvent], bool]] = None, ) -> _CallbackHandler: - logger.debug( - f"Adding one-time callback handler for `{event_type}` with callback `{callback}`" - f" and filter `{filter_func}`..." - ) - callback_info = _CallbackInfo( callback=callback, filter_func=filter_func, @@ -108,15 +94,11 @@ async def on_once( callback_handler = (event_type, callback_info) - logger.debug(f"Adding one-time callback handler done with `{id(callback_handler)}`") - return callback_handler + @trace_span(show_arguments=True) async def off(self, callback_handler: _CallbackHandler) -> None: - logger.debug(f"Removing callback handler `{id(callback_handler)}`...") - event_type, callback_info = callback_handler - try: self._callbacks[event_type].remove(callback_info) except (KeyError, ValueError): @@ -126,11 +108,8 @@ async def off(self, callback_handler: _CallbackHandler) -> None: ) raise EventBusError(message) - logger.debug(f"Removing callback handler `{id(callback_handler)}` done") - + @trace_span(show_arguments=True) async def emit(self, event: TEvent) -> None: - logger.debug(f"Emitting event `{event}`...") - if not self.is_started(): message = "Event bus is not started!" logger.debug(f"Emitting event `{event}` failed with `message`") @@ -138,30 +117,21 @@ async def emit(self, event: TEvent) -> None: await self._event_queue.put(event) - logger.debug(f"Emitting event `{event}` done") - async def _process_event_queue_loop(self): while True: logger.debug("Getting event from queue...") - event = await self._event_queue.get() - logger.debug(f"Getting event from queue done with `{event}`") - logger.debug(f"Processing callbacks for event `{event}`...") - for event_type, callback_infos in self._callbacks.items(): await self._process_event(event, event_type, callback_infos) - logger.debug(f"Processing callbacks for event `{event}` done") - self._event_queue.task_done() + @trace_span(show_arguments=True) async def _process_event( self, event: Event, event_type: Type[Event], callback_infos: List[_CallbackInfo] ): - logger.debug(f"Processing event `{event}` on event type `{event_type}`...") - if not isinstance(event, event_type): logger.debug( f"Processing event `{event}` on event type `{event_type}` ignored as event is" @@ -222,5 +192,3 @@ async def _process_event( callback_infos.remove(callback_info) logger.debug(f"Removing callbacks `{callback_infos_to_remove}` done") - - logger.debug(f"Processing event `{event}` on event type `{event_type}` done") From cc3e6c91e2f2987bb1f654f244191252f421c2b0 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Fri, 28 Jul 2023 14:00:42 +0200 Subject: [PATCH 122/123] Remove AsynchronousWorkManager --- golem/managers/__init__.py | 2 -- golem/managers/work/__init__.py | 2 -- golem/managers/work/asynchronous.py | 28 ---------------------------- 3 files changed, 32 deletions(-) delete mode 100644 golem/managers/work/asynchronous.py diff --git a/golem/managers/__init__.py b/golem/managers/__init__.py index 1e6153eb..996db836 100644 --- a/golem/managers/__init__.py +++ b/golem/managers/__init__.py @@ -40,7 +40,6 @@ retry, work_plugin, ) -from golem.managers.work.asynchronous import AsynchronousWorkManager __all__ = ( "ActivityPoolManager", @@ -79,5 +78,4 @@ "redundancy_cancel_others_on_first_done", "retry", "work_plugin", - "AsynchronousWorkManager", ) diff --git a/golem/managers/work/__init__.py b/golem/managers/work/__init__.py index b8f2740d..1b22a322 100644 --- a/golem/managers/work/__init__.py +++ b/golem/managers/work/__init__.py @@ -1,11 +1,9 @@ -from golem.managers.work.asynchronous import AsynchronousWorkManager from golem.managers.work.mixins import WorkManagerPluginsMixin from golem.managers.work.plugins import redundancy_cancel_others_on_first_done, retry, work_plugin from golem.managers.work.queue import QueueWorkManager from golem.managers.work.sequential import SequentialWorkManager __all__ = ( - "AsynchronousWorkManager", "QueueWorkManager", "WorkManagerPluginsMixin", "SequentialWorkManager", diff --git a/golem/managers/work/asynchronous.py b/golem/managers/work/asynchronous.py deleted file mode 100644 index 5713ead2..00000000 --- a/golem/managers/work/asynchronous.py +++ /dev/null @@ -1,28 +0,0 @@ -import asyncio -import logging -from typing import List - -from golem.managers.base import DoWorkCallable, Work, WorkManager, WorkResult -from golem.managers.work.mixins import WorkManagerPluginsMixin -from golem.node import GolemNode -from golem.utils.logging import trace_span - -logger = logging.getLogger(__name__) - - -class AsynchronousWorkManager(WorkManagerPluginsMixin, WorkManager): - def __init__(self, golem: GolemNode, do_work: DoWorkCallable, *args, **kwargs): - self._do_work = do_work - - super().__init__(*args, **kwargs) - - @trace_span(show_arguments=True, show_results=True) - async def do_work(self, work: Work) -> WorkResult: - result = await self._do_work_with_plugins(self._do_work, work) - logger.info(f"Work `{work}` completed") - return result - - @trace_span(show_arguments=True, show_results=True) - async def do_work_list(self, work_list: List[Work]) -> List[WorkResult]: - results = await asyncio.gather(*[self.do_work(work) for work in work_list]) - return results From 9a11a77754dca15a1a1e639982a45dcc4ae498c5 Mon Sep 17 00:00:00 2001 From: Lucjan Dudek Date: Fri, 28 Jul 2023 14:33:56 +0200 Subject: [PATCH 123/123] Rename queue work manager to concurrent --- examples/managers/blender/blender.py | 4 ++-- golem/managers/__init__.py | 4 ++-- golem/managers/work/__init__.py | 4 ++-- golem/managers/work/{queue.py => concurrent.py} | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) rename golem/managers/work/{queue.py => concurrent.py} (95%) diff --git a/examples/managers/blender/blender.py b/examples/managers/blender/blender.py index 897b534b..8a7ee77c 100644 --- a/examples/managers/blender/blender.py +++ b/examples/managers/blender/blender.py @@ -9,10 +9,10 @@ ActivityPoolManager, AddChosenPaymentPlatform, AutoDemandManager, + ConcurrentWorkManager, LinearAverageCostPricing, MapScore, PayAllPaymentManager, - QueueWorkManager, ScoredAheadOfTimeAgreementManager, SequentialNegotiationManager, WorkContext, @@ -62,7 +62,7 @@ async def run_on_golem( activity_manager = ActivityPoolManager( golem, agreement_manager.get_agreement, size=threads, on_activity_start=init_func ) - work_manager = QueueWorkManager( + work_manager = ConcurrentWorkManager( golem, activity_manager.do_work, size=threads, plugins=task_plugins ) diff --git a/golem/managers/__init__.py b/golem/managers/__init__.py index 996db836..68834175 100644 --- a/golem/managers/__init__.py +++ b/golem/managers/__init__.py @@ -33,7 +33,7 @@ from golem.managers.network import SingleNetworkManager from golem.managers.payment import PayAllPaymentManager from golem.managers.work import ( - QueueWorkManager, + ConcurrentWorkManager, SequentialWorkManager, WorkManagerPluginsMixin, redundancy_cancel_others_on_first_done, @@ -73,7 +73,7 @@ "SingleNetworkManager", "PayAllPaymentManager", "SequentialWorkManager", - "QueueWorkManager", + "ConcurrentWorkManager", "WorkManagerPluginsMixin", "redundancy_cancel_others_on_first_done", "retry", diff --git a/golem/managers/work/__init__.py b/golem/managers/work/__init__.py index 1b22a322..cc514c83 100644 --- a/golem/managers/work/__init__.py +++ b/golem/managers/work/__init__.py @@ -1,10 +1,10 @@ +from golem.managers.work.concurrent import ConcurrentWorkManager from golem.managers.work.mixins import WorkManagerPluginsMixin from golem.managers.work.plugins import redundancy_cancel_others_on_first_done, retry, work_plugin -from golem.managers.work.queue import QueueWorkManager from golem.managers.work.sequential import SequentialWorkManager __all__ = ( - "QueueWorkManager", + "ConcurrentWorkManager", "WorkManagerPluginsMixin", "SequentialWorkManager", "work_plugin", diff --git a/golem/managers/work/queue.py b/golem/managers/work/concurrent.py similarity index 95% rename from golem/managers/work/queue.py rename to golem/managers/work/concurrent.py index 99239eee..46c0d93c 100644 --- a/golem/managers/work/queue.py +++ b/golem/managers/work/concurrent.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) -class QueueWorkManager(WorkManagerPluginsMixin, WorkManager): +class ConcurrentWorkManager(WorkManagerPluginsMixin, WorkManager): def __init__(self, golem: GolemNode, do_work: DoWorkCallable, size: int, *args, **kwargs): self._do_work = do_work self._size = size