Set up monorepo:

this monorepo is organized as a PEP 420 compliant namespace
(https://www.python.org/dev/peps/pep-0420/).

The sdk package has been restructured to conform to the PEP 420
namespace package specification and it's under the "snet" namespace.

The cli package is both a regular package and a namespace package.

setup.py for the monorepo itself installs the dependencies for all
packages in the repository.

Each package and the overarching monorepo also have "requirements.txt"
files for local installations and CI enviroments: by installing via
"requirements.txt", the dependencies listed in "setup.py" are only
pulled and installed from PyPI if a local version is not already
available. This allows to get dependencies within the monorepo to be
installed from the local filesystem.

The whole package is called "snet" and gives the name to the namespace.
The subpackages are "snet.sdk", "snet.snet_cli" and snet_cli (since the
CLI package is both within the namespace and a regular package).
Going forward, all of the code in the "snet_cli" package can be migrated
to the namespace package: both were kept momentarily for compatibility.

In this scenario, everything inside the "snet.sdk" package has
visibility on the contents of the "snet.snet_cli" package and
vice-versa.
Everything inside the "snet_cli" regular package has visibility on the
contents of the "snet.snet_cli" package due to the regular Python import
mechanics, but it has no visibility on the contents of the "snet.sdk"
package (and vice-versa).
So, the shared code between the SDK and the CLI goes under the
"snet.snet_cli" package which is at "packages/snet_cli/snet/snet_cli".

Code and assets deduplication:

thanks to the new structure, everything under "resources" has been moved
to the "snet.snet_cli" package and is now shared between the SDK and
CLI. Specifically, this includes the json artifacts from the compilation
and deployment of the smart contracts, the raw ".proto" files to
interact with the daemon and the compiled python client libraries.

Changes:

since the CLI is both a namespace package and a regular package, changes
to the code have been kept to a minimum.
The only change of note is that state service does not compile its own
".proto" files at runtime inside the user's home directory anymore but
they are imported from the "resources" directory under the
"snet.snet_cli" namespace package, where they are compiled during setup.

.circleci/config.yml

@@ -21,13 +21,13 @@ jobs:
     - run:
         name: Install snet-cli 
         command: |
-            ./snet_cli/scripts/blockchain install
-            sudo pip3 install -e ./snet_cli
+            ./packages/snet_cli/scripts/blockchain install
+            sudo pip3 install -e ./packages/snet_cli
     - run:
         name: Install platform-contracts from master
         command: |
             # Install platform-contracts (from master)
-            # we will deploy contracts from snet_cli/test/utils/reset_enviroment.sh
+            # we will deploy contracts from packages/snet_cli/test/utils/reset_enviroment.sh
             cd ..
             git clone https://github.com/singnet/platform-contracts
             cd platform-contracts
@@ -42,7 +42,7 @@ jobs:
     - run:
         name: Functional tests
         command: |
-            bash -ex snet_cli/test/utils/run_all_functional.sh
+            bash -ex packages/snet_cli/test/utils/run_all_functional.sh
     - run:
         name: Trigger platform-pipeline build
         command: |

.gitignore

@@ -1,3 +1,5 @@
+venv/
 .idea/
 __pycache__
 blockchain/node_modules
+snet.egg-info/

packages/sdk/.gitignore

@@ -0,0 +1,4 @@
+venv/
+snet.sdk.egg-info/
+build/
+dist/

packages/sdk/requirements.txt

@@ -0,0 +1,3 @@
+--index-url https://pypi.python.org/simple
+../snet_cli/
+-e .

packages/sdk/setup.py

@@ -0,0 +1,43 @@
+import pkg_resources
+from setuptools import setup, find_namespace_packages
+
+
+PACKAGE_NAME = 'snet.sdk'
+
+
+def is_package_installed(package_name):
+    installed_modules = [p.project_name for p in pkg_resources.working_set] 
+    return package_name in installed_modules
+
+
+dependencies = []
+
+
+if is_package_installed('snet-cli'):
+    # The default setup.py in the snet_cli package for local development installs the whole snet_cli package, 
+    # not the standalone snet.snet_cli namespace package; if a strict dependency on snet.snet_cli was enforced,
+    # this setup.py would fetch it from PyPI. So, if snet_cli is installed and in your Python path, the
+    # dependency on snet.snet_cli will be skipped.
+    # If snet_cli is not available, snet.snet_cli will be fetched from PyPI.
+    print("Package 'snet_cli' is installed and in your PYTHONPATH: skipping snet.snet_cli dependency")
+else:
+    dependencies.append('snet.snet_cli')
+
+
+version_dict = {}
+with open("./snet/sdk/version.py") as fp:
+    exec(fp.read(), version_dict)
+
+setup(
+    name=PACKAGE_NAME,
+    version=version_dict['__version__'],
+    packages=find_namespace_packages(include=['snet.*']),
+    namespace_packages=['snet'],
+    url='https://github.com/singnet/snet-cli/tree/master/snet_sdk',
+    license='MIT',
+    author='SingularityNET Foundation',
+    author_email='[email protected]',
+    description='SingularityNET Python SDK',
+    python_requires='>=3.6',
+    install_requires=dependencies
+)

snet_sdk/snet_sdk/__init__.py → packages/sdk/snet/sdk/__init__.py

@@ -6,13 +6,14 @@
 from web3.gas_strategies.rpc import rpc_gas_price_strategy
 from rfc3986 import urlparse
 import ipfsapi
-from web3.utils.datastructures import AttributeDict
+from web3.datastructures import AttributeDict
 
-from snet_sdk.utils import get_contract_object
-from snet_sdk.service_client import ServiceClient
-from snet_sdk.account import Account
-from snet_sdk.mpe_contract import MPEContract
-from snet_sdk.payment_channel_management_strategies.default import PaymentChannelManagementStrategy
+from snet.sdk.service_client import ServiceClient
+from snet.sdk.account import Account
+from snet.sdk.mpe_contract import MPEContract
+from snet.sdk.payment_channel_management_strategies.default import PaymentChannelManagementStrategy
+
+from snet.snet_cli.utils import get_contract_object
 
 
 class SnetSDK:

snet_sdk/snet_sdk/account.py → packages/sdk/snet/sdk/account.py

@@ -1,6 +1,6 @@
 import json
 
-from snet_sdk.utils import normalize_private_key, get_address_from_private, get_contract_object
+from snet.snet_cli.utils import normalize_private_key, get_address_from_private, get_contract_object
 
 DEFAULT_GAS = 210000
 

snet_sdk/snet_sdk/mpe_contract.py → packages/sdk/snet/sdk/mpe_contract.py

@@ -1,8 +1,10 @@
 import base64
 
 import web3
-from snet_sdk.utils import get_contract_object, get_contract_deployment_block 
-from snet_sdk.payment_channel import PaymentChannel
+from snet.sdk.payment_channel import PaymentChannel
+
+from snet.snet_cli.utils import get_contract_object, get_contract_deployment_block 
+
 
 BLOCKS_PER_BATCH = 20000
 EVENT_ABI = {'anonymous': False, 'inputs': [{'indexed': False, 'name': 'channelId', 'type': 'uint256'}, {'indexed': False, 'name': 'nonce', 'type': 'uint256'}, {'indexed': True, 'name': 'sender', 'type': 'address'}, {'indexed': False, 'name': 'signer', 'type': 'address'}, {'indexed': True, 'name': 'recipient', 'type': 'address'}, {'indexed': True, 'name': 'groupId', 'type': 'bytes32'}, {'indexed': False, 'name': 'amount', 'type': 'uint256'}, {'indexed': False, 'name': 'expiration', 'type': 'uint256'}], 'name': 'ChannelOpen', 'type': 'event'} 

snet_sdk/snet_sdk/payment_channel.py → packages/sdk/snet/sdk/payment_channel.py

@@ -1,7 +1,8 @@
 import web3
+import importlib
 from eth_account.messages import defunct_hash_message
 
-import snet_sdk.state_service_pb2 as state_service_pb2
+from snet.snet_cli.utils import RESOURCES_PATH, add_to_path
 
 class PaymentChannel:
     def __init__(self, channel_id, w3, account, service, mpe_contract):
@@ -49,6 +50,8 @@ def _get_current_channel_state(self):
         stub = self.payment_channel_state_service_client
         message = web3.Web3.soliditySha3(["uint256"], [self.channel_id])
         signature = self.web3.eth.account.signHash(defunct_hash_message(message), self.account.signer_private_key).signature
+        with add_to_path(str(RESOURCES_PATH.joinpath("proto"))):
+            state_service_pb2 = importlib.import_module("state_service_pb2")
         request = state_service_pb2.ChannelStateRequest(channel_id=web3.Web3.toBytes(self.channel_id), signature=bytes(signature))
         response = stub.GetChannelState(request)
         return int.from_bytes(response.current_nonce, byteorder="big"), int.from_bytes(response.current_signed_amount, byteorder="big")

snet_sdk/snet_sdk/service_client.py → packages/sdk/snet/sdk/service_client.py

@@ -1,12 +1,13 @@
 import collections
+import importlib
+
 import grpc
 import web3
 from rfc3986 import urlparse
 from eth_account.messages import defunct_hash_message
 
-import snet_sdk.generic_client_interceptor as generic_client_interceptor
-
-import snet_sdk.state_service_pb2_grpc as state_service_pb2_grpc
+import snet.sdk.generic_client_interceptor as generic_client_interceptor
+from snet.snet_cli.utils import RESOURCES_PATH, add_to_path
 
 
 class _ClientCallDetails(
@@ -111,6 +112,8 @@ def default_channel_expiration(self):
 
     def _generate_payment_channel_state_service_client(self):
         grpc_channel = self._base_grpc_channel
+        with add_to_path(str(RESOURCES_PATH.joinpath("proto"))):
+            state_service_pb2_grpc = importlib.import_module("state_service_pb2_grpc")
         return state_service_pb2_grpc.PaymentChannelStateServiceStub(grpc_channel)
 
 

packages/sdk/snet/sdk/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.4"

packages/snet_cli/.gitignore

@@ -0,0 +1,8 @@
+venv/
+snet_cli.egg-info/
+snet/snet_cli/resources/contracts/abi
+snet/snet_cli/resources/contracts/networks
+snet/snet_cli/resources/node_modules
+snet/snet_cli/resources/proto/*.py
+build/
+dist/

packages/snet_cli/MANIFEST.in

@@ -0,0 +1,11 @@
+include snet/snet_cli/resources/proto/*
+include snet/snet_cli/resources/contracts/abi/*
+include snet/snet_cli/resources/contracts/networks/*
+include snet/snet_cli/resources/package.json
+
+recursive-include snet_cli/_vendor *
+
+recursive-exclude * .git
+recursive-exclude * node_modules
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]

packages/snet_cli/requirements.txt

@@ -0,0 +1,2 @@
+--index-url https://pypi.python.org/simple
+-e .

snet_sdk/scripts/blockchain → packages/snet_cli/scripts/blockchain

@@ -10,12 +10,13 @@ import sys
 def main():
     assert len(sys.argv) > 1, "please select a target from 'install', 'uninstall'"
     target = sys.argv[1]
-    blockchain_dir = pathlib.Path(__file__).absolute().parent.parent.parent.joinpath("blockchain")
+    cur_dir = pathlib.Path(__file__).absolute().parent
+    blockchain_dir = cur_dir.parent.parent.parent.joinpath("blockchain")
     node_modules_dir = blockchain_dir.joinpath("node_modules")
     platform_json_src_dir = node_modules_dir.joinpath("singularitynet-platform-contracts")
     token_json_src_dir = node_modules_dir.joinpath("singularitynet-token-contracts")
     token_contract_name = "SingularityNetToken"
-    contract_json_dest_dir = pathlib.Path(__file__).absolute().parent.parent.joinpath("snet_sdk", "resources", "contracts")
+    contract_json_dest_dir = cur_dir.parent.joinpath("snet", "snet_cli", "resources", "contracts")
     abi_contract_names = ["Registry", "MultiPartyEscrow"]
     networks_contract_names = ["Registry", "MultiPartyEscrow"]
 

snet_cli/setup.py → packages/snet_cli/setup.py

@@ -1,14 +1,14 @@
-from setuptools import setup, find_packages
+from setuptools import setup, find_packages, find_namespace_packages
 from setuptools.command.develop import develop as _develop
 from setuptools.command.install import install as _install
 
 
 def install_and_compile_proto():
-    import snet_cli
-    from snet_cli.utils import compile_proto as compile_proto
+    import snet.snet_cli
+    from snet.snet_cli.utils import compile_proto as compile_proto
     from pathlib import Path
-    proto_dir = Path(__file__).absolute().parent.joinpath("snet_cli", "resources", "proto")
-    dest_dir = Path(snet_cli.__file__).absolute().parent.joinpath("resources", "proto")
+    proto_dir = Path(__file__).absolute().parent.joinpath("snet", "snet_cli", "resources", "proto")
+    dest_dir = Path(snet.snet_cli.__file__).absolute().parent.joinpath("resources", "proto")
     print(proto_dir, "->", dest_dir)
     for fn in proto_dir.glob('*.proto'):
         print("Compiling protobuf", fn)
@@ -37,7 +37,8 @@ def run(self):
 setup(
     name='snet-cli',
     version=version_dict['__version__'],
-    packages=find_packages(),
+    packages=find_namespace_packages(include=['snet.*'])+find_packages(),
+    namespace_packages=['snet'],
     url='https://github.com/singnet/snet-cli',
     license='MIT',
     author='SingularityNET Foundation',

snet_cli/snet_cli/utils.py → packages/snet_cli/snet/snet_cli/utils.py

@@ -2,16 +2,20 @@
 import os
 import subprocess
 import functools
+import sys
 
 from urllib.parse import urlparse
-from pathlib import Path
+from pathlib import Path, PurePath
 
 import web3
 import pkg_resources
 import grpc
 from grpc_tools.protoc import main as protoc
 
 
+RESOURCES_PATH = PurePath(os.path.realpath(__file__)).parent.joinpath("resources")
+
+
 class DefaultAttributeObject(object):
     def __init__(self, **kwargs):
         for k, v in kwargs.items():
@@ -122,7 +126,7 @@ def walk_imports(entry_path):
     return seen_paths
 
 
-def get_contract_def(contract_name, contract_artifacts_root=Path(__file__).absolute().parent.joinpath("resources", "contracts")):
+def get_contract_def(contract_name, contract_artifacts_root=RESOURCES_PATH.joinpath("contracts")):
     contract_def = {}
     with open(Path(__file__).absolute().parent.joinpath(contract_artifacts_root, "abi", "{}.json".format(contract_name))) as f:
         contract_def["abi"] = json.load(f)
@@ -159,12 +163,11 @@ def compile_proto(entry_path, codegen_dir, proto_file=None, target_language="pyt
             compiler_args.append("--grpc_python_out={}".format(codegen_dir)) 
             compiler = protoc
         elif target_language == "nodejs":
-            resources_path = Path(os.path.dirname(os.path.realpath(__file__))).joinpath("resources")
-            protoc_node_compiler_path = resources_path.joinpath("node_modules").joinpath("grpc-tools").joinpath("bin").joinpath("protoc.js").absolute()
-            grpc_node_plugin_path = resources_path.joinpath("node_modules").joinpath("grpc-tools").joinpath("bin").joinpath("grpc_node_plugin").resolve()
+            protoc_node_compiler_path = Path(RESOURCES_PATH.joinpath("node_modules").joinpath("grpc-tools").joinpath("bin").joinpath("protoc.js")).absolute()
+            grpc_node_plugin_path = Path(RESOURCES_PATH.joinpath("node_modules").joinpath("grpc-tools").joinpath("bin").joinpath("grpc_node_plugin")).resolve()
             if not os.path.isfile(protoc_node_compiler_path) or not os.path.isfile(grpc_node_plugin_path):
                 print("Missing required node.js protoc compiler. Retrieving from npm...")
-                subprocess.run(["npm", "install"], cwd=resources_path)
+                subprocess.run(["npm", "install"], cwd=RESOURCES_PATH)
             compiler_args.append("--js_out=import_style=commonjs,binary:{}".format(codegen_dir))
             compiler_args.append("--grpc_out={}".format(codegen_dir))
             compiler_args.append("--plugin=protoc-gen-grpc={}".format(grpc_node_plugin_path))
@@ -181,6 +184,7 @@ def compile_proto(entry_path, codegen_dir, proto_file=None, target_language="pyt
             return False
 
     except Exception as e:
+        print(e)
         return False
 
 def abi_get_element_by_name(abi, name):
@@ -253,3 +257,43 @@ def rgetattr(obj, attr):
     2
     """
     return functools.reduce(getattr, [obj] + attr.split('.'))
+
+
+def get_contract_object(w3, contract_file):
+    with open(RESOURCES_PATH.joinpath("contracts", "abi", contract_file)) as f:
+        abi = json.load(f)
+    with open(RESOURCES_PATH.joinpath("contracts", "networks", contract_file)) as f:
+        networks = json.load(f)
+        address = w3.toChecksumAddress(networks[w3.version.network]["address"])
+    return w3.eth.contract(abi=abi, address=address)
+
+
+def get_contract_deployment_block(w3, contract_file):
+    with open(RESOURCES_PATH.joinpath("contracts", "networks", contract_file)) as f:
+        networks = json.load(f)
+        txn_hash = networks[w3.version.network]["transactionHash"]
+    return w3.eth.getTransactionReceipt(txn_hash).blockNumber
+
+
+def normalize_private_key(private_key):
+    if private_key.startswith("0x"):
+        private_key = bytes(bytearray.fromhex(private_key[2:]))
+    else:
+        private_key = bytes(bytearray.fromhex(private_key))
+    return private_key
+
+
+def get_address_from_private(private_key):
+    return web3.eth.Account.privateKeyToAccount(private_key).address
+
+
+class add_to_path():
+    def __init__(self, path):
+        self.path = path
+    def __enter__(self):
+        sys.path.insert(0, self.path)
+    def __exit__(self, exc_type, exc_value, traceback):
+        try:
+            sys.path.remove(self.path)
+        except ValueError:
+            pass

snet_cli/snet_cli/__init__.py → packages/snet_cli/snet_cli/__init__.py

@@ -7,6 +7,7 @@
 from snet_cli.config import Config
 import argcomplete
 
+
 def main():
     try:
         argv = sys.argv[1:]

snet_cli/snet_cli/arguments.py → packages/snet_cli/snet_cli/arguments.py

@@ -7,7 +7,6 @@
 from snet_cli.commands import IdentityCommand, SessionSetCommand, SessionShowCommand, NetworkCommand, ContractCommand, \
     OrganizationCommand, VersionCommand
 from snet_cli.identity import get_identity_types
-from snet_cli.utils import type_converter, get_contract_def
 from snet_cli.mpe_account_command import MPEAccountCommand
 from snet_cli.mpe_service_command import MPEServiceCommand
 from snet_cli.mpe_channel_command import MPEChannelCommand
@@ -17,6 +16,8 @@
 from snet_cli.utils_agi2cogs import stragi2cogs
 from snet_cli.config import Config, get_session_keys, get_session_network_keys_removable
 
+from snet.snet_cli.utils import type_converter, get_contract_def, RESOURCES_PATH
+
 
 class CustomParser(argparse.ArgumentParser):
     def __init__(self, default_choice=None, *args, **kwargs):
@@ -234,7 +235,7 @@ def add_contract_options(parser):
     subparsers = parser.add_subparsers(title="contracts", metavar="CONTRACT")
     subparsers.required = True
 
-    for path in Path(__file__).absolute().parent.joinpath("resources", "contracts", "abi").glob("*json"):
+    for path in Path(RESOURCES_PATH.joinpath("contracts", "abi")).glob("*json"):
         contract_name = re.search(r"([^.]*)\.json", os.path.basename(path)).group(1)
         contract_p = subparsers.add_parser(contract_name, help="{} contract".format(contract_name))
         add_contract_function_options(contract_p, contract_name)