diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index e4fa55d..a4bfa2c 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -16,6 +16,8 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt -r requirements-dev.txt - pylint src --errors-only + pylint src tests --errors-only - name: Build package - run: pytest --junitxml output/report.xml + run: | + pip install -e . + pytest --junitxml output/report.xml diff --git a/.gitignore b/.gitignore index c78c064..137644c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,5 @@ output/ venv/ build/ dist/ -.pypirc +.env dmtgen.egg-info/ \ No newline at end of file diff --git a/.pypirc b/.pypirc new file mode 100644 index 0000000..f726079 --- /dev/null +++ b/.pypirc @@ -0,0 +1,8 @@ +[distutils] +index-servers = + gitlab + +[gitlab] +repository = https://gitlab.sintef.no/api/v4/projects/${env.CI_PROJECT_ID}/packages/pypi +username = gitlab-ci-token +password = ${env.CI_JOB_TOKEN} diff --git a/publish.sh b/publish.sh index c5679f9..73e9d30 100644 --- a/publish.sh +++ b/publish.sh @@ -1,6 +1,6 @@ set -e -python setup.py clean --all sdist bdist_wheel +python -m build --wheel --sdist if $PUBLISH_LIB; then python -m twine upload dist/* --config-file .pypirc diff --git a/requirements-dev.txt b/requirements-dev.txt index b49a23e..086c32a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,4 @@ -setuptools==65.5.1 -wheel==0.38.1 -twine==3.2.0 +build +twine pytest pylint \ No newline at end of file diff --git a/setup.py b/setup.py index 2ddfad7..2957910 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ setup( name='dmtgen', - version='0.5.0.dev1', + version='0.5.0', author="SINTEF Ocean", description="Python generator utilities for DMT", long_description=long_description, diff --git a/src/dmtgen/base_generator.py b/src/dmtgen/base_generator.py index 5c7bd8a..3ee84d9 100644 --- a/src/dmtgen/base_generator.py +++ b/src/dmtgen/base_generator.py @@ -27,7 +27,7 @@ def __init__(self, root_dir: Path, package_name: str, self.source_only = False self.root_package = root_package - # pylint: disable=unused-argument, no-self-use + # pylint: disable=unused-argument def get_template_generator(self, template: Path, config: Dict) -> TemplateBasedGenerator: """ Override in subclasses to control which template generator to use""" return BasicTemplateGenerator() @@ -72,10 +72,10 @@ def __find_templates_and_generate(self, output_dir: Path, config: Dict): @staticmethod def __read_template(templatefile: Path): loader = jinja2.FileSystemLoader(templatefile.parents[0]) - env_parameters = dict( - loader=loader, - undefined=jinja2.StrictUndefined - ) + env_parameters = { + "loader": loader, + "undefined":jinja2.StrictUndefined + } environment = jinja2.Environment(**env_parameters) environment.filters["escape_string"] = escape_string return environment.get_template(templatefile.name) diff --git a/src/dmtgen/common/blueprint.py b/src/dmtgen/common/blueprint.py index d6fd738..885d5ed 100644 --- a/src/dmtgen/common/blueprint.py +++ b/src/dmtgen/common/blueprint.py @@ -9,28 +9,23 @@ class Blueprint: """ " A basic SIMOS Blueprint""" - def __init__(self, bp_dict: Dict, parent: Package) -> None: + # pylint: disable=too-many-instance-attributes + def __init__(self, content: Dict, parent: Package) -> None: self.parent = parent - self.blueprint = bp_dict - self.name = self.blueprint["name"] - self.description = bp_dict.get("description","") - self.__abstract = bp_dict.get("abstract",False) + self.content = content + self.name: str = self.content["name"] + self.description: str = content.get("description",None) attributes = {} - for a_dict in bp_dict.get("attributes",[]): + for a_dict in content.get("attributes",[]): attribute = BlueprintAttribute(a_dict, self) attributes[attribute.name]=attribute + self.__abstract = content.get("abstract",False) self.__attributes = attributes - extends = bp_dict.get("extends",[]) - self.__extends = extends + self.__extends = content.get("extends",[]) # We will resolve this later self.__extensions = None - @property - def abstract(self) -> bool: - """If the blueprint represent an abstract type""" - return self.__abstract - @property def attributes(self) -> Sequence[BlueprintAttribute]: """Attributes""" @@ -53,17 +48,33 @@ def extensions(self) -> Sequence[Blueprint]: if self.__extensions is not None: return self.__extensions - self.__extensions = [self.__resolve(extension) for extension in self.__extends] + self.__extensions = [self.__resolve_extension(extension) for extension in self.__extends] return self.__extensions - def __resolve(self,extension: str): + def resolve(self): + """ Resolve all attributes""" + for attribute in self.__attributes.values(): + attribute.resolve() + + + def __resolve_extension(self,extension: str): package: Package = self.parent - return package.get_blueprint(extension) + resolved = package.resolve_type(extension) + return package.get_blueprint(resolved) + + def is_abstract(self) -> bool: + """If the blueprint represent an abstract type. + In object oriented terms, this would be an interface or abstract class.""" + return self.__abstract def get_path(self): - """ Get full path to blueprint """ + """ Get full path to blueprint""" parent = self.parent if parent: return parent.get_path() + "/" + self.name # Then we are at root - return "/" + self.name + return self.name + + def get_attribute(self, name:str) -> BlueprintAttribute: + """ Return the attribute if it exists, otherwise None""" + return self.all_attributes.get(name,None) diff --git a/src/dmtgen/common/blueprint_attribute.py b/src/dmtgen/common/blueprint_attribute.py index ee28170..d92a06d 100644 --- a/src/dmtgen/common/blueprint_attribute.py +++ b/src/dmtgen/common/blueprint_attribute.py @@ -8,7 +8,8 @@ class BlueprintAttribute: """ " A basic SIMOS Attribute""" - def __init__(self, content: Dict, parent_blueprint: Blueprint) -> None: + # pylint: disable=too-many-instance-attributes + def __init__(self, content: Dict, parent: Blueprint) -> None: self.content = content name = content["name"] if len(name)==0: @@ -24,48 +25,94 @@ def __init__(self, content: Dict, parent_blueprint: Blueprint) -> None: self.dimensions = [] atype = content["attributeType"] - self.parent = parent_blueprint - package = parent_blueprint.parent - self.type = package.resolve_type(atype) - self.is_primitive = atype in ['boolean', 'number', 'string', 'integer'] - self.is_enum = self.content.get("enumType",None) is not None - self.is_blueprint = not (self.is_primitive or self.is_enum) - self.is_optional = self.content.get("optional",True) - self.is_array = len(self.dimensions)>0 - self.is_contained = content.get("contained",True) + self.__parent = parent + self.__type = atype + self.__optional = self.content.get("optional",True) + self.__is_primitive = atype in ['boolean', 'number', 'string', 'integer'] + self.enum_type = self.content.get("enumType",None) + self.__is_enum = self.enum_type is not None + self.__is_blueprint = not (self.__is_primitive or self.__is_enum) + self.__is_array = len(self.dimensions)>0 + self.__is_string = self.type == "string" + self.__is_boolean = self.type == "boolean" + self.__is_integer = self.type == "integer" + self.__is_number = self.type == "number" + self.__is_contained = content.get("contained",True) + + def resolve(self): + """ Resolve to correct type""" + package = self.parent.parent + self.__type = package.resolve_type(self.__type) + if self.enum_type: + self.enum_type = package.resolve_type(self.enum_type) + + @property + def parent(self) -> Blueprint: + """The parent blueprint""" + return self.__parent + + @property + def type(self) -> str: + """The type of the attribute""" + return self.__type @property + def optional(self) -> bool: + """Is this an optional attribute""" + return self.__optional + + @property + def contained(self) -> bool: + """Is this a contained attribute""" + return self.__is_contained + + def is_primitive(self) -> bool: + """Is this a primitive type""" + return self.__is_primitive + + def is_enum(self) -> bool: + """Is this an enum type""" + return self.__is_enum + + def is_blueprint(self) -> bool: + """Is this a blueprint type""" + return self.__is_blueprint + def is_string(self) -> bool: """Is this a string""" - return self.type == "string" + return self.__is_string - @property def is_boolean(self) -> bool: """Is this a boolean""" - return self.type == "boolean" + return self.__is_boolean - @property def is_integer(self) -> bool: """Is this an integer""" - return self.type == "integer" + return self.__is_integer - @property def is_number(self) -> bool: """Is this a number""" - return self.type == "number" + return self.__is_number + + def is_optional(self) -> bool: + """Is an optional relation""" + return self.__optional - @property def is_required(self) -> bool: """Is a required relation""" - return not self.is_optional + return not self.__optional + + def is_array(self) -> bool: + """Is this an array""" + return self.__is_array def is_fixed_array(self) -> bool: """Is this a fixed array""" - return self.is_array and "*" not in self.dimensions + return self.__is_array and "*" not in self.dimensions def is_variable_array(self) -> bool: """Is this a variable array""" - return self.is_array and "*" in self.dimensions + return self.__is_array and "*" in self.dimensions def get(self, key, default=None): """Return the content value or an optional default""" diff --git a/src/dmtgen/common/package.py b/src/dmtgen/common/package.py index 2efb089..ee3f11f 100644 --- a/src/dmtgen/common/package.py +++ b/src/dmtgen/common/package.py @@ -7,7 +7,7 @@ import os import re from pathlib import Path -from typing import List, Sequence +from typing import List, Sequence,Dict from .enum_description import EnumDescription from .blueprint import Blueprint @@ -15,7 +15,8 @@ class Package: """ " A basic SIMOS package""" - def __init__(self, pkg_dir: Path, parent: Package) -> None: + # pylint: disable=too-many-instance-attributes + def __init__(self, pkg_dir: Path, parent: Package, config: Dict=None) -> None: self.package_dir = pkg_dir self.version = 0 self.name = pkg_dir.name @@ -24,7 +25,23 @@ def __init__(self, pkg_dir: Path, parent: Package) -> None: self.__blueprints = {} self.__enums = {} self.__packages = {} + self.__dependencies = {} + if parent is None and config is not None: + # This is a root package and we might have dependencies + dependencies: dict = config.get('dependencies', {}) + for alias,location in dependencies.items(): + self.__dependencies[alias] = Package(Path(location),None) + self.__read_package(pkg_dir) + if parent is None: + self.resolve() + + def resolve(self): + """ Resolve all references in package and subpackages""" + for blueprint in self.blueprints: + blueprint.resolve() + for package in self.packages: + package.resolve() def __read_package(self, pkg_dir: Path): blueprints = {} @@ -36,6 +53,7 @@ def __read_package(self, pkg_dir: Path): pkg_filename = "package.json" package_file = pkg_dir / pkg_filename if package_file.exists(): + # Use with to ensure file is closed with open(package_file, encoding="utf-8") as file: package = json.load(file) self.__read_package_info(package) @@ -43,19 +61,23 @@ def __read_package(self, pkg_dir: Path): for file in pkg_dir.glob("*.json"): if file.name == pkg_filename: continue - with open(file, encoding="utf-8") as file: - entity = json.load(file) - etype = self.resolve_type(entity["type"]) - if etype == "system/SIMOS/Blueprint": - blueprint = Blueprint(entity, self) - name = blueprint.name - blueprints[name] = blueprint - elif etype == "system/SIMOS/Enum": - enum = EnumDescription(entity, self) - name = enum.name - enums[name] = enum - else: - raise ValueError("Unhandled entity type: " + etype) + + if file.name == "__versions__.json": + self.__read_version(entity) + else: + with open(file, encoding="utf-8") as file: + entity = json.load(file) + etype = self.resolve_type(entity["type"]) + if etype == "system/SIMOS/Blueprint": + blueprint = Blueprint(entity, self) + name = blueprint.name + blueprints[name] = blueprint + elif etype == "system/SIMOS/Enum": + enum = EnumDescription(entity, self) + name = enum.name + enums[name] = enum + else: + raise ValueError("Unhandled entity type: " + etype) for folder in pkg_dir.glob("*/"): if folder.is_dir(): @@ -87,9 +109,7 @@ def resolve_type(self, etype:str) -> str: idx=etype.find(":") if idx > 0: alias = etype[:idx].lower() - adress = self.aliases.get(alias,None) - if not adress: - raise ValueError(f"Alias not found \"{alias}\" in {self.name}") + adress = self.aliases.get(alias,alias) return adress + "/" + etype[idx+1:] return etype @@ -153,7 +173,7 @@ def get_parent(self) -> Package: """Get parent package""" return self.parent - def get_root(self): + def get_root(self) -> Package: """Get root package""" parent: Package = self.parent if parent: @@ -164,13 +184,6 @@ def get_root(self): def get_blueprint(self, path:str) -> Blueprint: """Get Blueprint from path""" - path = self.resolve_type(path) - idx=path.find(":") - if idx > 0: - alias = path[:idx].lower() - adress = self.aliases[alias] - path = adress + "/" + path[idx+1:] - parts = re.split("/",path) bp_name = parts.pop() package = self.__get_package(parts) @@ -178,7 +191,6 @@ def get_blueprint(self, path:str) -> Blueprint: def get_enum(self, path:str) -> EnumDescription: """Get enum from path""" - path = self.resolve_type(path) parts = re.split("/",path) enum_name = parts.pop() package = self.__get_package(parts) @@ -190,15 +202,20 @@ def __get_package(self, parts: Sequence[str]) -> Package: if part == '.': raise ValueError("Relative path not allowed. Should have been resolved by now.") if part == '': - package = self.get_root() - elif part == 'system': + continue + if part == 'system': # pylint: disable=import-outside-toplevel from .system_package import system_package package = system_package elif package is None: - package = self.get_root() - if part != package.name: - raise ValueError(f"expected root {package.name} but got {part}") + # Use the root package to resolve + root_package: Package = self.get_root() + if part == root_package.name: + package = root_package + else: + package = root_package.__dependencies.get(part,None) + if not package: + raise ValueError(f"Package not found \"{part}\" in {root_package.name} or dependencies.") else: package = package.package(part) return package diff --git a/src/dmtgen/data/system/SIMOS/entity.json b/src/dmtgen/data/system/SIMOS/entity.json index 43c7935..808a97d 100644 --- a/src/dmtgen/data/system/SIMOS/entity.json +++ b/src/dmtgen/data/system/SIMOS/entity.json @@ -12,7 +12,5 @@ "default": "", "optional": true } - ], - "storageRecipes": [], - "uiRecipes": [] + ] } \ No newline at end of file diff --git a/src/dmtgen/data/system/SIMOS/named_enity.json b/src/dmtgen/data/system/SIMOS/named_enity.json index 3ae2cf2..77cf70a 100644 --- a/src/dmtgen/data/system/SIMOS/named_enity.json +++ b/src/dmtgen/data/system/SIMOS/named_enity.json @@ -2,7 +2,7 @@ "name": "NamedEntity", "type": "system/SIMOS/Blueprint", "abstract" : true, - "extends" : ["system/SIMOS/Entity"], + "extends" : ["./Entity"], "description": "Describes the required attributes (name, type, description). All other blueprints should extend this one, or implement the attributes itself", "attributes": [ { @@ -12,7 +12,5 @@ "label": "Name", "optional": true } - ], - "storageRecipes": [], - "uiRecipes": [] + ] } \ No newline at end of file diff --git a/src/tests/__init__.py b/tests/__init__.py similarity index 100% rename from src/tests/__init__.py rename to tests/__init__.py diff --git a/src/tests/test_data/apps/EmployeeApp/Employee.blueprint.json b/tests/test_data/apps/EmployeeApp/Employee.blueprint.json similarity index 100% rename from src/tests/test_data/apps/EmployeeApp/Employee.blueprint.json rename to tests/test_data/apps/EmployeeApp/Employee.blueprint.json diff --git a/src/tests/test_data/apps/EmployeeApp/EmployeeApp.blueprint.json b/tests/test_data/apps/EmployeeApp/EmployeeApp.blueprint.json similarity index 100% rename from src/tests/test_data/apps/EmployeeApp/EmployeeApp.blueprint.json rename to tests/test_data/apps/EmployeeApp/EmployeeApp.blueprint.json diff --git a/src/tests/test_data/apps/package.json b/tests/test_data/apps/package.json similarity index 100% rename from src/tests/test_data/apps/package.json rename to tests/test_data/apps/package.json diff --git a/src/tests/test_relative_paths.py b/tests/test_relative_paths.py similarity index 100% rename from src/tests/test_relative_paths.py rename to tests/test_relative_paths.py