Merge branch 'develop' into 'main'

add tests, refactor queries

See merge request elds/plugins/cmem-plugin-irdi!1

.copier-answers.yml

@@ -1,10 +1,10 @@
 # Changes here will be overwritten by Copier
-_commit: v5.3.4
+_commit: v6.0.1
 _src_path: gh:eccenca/cmem-plugin-template
 author_mail: [email protected]
-author_name: TODO
+author_name: Robert Klose
 github_page: ''
-project_description: TODO
+project_description: Create unique IRDIs
 project_slug: irdi
 pypi: false
 

.github/workflows/check.yml

@@ -42,21 +42,13 @@ jobs:
         run: |
           poetry self add "poetry-dynamic-versioning[plugin]"
 
-      - name: bandit
-        run: |
-          task check:bandit
-
-      - name: flake8
-        run: |
-          task check:flake8
-
       - name: mypy
         run: |
           task check:mypy
 
-      - name: pylint
+      - name: ruff
         run: |
-          task check:pylint
+          task check:ruff
 
       - name: pytest
         env:

.gitlab-ci.yml

@@ -19,25 +19,15 @@ stages:
   - build
   - publish
 
-bandit:
+ruff:
   stage: test
   script:
-    - task check:bandit
+    - task check:ruff
   artifacts:
     when: always
     reports:
       junit:
-        - dist/junit-bandit.xml
-
-flake8:
-  stage: test
-  script:
-    - task check:flake8
-  artifacts:
-    when: always
-    reports:
-      junit:
-        - dist/junit-flake8.xml
+        - dist/junit-ruff.xml
 
 mypy:
   stage: test
@@ -49,16 +39,6 @@ mypy:
       junit:
         - dist/junit-mypy.xml
 
-pylint:
-  stage: test
-  script:
-    - task check:pylint
-  artifacts:
-    when: always
-    reports:
-      junit:
-        - dist/junit-pylint.xml
-
 pytest:
   stage: test
   coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
@@ -86,7 +66,6 @@ safety:
 build:
   stage: build
   needs:
-    - bandit
     - mypy
     - pytest
     - safety
@@ -102,8 +81,7 @@ pypi:
   # publishing only available on a tag
   stage: publish
   needs:
-    - flake8
-    - pylint
+    - ruff
     - build
   allow_failure: true
   when: manual

.pre-commit-config.yaml

@@ -2,17 +2,37 @@
 repos:
   - repo: local
     hooks:
-      - id: pylint
-        name: pylint
-        entry: poetry run pylint
-        exclude: '^tests/.*$'
-        language: system
-        types: [python]
-  - repo: local
-    hooks:
-      - id: forbidden-files
-        name: forbidden files
-        entry: found copier update rejection files; review them and remove them
-        language: fail
-        files: "\\.rej$"
+
+      - id: ruff
+        name: check:ruff
+        entry: task check:ruff
+        language: python
+        types_or: [python, pyi]
+        pass_filenames: false
+
+      - id: poetry-check
+        name: poetry-check
+        description: run poetry check to validate config
+        entry: poetry check
+        language: python
+        pass_filenames: false
+        files: ^(.*/)?pyproject\.toml$
+
+      - id: poetry-lock
+        name: poetry-lock
+        description: run poetry lock to update lock file
+        entry: poetry lock
+        language: python
+        pass_filenames: false
+        files: ^(.*/)?(poetry\.lock|pyproject\.toml)$
+
+      - id: poetry-install
+        name: poetry-install
+        description: >
+          run poetry install to install dependencies from the lock file
+        entry: poetry install
+        language: python
+        pass_filenames: false
+        stages: [post-checkout, post-merge]
+        always_run: true
 

README-public.md

@@ -1,6 +1,7 @@
 # cmem-plugin-irdi
 
-TODO
+Create unique [ECLASS](https://eclass.eu/support/technical-specification/structure-and-elements/irdi) IRDIs
+
 
 This is a plugin for [eccenca](https://eccenca.com) [Corporate Memory](https://documentation.eccenca.com).
 
@@ -11,3 +12,10 @@ clients like this:
 cmemc admin workspace python install cmem-plugin-irdi
 ```
 
+## Plugin Usage
+
+- All fields of the IRDI are configurable, minus `Item Code`, which is created by the plugin
+  - Created IRDIs are unique per configuration
+- Specify a graph that stores the state of `Item Codes`
+- Input and output paths are configurable
+  - if no input path is configured, values are read from the URIs of the input (Transformation Input)

README.md

@@ -1,6 +1,6 @@
 # cmem-plugin-irdi
 
-TODO
+Create unique [ECLASS](https://eclass.eu/support/technical-specification/structure-and-elements/irdi) IRDIs
 
 [![eccenca Corporate Memory](https://img.shields.io/badge/eccenca-Corporate%20Memory-orange)](https://documentation.eccenca.com)   
 
@@ -10,3 +10,10 @@ TODO
 - Use [pre-commit](https://pre-commit.com/) to avoid errors before commit.
 - This repository was created with [this copier template](https://github.com/eccenca/cmem-plugin-template).
 
+## Plugin Usage
+
+- All fields of the IRDI are configurable, minus `Item Code`, which is created by the plugin
+  - Created IRDIs are unique per configuration
+- Specify a graph that stores the state of `Item Codes`
+- Input and output paths are configurable
+  - if no input path is configured, values are read from the URIs of the input (Transformation Input) 

Taskfile.yaml

@@ -66,11 +66,19 @@ tasks:
     cmds:
       - poetry install
 
-  python:format:
-    desc: Format Python files
+  format:fix:
+    desc: Format Python files and fix obvious issues
     <<: *preparation
     cmds:
-      - poetry run black .
+      - poetry run ruff format tests {{.PACKAGE}}
+      - poetry run ruff check tests {{.PACKAGE}} --fix-only
+
+  format:fix-unsafe:
+    desc: Format Python files and fix 'unsafe' issues
+    <<: *preparation
+    cmds:
+      - poetry run ruff format tests {{.PACKAGE}}
+      - poetry run ruff check tests {{.PACKAGE}} --fix-only --unsafe-fixes
 
   clean:
     desc: Removes dist, *.pyc and some caches
@@ -83,18 +91,16 @@ tasks:
 
   check:
     desc: Run whole test suite incl. unit and integration tests
-    deps:
-      - check:linters
-      - check:pytest
+    cmds:
+      - task: check:linters
+      - task: check:pytest
 
   check:linters:
     desc: Run all linter and static code analysis tests
-    deps:
-      - check:bandit
-      - check:flake8
-      - check:mypy
-      - check:pylint
-      - check:safety
+    cmds:
+      - task: check:ruff
+      - task: check:mypy
+      - task: check:safety
 
   check:pytest:
     desc: Run unit and integration tests
@@ -125,59 +131,39 @@ tasks:
       BADGE_COVERAGE: ./{{.DIST_DIR}}/badge-coverage.svg
       BADGE_TESTS: ./{{.DIST_DIR}}/badge-tests.svg
 
-  check:pylint:
-    desc: Find code smells, errors and style issues
-    <<: *preparation
-    cmds:
-      - poetry run pylint --exit-zero {{.PACKAGE}}
-      - poetry run pylint {{.PACKAGE}} {{.XML_PARAMS}}
-    vars:
-      FORMAT: --output-format=pylint_junit.JUnitReporter
-      JUNIT_FILE: ./{{.DIST_DIR}}/junit-pylint.xml
-      XML_PARAMS: --output={{.JUNIT_FILE}} {{.FORMAT}}
-
   check:mypy:
-    desc: Find type errors
+    desc: Complain about typing errors
     <<: *preparation
     cmds:
       - poetry run mypy -p tests -p {{.PACKAGE}} --junit-xml {{.JUNIT_FILE}}
     vars:
       JUNIT_FILE: ./{{.DIST_DIR}}/junit-mypy.xml
 
   check:safety:
-    desc: Scan dependencies for vulnerabilities
+    desc: Complain about vulnerabilities in dependencies
     <<: *preparation
     cmds:
       # ignore 51358 safety - dev dependency only
       # ignore 61489 pillow - dev dependency only
       - poetry run safety check -i 51358 -i 61489
 
-  check:bandit:
-    desc: Find common security issues
-    <<: *preparation
-    cmds:
-      - poetry run bandit --exit-zero -r {{.PACKAGE}}
-      - poetry run bandit --format xml -r {{.PACKAGE}} -o {{.JUNIT_FILE}}
-    vars:
-      JUNIT_FILE: ./{{.DIST_DIR}}/junit-bandit.xml
-
-  check:flake8:
-    desc: Enforce standard source code style guide
+  check:ruff:
+    desc: Complain about everything else
     <<: *preparation
     cmds:
-      - poetry run flake8 --exit-zero tests {{.PACKAGE}} {{.XML_PARAMS}}
-      - poetry run flake8 --show-source tests {{.PACKAGE}}
+      - poetry run ruff check --show-source tests {{.PACKAGE}}
+      - poetry run ruff check --exit-zero tests {{.PACKAGE}} {{.XML_PARAMS}}
+      - poetry run ruff format --check tests {{.PACKAGE}}
     vars:
-      JUNIT_FILE: ./{{.DIST_DIR}}/junit-flake8.xml
-      XML_PARAMS: --format junit-xml --output-file {{.JUNIT_FILE}}
+      JUNIT_FILE: ./{{.DIST_DIR}}/junit-ruff.xml
+      XML_PARAMS: --output-format junit --output-file {{.JUNIT_FILE}}
 
   # }}}
   # {{{ build and deploy tasks
 
   deploy:
     desc: Install plugin package in Corporate Memory
     deps:
-      - clean
       - build
     cmds:
       - cmemc admin workspace python install dist/*.tar.gz
@@ -186,6 +172,9 @@ tasks:
   build:
     desc: Build a tarball and a wheel package
     <<: *preparation
+    deps:
+      - clean
+      - poetry:check
     cmds:
       - poetry build
 

cmem_plugin_irdi/__init__.py

@@ -0,0 +1 @@
+"""irdi - main package"""

cmem_plugin_irdi/components.py

@@ -0,0 +1,50 @@
+"""PluginParameters and validation regex's for IRDI components"""
+from cmem_plugin_base.dataintegration.description import PluginParameter
+
+components = {
+    "icd": {
+        "parameter": PluginParameter(
+            name="icd", label="International Code Designator (ICD): Numeric, 4 characters"
+        ),
+        "regex": r"^\d{4}$",
+    },
+    "oi": {
+        "parameter": PluginParameter(
+            name="oi",
+            label="Organization Identifier (OI): Numeric, 4 characters",
+        ),
+        "regex": r"^\d{4}$",
+    },
+    "opi": {
+        "parameter": PluginParameter(
+            name="opi",
+            label="Organization Part Identifier (OPI): Alphanumeric, up to 35 "
+            "characters (base36)",
+            default_value="",
+        ),
+        "regex": r"^[a-zA-Z0-9]{0,35}$",
+    },
+    "opis": {
+        "parameter": PluginParameter(
+            name="opis",
+            label="OPI Source Indicator (OPIS): Numeric, 1 character",
+            default_value="",
+        ),
+        "regex": r"^\d$",
+    },
+    "ai": {
+        "parameter": PluginParameter(
+            name="ai",
+            label="Additional information (AI): Numeric, 4 characters",
+            default_value="",
+        ),
+        "regex": r"^\d{4}$",
+    },
+    "csi": {
+        "parameter": PluginParameter(
+            name="csi",
+            label="Code-space identifier (CSI): Alphanumeric, 2 character (base36)",
+        ),
+        "regex": r"^[a-zA-Z0-9]{2}$",
+    },
+}