Skip to content

Commit

Permalink
Feature/missing fields datamodel (#236)
Browse files Browse the repository at this point in the history
* Correct handling of new data model fields in ModelImporterExporter.

* adapt master-solicitor.asciidoc

* fixed issue with test data

* removed two duplicated lines

---------

Co-authored-by: ohecker <[email protected]>
  • Loading branch information
mahmoudAlkam and ohecker authored Mar 4, 2024
1 parent d60fc02 commit 1b2368d
Show file tree
Hide file tree
Showing 4 changed files with 205 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@
import com.fasterxml.jackson.databind.SerializationFeature;

/**
* Imports and exports the data model.
* The {@code ModelImporterExporter} class handles the import and export of the data model. It provides methods for
* loading a data model from a JSON file, checking model version compatibility, and saving the data model to a file.
* Additionally, it facilitates the transformation of JSON data into model objects, ensuring proper association between
* the various components of the data model.
*/
@Component
public class ModelImporterExporter {
Expand All @@ -54,10 +57,12 @@ public class ModelImporterExporter {
private ModelFactoryImpl modelFactory;

/**
* Loads the data model from a file.
* Loads the data model from a JSON file. The loaded data model is represented by a root object of type
* {@code ModelRootImpl}.
*
* @param filename the name of the file to load from.
* @return the root object of the loaded data model
* @param filename the name of the file to load the data model from.
* @return the root object of the loaded data model.
* @throws SolicitorRuntimeException if there is an issue loading the data model from the file.
*/
public ModelRootImpl loadModel(String filename) {

Expand Down Expand Up @@ -102,10 +107,11 @@ public ModelRootImpl loadModel(String filename) {
}

/**
* Checks if the version of the model to be loaded is supported.
* Checks if the version of the model to be loaded is supported and compatible with the current model version.
*
* @param readModelVersion the model version of the model to be loaded
* @param currentModelRoot the root object of the current (target) model
* @throws SolicitorRuntimeException if the model version is unsupported or incompatible.
*/
private void checkModelVersion(int readModelVersion, ModelRootImpl currentModelRoot) {

Expand All @@ -117,11 +123,13 @@ private void checkModelVersion(int readModelVersion, ModelRootImpl currentModelR
}

/**
* Read the {@link ApplicationComponent}s from the JSON data structure.
* Reads the {@link ApplicationComponent}s from the JSON data structure and populates the provided {@link Application}
* with the corresponding data.
*
* @param application the {@link Application} to which the {@link ApplicationComponent}s belong to
* @param applicationComponentsNode the relevant part of the parse JSON model
* @param readModelVersion the model version of the model to be read
*
*/
private void readApplicationComponents(ApplicationImpl application, JsonNode applicationComponentsNode,
int readModelVersion) {
Expand All @@ -148,6 +156,21 @@ private void readApplicationComponents(ApplicationImpl application, JsonNode app
String noticeFileUrl = noticeFileUrlNode != null ? noticeFileUrlNode.asText(null) : null;
JsonNode normalizedLicensesNode = applicationComponentNode.get("normalizedLicenses");
JsonNode rawLicensesNode = applicationComponentNode.get("rawLicenses");
String dataStatus = applicationComponentNode.has("dataStatus")
? applicationComponentNode.get("dataStatus").asText(null)
: null;
String traceabilityNotes = applicationComponentNode.has("traceabilityNotes")
? applicationComponentNode.get("traceabilityNotes").asText(null)
: null;
String sourceDownloadUrl = applicationComponentNode.has("sourceDownloadUrl")
? applicationComponentNode.get("sourceDownloadUrl").asText(null)
: null;
String packageDownloadUrl = applicationComponentNode.has("packageDownloadUrl")
? applicationComponentNode.get("packageDownloadUrl").asText(null)
: null;
String noticeFileContentKey = applicationComponentNode.has("noticeFileContentKey")
? applicationComponentNode.get("noticeFileContentKey").asText(null)
: null;

ApplicationComponentImpl applicationComponent = this.modelFactory.newApplicationComponent();
applicationComponent.setApplication(application);
Expand All @@ -165,6 +188,11 @@ private void readApplicationComponents(ApplicationImpl application, JsonNode app
applicationComponent.setRepoType(repoType);
applicationComponent.setCopyrights(copyrights);
applicationComponent.setNoticeFileUrl(noticeFileUrl);
applicationComponent.setDataStatus(dataStatus);
applicationComponent.setTraceabilityNotes(traceabilityNotes);
applicationComponent.setSourceDownloadUrl(sourceDownloadUrl);
applicationComponent.setPackageDownloadUrl(packageDownloadUrl);
applicationComponent.setNoticeFileContentKey(noticeFileContentKey);

readNormalizedLicenses(applicationComponent, normalizedLicensesNode, readModelVersion);
readRawLicenses(applicationComponent, rawLicensesNode, readModelVersion);
Expand Down Expand Up @@ -226,7 +254,8 @@ private void readEngagement(ModelRootImpl modelRoot, JsonNode engagementNode, in
}

/**
* Read the {@link NormalizedLicense}s from the JSON data structure.
* Reads the {@link NormalizedLicense}s from the JSON data structure and associates them with the provided
* {@link ApplicationComponent}.
*
* @param applicationComponent The {@link ApplicationComponent} to which the license belongs
* @param normalizedLicensesNode the relevant part of the parsed JSON model
Expand All @@ -236,6 +265,7 @@ private void readNormalizedLicenses(ApplicationComponentImpl applicationComponen
int readModelVersion) {

for (JsonNode normalizedLicenseNode : normalizedLicensesNode) {
// Extracting information from the JSON node
String declaredLicense = normalizedLicenseNode.get("declaredLicense").asText(null);
String licenseUrl = normalizedLicenseNode.get("licenseUrl").asText(null);
String normalizedLicenseType = normalizedLicenseNode.get("normalizedLicenseType").asText(null);
Expand All @@ -261,6 +291,7 @@ private void readNormalizedLicenses(ApplicationComponentImpl applicationComponen
guessedLicenseUrl = normalizedLicenseNode.get("guessedLicenseUrl").asText(null);
guessedLicenseUrlAuditInfo = normalizedLicenseNode.get("guessedLicenseUrlAuditInfo").asText(null);
}
// Text pool keys introduced in certain model versions
String effectiveNormalizedLicenseContentKey = null;
String declaredLicenseContentKey = null;
String licenseRefContentKey = null;
Expand All @@ -274,7 +305,7 @@ private void readNormalizedLicenses(ApplicationComponentImpl applicationComponen
normalizedLicenseContentKey = normalizedLicenseNode.get("normalizedLicenseContentKey").asText(null);
guessedLicenseContentKey = normalizedLicenseNode.get("guessedLicenseContentKey").asText(null);
}

// Creating a new NormalizedLicense object and populating its fields
NormalizedLicenseImpl normalizedLicense = this.modelFactory.newNormalizedLicense();
normalizedLicense.setApplicationComponent(applicationComponent);
normalizedLicense.setDeclaredLicense(declaredLicense);
Expand Down Expand Up @@ -303,6 +334,7 @@ private void readNormalizedLicenses(ApplicationComponentImpl applicationComponen
normalizedLicense.setLicenseRefContentKey(licenseRefContentKey);
normalizedLicense.setNormalizedLicenseContentKey(normalizedLicenseContentKey);
normalizedLicense.setGuessedLicenseContentKey(guessedLicenseContentKey);

}
}

Expand Down Expand Up @@ -331,7 +363,6 @@ private void readRawLicenses(ApplicationComponentImpl applicationComponent, Json
rawLicense.setTrace(trace);
rawLicense.setOrigin(origin);
rawLicense.setSpecialHandling(specialHandling);

}
}

Expand All @@ -349,6 +380,7 @@ private void readTextPool(ModelRootImpl modelRoot, JsonNode textPoolNode, int re
}
TextPool textPool = modelRoot.getTextPool();
JsonNode dataMapNode = textPoolNode.get("dataMap");

for (JsonNode singleEntryValue : dataMapNode) {
// only store values; keys will be reconstructed based on values
textPool.store(singleEntryValue.asText());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,17 @@

import java.io.File;
import java.io.IOException;
import java.util.List;

import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;

import com.devonfw.tools.solicitor.common.SolicitorRuntimeException;
import com.devonfw.tools.solicitor.model.inventory.ApplicationComponent;
import com.devonfw.tools.solicitor.model.masterdata.Application;
import com.devonfw.tools.solicitor.model.masterdata.Engagement;

/**
* Tests of {@link ModelImporterExporter}.
Expand Down Expand Up @@ -68,6 +72,40 @@ public void testLoadModelVersion5() {
this.mie.loadModel("src/test/resources/models/model_version_5.json");
}

/**
* Test method for {@link com.devonfw.tools.solicitor.model.ModelImporterExporter#loadModel(java.lang.String)}.
*/
@Test
public void testLoadModelVersion6() {

ModelRoot modelRoot = this.mie.loadModel("src/test/resources/models/model_version_6.json");

// Assert
Assertions.assertNotNull(modelRoot);
Engagement engagement = modelRoot.getEngagement();
Assertions.assertNotNull(engagement);
List<Application> applications = engagement.getApplications();
Assertions.assertFalse(applications.isEmpty());

Application application = applications.get(0);
Assertions.assertNotNull(application);

List<ApplicationComponent> applicationComponents = application.getApplicationComponents();
Assertions.assertFalse(applicationComponents.isEmpty());

ApplicationComponent component = applicationComponents.get(0);
Assertions.assertNotNull(component);

// Add assertions for specific fields
Assertions.assertEquals("DA:NO_ISSUES", component.getDataStatus());
Assertions.assertEquals("", component.getTraceabilityNotes()); // Assuming it's an empty string
Assertions.assertEquals("https://github.com/qos-ch/logback/archive/refs/tags/logback-1.2.3.zip",
component.getSourceDownloadUrl());
Assertions.assertEquals(
"https://repo.maven.apache.org/maven2/ch/qos/logback/logback-classic/1.2.3/logback-classic-1.2.3.jar",
component.getPackageDownloadUrl());
}

/**
* Test method for
* {@link com.devonfw.tools.solicitor.model.ModelImporterExporter#saveModel(com.devonfw.tools.solicitor.model.ModelRoot, java.lang.String)}.
Expand All @@ -77,7 +115,7 @@ public void testLoadModelVersion5() {
@Test
public void testSaveModel() throws IOException {

ModelRoot mr = this.mie.loadModel("src/test/resources/models/model_version_3.json");
ModelRoot mr = this.mie.loadModel("src/test/resources/models/model_version_6.json");
File tempFile = File.createTempFile("solicitor_model", "json");
String fileName = tempFile.getPath();

Expand Down
124 changes: 124 additions & 0 deletions core/src/test/resources/models/model_version_6.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
{
"executionTime" : "Fri Oct 01 18:38:31 CEST 2021",
"modelVersion" : 6,
"solicitorVersion" : "1.3.0-SNAPSHOT",
"solicitorGitHash" : "a507f22",
"solicitorBuilddate" : "2021-10-01 18:34:12 +0200",
"extensionArtifactId" : "cap-solicitor-extension",
"extensionVersion" : "1.3.0-RC1",
"extensionGitHash" : "043a961",
"extensionBuilddate" : "2021-09-13 22:24:45",
"engagement" : {
"engagementName" : "Some Engagement",
"engagementType" : "INTERN",
"clientName" : "none",
"goToMarketModel" : "LICENSE",
"contractAllowsOss" : true,
"ossPolicyFollowed" : true,
"customerProvidesOss" : false,
"applications" : [ {
"name" : "Some Application",
"releaseId" : "1.2.3-SNAPSHOT",
"releaseDate" : "-UNDEFINED-",
"sourceRepo" : "https://point/to/your/repo.git",
"programmingEcosystem" : "Java8",
"applicationComponents" : [ {
"usagePattern" : "DYNAMIC_LINKING",
"ossModified" : false,
"ossHomepage" : null,
"sourceRepoUrl" : "https://github.com/qos-ch/logback",
"noticeFileUrl" : "http://some.url",
"noticeFileContentKey" : "31cb574375eadadae8835a406879719d85da9fbb5f0f0c6fdc62da741dc49de5",
"groupId" : "ch.qos.logback",
"artifactId" : "logback-classic",
"version" : "1.2.3",
"repoType" : "maven",
"packageUrl" : "pkg:maven/ch.qos.logback/[email protected]",
"copyrights" : null,
"packageDownloadUrl" : "https://repo.maven.apache.org/maven2/ch/qos/logback/logback-classic/1.2.3/logback-classic-1.2.3.jar",
"sourceDownloadUrl" : "https://github.com/qos-ch/logback/archive/refs/tags/logback-1.2.3.zip",
"normalizedLicenses" : [ {
"declaredLicense" : "Eclipse Public License - v 1.0",
"licenseUrl" : "http://www.eclipse.org/legal/epl-v10.html",
"declaredLicenseContentKey" : "cf630ee3446f9e10c1b0814fb8a652f4733b491eaea9d5d1f89f803daaaa6075",
"normalizedLicenseType" : "OSS-SPDX",
"normalizedLicense" : "EPL-1.0",
"normalizedLicenseUrl" : "http://www.eclipse.org/legal/epl-v10.html",
"normalizedLicenseContentKey" : "cf630ee3446f9e10c1b0814fb8a652f4733b491eaea9d5d1f89f803daaaa6075",
"effectiveNormalizedLicenseType" : "IGNORE",
"effectiveNormalizedLicense" : "Ignore",
"effectiveNormalizedLicenseUrl" : null,
"effectiveNormalizedLicenseContentKey" : "9d086420212f290c944042014500f00716c30d53f316d224b4cdd0c20756ab6b",
"legalPreApproved" : "N/A",
"copyLeft" : "N/A",
"licenseCompliance" : "N/A",
"licenseRefUrl" : "http://another.url",
"licenseRefContentKey" : "7dbd9399a4684cdc5c1c51d6359294caea160c648f1792e273a9489e46321b88",
"includeLicense" : "no",
"includeSource" : "no",
"reviewedForRelease" : null,
"comments" : null,
"legalApproved" : "N/A",
"legalComments" : null,
"trace" : "+ Component/License info read in '[maven]' format from 'file:./input/licenses_starter.xml'\r\n+ Rule Group: LicenseNameMappingDefaults; RuleId: 57; Matching: declaredLicense==Eclipse Public License - v 1.0; Setting: normalizedLicenseType=OSS-SPDX, normalizedLicense=EPL-1.0, normalizedLicenseUrl=http://www.eclipse.org/legal/epl-v10.html (taking data from input)\r\n+ Rule Group: MultiLicenseSelection; RuleId: 6; Matching: groupId==ch.qos.logback, normalizedLicense==EPL-1.0; Setting: effectiveNormalizedLicenseType=IGNORE (multilicensing: ignore this, prefer LGPL-2.1), effectiveNormalizedLicense=Ignore\r\n+ Rule Group: LegalPreEvaluation; RuleId: 29; Matching: effectiveNormalizedLicenseType==IGNORE; Setting: legalPreApproved=N/A, copyLeft=N/A, licenseCompliance=N/A, includeLicense=no, includeSource=no\r\n+ Rule Group: LegalEvaluation; RuleId: 1; Matching: effectiveNormalizedLicenseType==IGNORE; Setting: legalApproved=N/A",
"guessedLicenseUrl" : null,
"guessedLicenseContentKey" : "90fa1cb2ff3629e0f7a3c78dc887134e5e05224de1b13fed1dedb33b1e920724",
"guessedLicenseUrlAuditInfo" : null
}, {
"declaredLicense" : "GNU Lesser General Public License",
"licenseUrl" : "http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html",
"declaredLicenseContentKey" : "cf630ee3446f9e10c1b0814fb8a652f4733b491eaea9d5d1f89f803daaaa6075",
"normalizedLicenseType" : "OSS-SPDX",
"normalizedLicense" : "LGPL-2.1",
"normalizedLicenseUrl" : "http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html",
"normalizedLicenseContentKey" : "cf630ee3446f9e10c1b0814fb8a652f4733b491eaea9d5d1f89f803daaaa6075",
"effectiveNormalizedLicenseType" : "OSS-SPDX",
"effectiveNormalizedLicense" : "LGPL-2.1",
"effectiveNormalizedLicenseUrl" : "http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html",
"effectiveNormalizedLicenseContentKey" : "cf630ee3446f9e10c1b0814fb8a652f4733b491eaea9d5d1f89f803daaaa6075",
"legalPreApproved" : "no",
"copyLeft" : "weak",
"licenseCompliance" : "check legal",
"licenseRefUrl" : "https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt",
"licenseRefContentKey" : "7dbd9399a4684cdc5c1c51d6359294caea160c648f1792e273a9489e46321b88",
"includeLicense" : "yes",
"includeSource" : "yes",
"reviewedForRelease" : null,
"comments" : null,
"legalApproved" : "Conditional",
"legalComments" : "OK, in case of dynamic linking.",
"trace" : "+ Component/License info read in '[maven]' format from 'file:./input/licenses_starter.xml'\r\n+ Rule Group: LicenseNameMappingDefaults; RuleId: 101; Matching: licenseUrl==REGEX:https?://www.gnu.org/licenses/old-licenses/lgpl-2.1.*; Setting: normalizedLicenseType=OSS-SPDX, normalizedLicense=LGPL-2.1, normalizedLicenseUrl=http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html (taking data from input)\r\n+ Rule Group: LicenseSelection; RuleId: DEFAULT; Matching: -default-; Setting: effectiveNormalizedLicenseType=OSS-SPDX (taking data from input), effectiveNormalizedLicense=LGPL-2.1 (taking data from input), effectiveNormalizedLicenseUrl=http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html (taking data from input)\r\n+ Rule Group: LegalPreEvaluation; RuleId: 6; Matching: effectiveNormalizedLicenseType==OSS-SPDX, effectiveNormalizedLicense==LGPL-2.1; Setting: legalPreApproved=no, copyLeft=weak, licenseCompliance=check legal, licenseRefUrl=https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt, includeLicense=yes, includeSource=yes\r\n+ Rule Group: LegalEvaluation; RuleId: 14; Matching: usagePattern==DYNAMIC_LINKING, effectiveNormalizedLicenseType==OSS-SPDX, effectiveNormalizedLicense==LGPL-2.1; Setting: legalApproved=Conditional, legalComments=OK, in case of dynamic linking.",
"guessedLicenseUrl" : "http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html",
"guessedLicenseContentKey" : "90fa1cb2ff3629e0f7a3c78dc887134e5e05224de1b13fed1dedb33b1e920724",
"guessedLicenseUrlAuditInfo" : ""
} ],
"rawLicenses" : [ {
"declaredLicense" : "Eclipse Public License - v 1.0",
"licenseUrl" : "http://www.eclipse.org/legal/epl-v10.html",
"declaredLicenseContentKey" : null,
"trace" : "+ Component/License info read in '[maven]' format from 'file:./input/licenses_starter.xml'",
"origin" : "scancode",
"specialHandling" : true
}, {
"declaredLicense" : "GNU Lesser General Public License",
"licenseUrl" : "http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html",
"declaredLicenseContentKey" : null,
"trace" : "+ Component/License info read in '[maven]' format from 'file:./input/licenses_starter.xml'",
"origin" : "scancode",
"specialHandling" : true
} ],
"dataStatus" : "DA:NO_ISSUES",
"traceabilityNotes" : ""
} ]
} ]
},
"textPool" : {
"dataMap" : {
"31cb574375eadadae8835a406879719d85da9fbb5f0f0c6fdc62da741dc49de5" : "Some content of a NOTICE file",
"7dbd9399a4684cdc5c1c51d6359294caea160c648f1792e273a9489e46321b88" : "Even some other license text",
"90fa1cb2ff3629e0f7a3c78dc887134e5e05224de1b13fed1dedb33b1e920724" : "Some license text",
"9d086420212f290c944042014500f00716c30d53f316d224b4cdd0c20756ab6b" : "Yet another license text",
"cf630ee3446f9e10c1b0814fb8a652f4733b491eaea9d5d1f89f803daaaa6075" : "Some other license text"
}
}
}
1 change: 1 addition & 0 deletions documentation/master-solicitor.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -1669,6 +1669,7 @@ Spring beans implementing this interface will be called at certain points in the
== Release Notes
Changes in 1.20.0::
* https://github.com/devonfw/solicitor/issues/232: Set a standard for ordering LicenseNameMapping rules. Rules with an 'or-later' suffix are put before '-only' rules.
* https://github.com/devonfw/solicitor/issues/234: Correct handling of new data model fields in ModelImporterExporter `dataStatus`,`traceabilityNotes` etc.

Changes in 1.19.0::
* https://github.com/devonfw/solicitor/issues/227: Fixed a bug where the `dataStatus` field in the aggregated OSS-Inventory was not filled.
Expand Down

0 comments on commit 1b2368d

Please sign in to comment.