Skip to content
This repository has been archived by the owner on Jul 23, 2024. It is now read-only.

Support access urls and curating node #17

Merged
merged 2 commits into from
Jun 11, 2024
Merged

Support access urls and curating node #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dc67e2e
Support access urls and curating node
jordanpadams May 30, 2024
4f2da3c
Update to support multiple resources
jordanpadams May 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 8 additions & 10 deletions src/main/java/gov/nasa/pds/citool/CIToolIngester.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,13 @@
package gov.nasa.pds.citool;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;
import gov.nasa.pds.citool.comparator.CatalogComparator;
import gov.nasa.pds.citool.ingestor.CatalogObject;
import gov.nasa.pds.citool.ingestor.CatalogVolumeIngester;
Expand All @@ -9,22 +17,12 @@
import gov.nasa.pds.citool.util.ReferenceUtils;
import gov.nasa.pds.citool.util.References;
import gov.nasa.pds.citool.util.RegistryObjectCache;

import gov.nasa.pds.tools.LabelParserException;
import gov.nasa.pds.tools.constants.Constants.ProblemType;
import gov.nasa.pds.tools.label.Label;
import gov.nasa.pds.tools.label.ManualPathResolver;
import gov.nasa.pds.tools.label.parser.DefaultLabelParser;

import java.net.URI;
import java.net.URL;
import java.net.URISyntaxException;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.util.logging.Logger;
import java.io.IOException;


public class CIToolIngester
{
Expand Down
28 changes: 24 additions & 4 deletions src/main/java/gov/nasa/pds/citool/ingestor/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package gov.nasa.pds.citool.ingestor;

import java.util.Map;
import java.util.HashMap;
import java.util.Map;
/**
* Class that holds constants used in ingest.
*
Expand All @@ -12,6 +12,8 @@ public class Constants {

public static final String LID_PREFIX = "urn:nasa:pds:context_pds3:";

public static final String RESOURCE_LID_SUFFIX = "_resource_info";

public static final String TARGET_PROD = "Product_Target_PDS3";

public static final String MISSION_PROD = "Product_Mission_PDS3";
Expand All @@ -25,7 +27,7 @@ public class Constants {
public static final String GUEST_PROD = "Product_PDS_Guest";

public static final String AFFIL_PROD = "Product_PDS_Affiliate";

// or Collection_Volume_Set_PDS3 ????
public static final String VOLUME_PROD = "Product_Volume_PDS3";

Expand Down Expand Up @@ -65,11 +67,11 @@ public class Constants {
public static final String HAS_TARGET = "target_ref";

public static final String HAS_RESOURCE = "resource_ref";

public static final String HAS_FILE = "file_ref";

public static final String HAS_NODE = "node_ref";

public static final String PRODUCT_VERSION = "version_id";


Expand All @@ -94,4 +96,22 @@ public class Constants {
pds3ToPds4Map.put("MISSION_ALIAS_NAME", "alternate_title");
pds3ToPds4Map.put("RESOURCE_LINK", "resource_url");
}

/** Mapping of PDS3 to PDS4 names */
public static final Map<String, String> nodeValueToIdMap = new HashMap<String, String>();

static {
nodeValueToIdMap.put("atmos", "planetary atmospheres");
nodeValueToIdMap.put("geoscience", "geosciences");
nodeValueToIdMap.put("isas/jaxa", "jaxa");
nodeValueToIdMap.put("naif", "navigation and ancillary information facility");
nodeValueToIdMap.put("ppi", "planetary plasma interactions");
nodeValueToIdMap.put("ppi-ucla", "planetary plasma interactions");
nodeValueToIdMap.put("rings", "planetary rings");
nodeValueToIdMap.put("sbn", "small bodies");
nodeValueToIdMap.put("esa", "planetary science archive");
nodeValueToIdMap.put("rs", "radio science");
nodeValueToIdMap.put("rad", "radio science");
nodeValueToIdMap.put("imaging-jpl", "imaging");
}
}
42 changes: 38 additions & 4 deletions src/main/java/gov/nasa/pds/citool/ingestor/ProductFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.Map.Entry;

import java.util.Set;
import java.util.logging.Logger;
import gov.nasa.pds.citool.file.FileObject;
import gov.nasa.pds.citool.registry.model.FileInfo;
import gov.nasa.pds.citool.registry.model.Metadata;
Expand All @@ -20,14 +20,17 @@

public class ProductFactory
{

private static Logger log = Logger.getLogger(ProductFactory.class.getName());

/**
* Create an extrinsic object
*
* @param catObj a catalog object
* @return an extrinsic object
*
*/
public static RegistryObject createProduct(CatalogObject catObj, CatalogVolumeIngester ingester)
public static RegistryObject createProduct(CatalogObject catObj, CatalogVolumeIngester ingester)
throws Exception
{
RegistryObject product = new RegistryObject();
Expand Down Expand Up @@ -167,7 +170,31 @@ else if (objType.equalsIgnoreCase(Constants.VOLUME_OBJ) && key.equals("VOLUME_ID
slots.put("name", product.getName());

slots.put(getKey("product_class"), Constants.VOLUME_PROD);
}
} else if (objType.equalsIgnoreCase(Constants.HK_OBJ)
&& key.equals("CURATING_NODE_ID")) {

String dsId = md.getMetadata("DATA_SET_ID");
String productLid = dsId + Constants.RESOURCE_LID_SUFFIX;
productLid = Utility.collapse(productLid);
productLid = Utility.replaceChars(productLid);
productLid = Constants.LID_PREFIX + "resource:resource." + productLid.toLowerCase();

product.setLid(productLid);
slots.put("lid", productLid);

product.setObjectType(Constants.RESOURCE_PROD);
slots.put("objectType", product.getObjectType());

slots.put(getKey("product_class"), Constants.RESOURCE_PROD);

String node_id = md.getMetadata("CURATING_NODE_ID");
node_id = Utility.collapse(node_id);
slots.put("node_id", getNodeId(node_id.toLowerCase()));

slots.put("resource_name", md.getAllMetadata("RESOURCE_NAME"));
slots.put("resource_url", md.getAllMetadata("RESOURCE_LINK"));

}

if (objType.equalsIgnoreCase(Constants.DATASET_OBJ) && ingester.getArchiveStatus() !=null)
{
Expand Down Expand Up @@ -356,4 +383,11 @@ private static String getKey(String key)
return key.toLowerCase();
}

private static String getNodeId(String value) {
if (Constants.nodeValueToIdMap.containsKey(value)) {
return Constants.nodeValueToIdMap.get(value);
}
return value;
}

}
61 changes: 8 additions & 53 deletions src/main/java/gov/nasa/pds/citool/util/ReferenceUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;

import gov.nasa.pds.citool.CIToolIngester;
import gov.nasa.pds.citool.ingestor.CatalogObject;
import gov.nasa.pds.citool.ingestor.CatalogVolumeIngester;
import gov.nasa.pds.citool.ingestor.Constants;
Expand Down Expand Up @@ -302,57 +300,14 @@ private static void handleResourceRefs(Map<String, List<String>> refs, CatalogOb
values = new ArrayList<String>();
}

String dsId = pdsLbl.get("DATA_SET_ID").getValue().toString();
String key = "RESOURCE_ID";
if (md.containsKey(key)) {
if (md.isMultiValued(key)) {
List<String> tmpValues = md.getAllMetadata(key);
for (String aVal: tmpValues) {
lidValue = dsId + "__" + aVal;
lidValue = Utility.collapse(lidValue);
lidValue = Utility.replaceChars(lidValue);
lidValue = lidValue.toLowerCase();
if (!Utility.valueExists(Constants.LID_PREFIX+"resource:resource."+lidValue, values))
values.add(Constants.LID_PREFIX+"resource:resource."+lidValue);
}
}
else {
lidValue = md.getMetadata(key);
lidValue = dsId + "__" + lidValue;
lidValue = Utility.collapse(lidValue);
lidValue = Utility.replaceChars(lidValue);
lidValue = lidValue.toLowerCase();
if (!Utility.valueExists(Constants.LID_PREFIX+"resource:resource."+lidValue, values))
values.add(Constants.LID_PREFIX+"resource:resource."+lidValue);
}
refs.put(Constants.HAS_RESOURCE, values);
}

key = "CURATING_NODE_ID";
if (md.containsKey(key)) {
if (refs.get(Constants.HAS_NODE) != null) {
values = refs.get(Constants.HAS_NODE);
} else {
values = new ArrayList<String>();
}
if (md.isMultiValued(key)) {
List<String> tmpValues = md.getAllMetadata(key);
for (String aVal : tmpValues) {
lidValue = aVal;
lidValue = Utility.collapse(lidValue);
lidValue = lidValue.toLowerCase();
if (!Utility.valueExists(Constants.LID_PREFIX + "node:node." + lidValue, values))
values.add(Constants.LID_PREFIX + "node:node." + lidValue);
}
} else {
lidValue = md.getMetadata(key);
lidValue = Utility.collapse(lidValue);
lidValue = lidValue.toLowerCase();
if (!Utility.valueExists(Constants.LID_PREFIX + "node:node." + lidValue, values))
values.add(Constants.LID_PREFIX + "node:node." + lidValue);
}
refs.put(Constants.HAS_NODE, values);
}
lidValue = pdsLbl.get("DATA_SET_ID").getValue().toString() + Constants.RESOURCE_LID_SUFFIX;
lidValue = Utility.collapse(lidValue);
lidValue = Utility.replaceChars(lidValue);
lidValue = lidValue.toLowerCase();
if (!Utility.valueExists(Constants.LID_PREFIX + "resource:resource." + lidValue, values))
values.add(Constants.LID_PREFIX + "resource:resource." + lidValue);

refs.put(Constants.HAS_RESOURCE, values);
}


Expand Down
9 changes: 6 additions & 3 deletions src/main/resources/search-conf/defaults/pds/dataset.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,11 +108,14 @@
<registryPath>target_ref.target_type</registryPath>
</field>

<field name="resource_ref" type="string">
<registryPath>resource_ref</registryPath>
<field name="resource_url" type="string">
<registryPath>resource_ref.resource_url</registryPath>
</field>
<field name="resource_name" type="string">
<registryPath>resource_ref.resource_name</registryPath>
</field>
<field name="node_id" type="string">
<registryPath>node_ref.node_name</registryPath>
<registryPath>resource_ref.node_id</registryPath>
</field>

<field name="external_reference_text" type="string">
Expand Down
Loading