Correct datatypes for string expressions

src/engine/ExportQueryExecutionTrees.cpp

@@ -14,6 +14,8 @@
 #include "util/ConstexprUtils.h"
 #include "util/http/MediaTypes.h"
 
+using LiteralOrIri = ad_utility::triple_component::LiteralOrIri;
+
 // Return true iff the `result` is nonempty.
 bool getResultForAsk(const std::shared_ptr<const Result>& result) {
   if (result->isFullyMaterialized()) {
@@ -347,11 +349,57 @@
   }
 }
 
+// _____________________________________________________________________________
+std::optional<LiteralOrIri>
+ExportQueryExecutionTrees::idToLiteralOrIriForEncodedValue(
+    Id id, bool onlyReturnLiteralsWithXsdString) {
+  if (onlyReturnLiteralsWithXsdString) {
+    return std::nullopt;
+  }
+  auto optionalStringAndType = idToStringAndTypeForEncodedValue(id);
+  if (!optionalStringAndType) {
+    return std::nullopt;
+  }
+
+  return LiteralOrIri::literalWithoutQuotes(optionalStringAndType->first);
+}
+
+// _____________________________________________________________________________
+bool ExportQueryExecutionTrees::isPlainLiteralOrLiteralWithXsdString(
+    const LiteralOrIri& word) {
+  return !word.hasDatatype() ||
+         asStringViewUnsafe(word.getDatatype()) == XSD_STRING;
+}
+
+// _____________________________________________________________________________
+std::optional<LiteralOrIri> ExportQueryExecutionTrees::handleIriOrLiteral(
+    LiteralOrIri word, bool onlyReturnLiterals,
+    bool onlyReturnLiteralsWithXsdString) {
+  if (!word.isLiteral()) {
+    if (onlyReturnLiterals || onlyReturnLiteralsWithXsdString) {
+      AD_THROW("The input is an IRI, but only literals are allowed.");
+      return std::nullopt;
+    }
+    return word;
+  }
+
+  if (onlyReturnLiteralsWithXsdString) {
+    if (isPlainLiteralOrLiteralWithXsdString(word)) {
+      return word;
+    }
+    return std::nullopt;
+  }
+
+  if (word.hasDatatype() && !isPlainLiteralOrLiteralWithXsdString(word)) {
+    word.getLiteral().removeDatatype();
+  }
+  return word;
+}
+
 // _____________________________________________________________________________
 ad_utility::triple_component::LiteralOrIri
 ExportQueryExecutionTrees::getLiteralOrIriFromVocabIndex(
     const Index& index, Id id, const LocalVocab& localVocab) {
-  using LiteralOrIri = ad_utility::triple_component::LiteralOrIri;
   switch (id.getDatatype()) {
     case Datatype::LocalVocabIndex:
       return localVocab.getWord(id.getLocalVocabIndex()).asLiteralOrIri();
@@ -412,6 +460,39 @@
       return idToStringAndTypeForEncodedValue(id);
   }
 }
+
+// _____________________________________________________________________________
+template <bool onlyReturnLiterals>
+std::optional<LiteralOrIri> ExportQueryExecutionTrees::idToLiteralOrIri(
+    const Index& index, Id id, const LocalVocab& localVocab,
+    bool onlyReturnLiteralsWithXsdString) {
+  using enum Datatype;
+  auto datatype = id.getDatatype();
+
+  if constexpr (onlyReturnLiterals) {
+    if (!(datatype == VocabIndex || datatype == LocalVocabIndex)) {
+      return std::nullopt;
+    }
+  }
+
+  switch (datatype) {
+    case WordVocabIndex:
+      return LiteralOrIri::literalWithoutQuotes(
+          index.indexToString(id.getWordVocabIndex()));
+    case VocabIndex:
+    case LocalVocabIndex:
+      return handleIriOrLiteral(
+          getLiteralOrIriFromVocabIndex(index, id, localVocab),
+          onlyReturnLiterals, onlyReturnLiteralsWithXsdString);
+    case TextRecordIndex:
+      AD_THROW("TextRecordIndex case is not implemented.");
+      return std::nullopt;
+    default:
+      return idToLiteralOrIriForEncodedValue(id,
+                                             onlyReturnLiteralsWithXsdString);
+  }
+}
+
 // ___________________________________________________________________________
 template std::optional<std::pair<std::string, const char*>>
 ExportQueryExecutionTrees::idToStringAndType<true, false, std::identity>(
@@ -433,6 +514,18 @@
                                              const LocalVocab& localVocab,
                                              std::identity&& escapeFunction);
 
+// ___________________________________________________________________________
+template std::optional<LiteralOrIri>
+ExportQueryExecutionTrees::idToLiteralOrIri<false>(
+    const Index& index, Id id, const LocalVocab& localVocab,
+    bool onlyReturnLiteralsWithXsdString);
+
+// ___________________________________________________________________________
+template std::optional<LiteralOrIri>
+ExportQueryExecutionTrees::idToLiteralOrIri<true>(
+    const Index& index, Id id, const LocalVocab& localVocab,
+    bool onlyReturnLiteralsWithXsdString);
+
 // Convert a stringvalue and optional type to JSON binding.
 static nlohmann::json stringAndTypeToBinding(std::string_view entitystr,
                                              const char* xsdType) {

src/engine/ExportQueryExecutionTrees.h

@@ -21,6 +21,7 @@ class ExportQueryExecutionTrees {
  public:
   using MediaType = ad_utility::MediaType;
   using CancellationHandle = ad_utility::SharedCancellationHandle;
+  using LiteralOrIri = ad_utility::triple_component::LiteralOrIri;
 
   // Compute the result of the given `parsedQuery` (created by the
   // `SparqlParser`) for which the `QueryExecutionTree` has been previously
@@ -69,6 +70,37 @@ class ExportQueryExecutionTrees {
   static std::optional<std::pair<std::string, const char*>>
   idToStringAndTypeForEncodedValue(Id id);
 
+  // Convert the `id` to a 'LiteralOrIri. Datatypes are always stripped unless
+  // they are 'xsd:string', so for literals with non-'xsd:string' datatypes
+  // (this includes IDs that directly store their value, like Doubles) the
+  // datatype is always empty. If 'onlyReturnLiteralsWithXsdString' is true, all
+  // IRIs and literals with non'-xsd:string' datatypes (including encoded IDs)
+  // return 'std::nullopt'. These semantics are useful for the string
+  // expressions in StringExpressions.cpp.
+  template <bool returnOnlyLiterals = false>
+  static std::optional<LiteralOrIri> idToLiteralOrIri(
+      const Index& index, Id id, const LocalVocab& localVocab,
+      bool onlyReturnLiteralsWithXsdString = false);
+
+  // Same as the previous function, but only handles the datatypes for which the
+  // value is encoded directly in the ID. For other datatypes an exception is
+  // thrown.
+  // If `onlyReturnLiteralsWithXsdString` is `true`, returns `std::nullopt`.
+  // If `onlyReturnLiteralsWithXsdString` is `false`, removes datatypes from
+  // literals (e.g. the integer `42` is converted to the plain literal `"42"`).
+  static std::optional<LiteralOrIri> idToLiteralOrIriForEncodedValue(
+      Id id, bool onlyReturnLiteralsWithXsdString = false);
+
+  // A helper function for the `idToLiteralOrIri` function. Checks and processes
+  // a LiteralOrIri based on the given parameters.
+  static std::optional<LiteralOrIri> handleIriOrLiteral(
+      LiteralOrIri word, bool onlyReturnLiterals,
+      bool onlyReturnLiteralsWithXsdString);
+
+  // Checks if a LiteralOrIri is either a plain literal (without datatype)
+  // or a literal with the `xsd:string` datatype.
+  static bool isPlainLiteralOrLiteralWithXsdString(const LiteralOrIri& word);
+
   // Acts as a helper to retrieve an LiteralOrIri object
   // from an Id, where the Id is of type `VocabIndex` or `LocalVocabIndex`.
   // This function should only be called with suitable `Datatype` Id's,

src/engine/sparqlExpressions/SparqlExpressionValueGetters.cpp

@@ -90,6 +90,32 @@
   }
 }
 
+// ____________________________________________________________________________
+std::optional<LiteralOrIri> LiteralOrIriValueGetter::operator()(
+    Id id, const EvaluationContext* context) const {
+  return ExportQueryExecutionTrees::idToLiteralOrIri(context->_qec.getIndex(),
+                                                     id, context->_localVocab);
+}
+
+// ____________________________________________________________________________
+std::optional<LiteralOrIri>
+LiteralOrIriValueGetterWithXsdStringFilter::operator()(
+    Id id, const EvaluationContext* context) const {
+  return ExportQueryExecutionTrees::idToLiteralOrIri(
+      context->_qec.getIndex(), id, context->_localVocab, true);
+}
+
+// ____________________________________________________________________________
+std::optional<LiteralOrIri>
+LiteralOrIriValueGetterWithXsdStringFilter::operator()(
+    const LiteralOrIri& s, const EvaluationContext*) const {
+  if (ExportQueryExecutionTrees::isPlainLiteralOrLiteralWithXsdString(s)) {
+    return s;
+  }
+  AD_THROW("Input is not a plain string or xsd:string.");
+  return std::nullopt;
+}
+
 // ____________________________________________________________________________
 template <auto isSomethingFunction, auto prefix>
 Id IsSomethingValueGetter<isSomethingFunction, prefix>::operator()(

src/engine/sparqlExpressions/SparqlExpressionValueGetters.h

@@ -141,6 +141,45 @@
   }
 };
 
+// This class can be used as the `ValueGetter` argument of Expression
+// templates. It produces a LiteralOrIri.
+struct LiteralOrIriValueGetter : Mixin<LiteralOrIriValueGetter> {
+  using Mixin<LiteralOrIriValueGetter>::operator();
+
+  std::optional<LiteralOrIri> operator()(ValueId,
+                                         const EvaluationContext*) const;
+
+  std::optional<LiteralOrIri> operator()(const LiteralOrIri& s,
+                                         const EvaluationContext*) const {
+    return s;
+  }
+};
+
+// Same as above but only literals with 'xsd:string' datatype or no datatype are
+// returned.
+struct LiteralOrIriValueGetterWithXsdStringFilter
+    : Mixin<LiteralOrIriValueGetterWithXsdStringFilter> {
+  using Mixin<LiteralOrIriValueGetterWithXsdStringFilter>::operator();
+
+  std::optional<LiteralOrIri> operator()(ValueId,
+                                         const EvaluationContext*) const;
+
+  std::optional<LiteralOrIri> operator()(const LiteralOrIri& s,
+                                         const EvaluationContext*) const;
+};
+
+// Value getter for `isBlank`.
+struct IsBlankNodeValueGetter : Mixin<IsBlankNodeValueGetter> {
+  using Mixin<IsBlankNodeValueGetter>::operator();
+  Id operator()(ValueId id, const EvaluationContext*) const {
+    return Id::makeFromBool(id.getDatatype() == Datatype::BlankNodeIndex);
+  }
+
+  Id operator()(const LiteralOrIri&, const EvaluationContext*) const {
+    return Id::makeFromBool(false);
+  }
+};
+
 // Boolean value getter that checks whether the given `Id` is a `ValueId` of the
 // given `datatype`.
 template <Datatype datatype>

src/engine/sparqlExpressions/StringExpressions.cpp

@@ -22,6 +22,29 @@
           asNormalizedStringViewUnsafe(normalizedContent))};
 };
 
+// Count UTF-8 characters by skipping continuation bytes (those starting with
+// "10").
+inline std::size_t utf8Length(std::string_view s) {
+  return ql::ranges::count_if(
+      s, [](char c) { return (static_cast<unsigned char>(c) & 0xC0) != 0x80; });
+}
+
+// Convert UTF-8 position to byte offset
+inline std::size_t utf8ToByteOffset(std::string_view str, int64_t utf8Pos) {
+  std::size_t byteOffset = 0;
+  int64_t charCount = 0;
+
+  for (char c : str) {
+    if ((static_cast<unsigned char>(c) & 0xC0) != 0x80) {
+      if (charCount++ == utf8Pos) {
+        break;
+      }
+    }
+    ++byteOffset;
+  }
+  return byteOffset;
+}
+
 // String functions.
 [[maybe_unused]] auto strImpl =
     [](std::optional<std::string> s) -> IdOrLiteralOrIri {
@@ -126,11 +149,7 @@
 
 // STRLEN
 [[maybe_unused]] auto strlen = [](std::string_view s) {
-  // Count UTF-8 characters by skipping continuation bytes (those starting with
-  // "10").
-  auto utf8Len = ql::ranges::count_if(
-      s, [](char c) { return (static_cast<unsigned char>(c) & 0xC0) != 0x80; });
-  return Id::makeFromInt(utf8Len);
+  return Id::makeFromInt(utf8Length(s));
 };
 using StrlenExpression =
     StringExpressionImpl<1, LiftStringFunction<decltype(strlen)>>;
@@ -182,7 +201,7 @@
   };
 
  public:
-  IdOrLiteralOrIri operator()(std::optional<std::string> s, NumericValue start,
+  IdOrLiteralOrIri operator()(std::optional<LiteralOrIri> s, NumericValue start,
                               NumericValue length) const {
     if (!s.has_value() || std::holds_alternative<NotNumeric>(start) ||
         std::holds_alternative<NotNumeric>(length)) {
@@ -202,29 +221,82 @@
     if (startInt < 0) {
       lengthInt += startInt;
     }
-
-    const auto& str = s.value();
+    const auto& str = asStringViewUnsafe(s.value().getContent());
+    std::size_t utf8len = utf8Length(str);
     // Clamp the number such that it is in `[0, str.size()]`. That way we end up
-    // with valid arguments for the `getUTF8Substring` method below for both
+    // with valid arguments for the `setSubstr` method below for both
     // starting position and length since all the other corner cases have been
     // dealt with above.
-    auto clamp = [sz = str.size()](int64_t n) -> std::size_t {
+    auto clamp = [utf8len](int64_t n) -> std::size_t {
       if (n < 0) {
         return 0;
       }
-      if (static_cast<size_t>(n) > sz) {
-        return sz;
+      if (static_cast<size_t>(n) > utf8len) {
+        return utf8len;
       }
       return static_cast<size_t>(n);
     };
 
-    return toLiteral(
-        ad_utility::getUTF8Substring(str, clamp(startInt), clamp(lengthInt)));
+    startInt = clamp(startInt);
+    lengthInt = clamp(lengthInt);
+    std::size_t startByteOffset = utf8ToByteOffset(str, startInt);
+    std::size_t endByteOffset = utf8ToByteOffset(str, startInt + lengthInt);
+    std::size_t byteLength = endByteOffset - startByteOffset;
+
+    s.value().getLiteral().setSubstr(startByteOffset, byteLength);
+    return std::move(s.value());
+  }
+};
+
+// Implementation of the `SUBSTR` SPARQL function. It dynamically
+// selects the appropriate value getter for the first argument based on whether
+// it is a `STR()` expression (using
+// `LiteralOrIriValueGetterWithXsdStringFilter`) or another type (using
+// `LiteralOrIriValueGetter`).
+class SubstrExpressionImpl : public SparqlExpression {
+ private:
+  using ExpressionWithStr =
+      NARY<3, FV<SubstrImpl, LiteralOrIriValueGetterWithXsdStringFilter,
+                 NumericValueGetter, NumericValueGetter>>;
+  using ExpressionWithoutStr =
+      NARY<3, FV<SubstrImpl, LiteralOrIriValueGetter, NumericValueGetter,
+                 NumericValueGetter>>;
+
+  SparqlExpression::Ptr impl_;
+
+ public:
+  explicit SubstrExpressionImpl(
+      SparqlExpression::Ptr child,
+      std::same_as<SparqlExpression::Ptr> auto... children)
+      requires(sizeof...(children) + 1 == 3) {
+    AD_CORRECTNESS_CHECK(child != nullptr);
+
+    if (child->isStrExpression()) {
+      auto childrenOfStr = std::move(*child).moveChildrenOut();
+      AD_CORRECTNESS_CHECK(childrenOfStr.size() == 1);
+      impl_ = std::make_unique<ExpressionWithStr>(
+          std::move(childrenOfStr.at(0)), std::move(children)...);
+    } else {
+      impl_ = std::make_unique<ExpressionWithoutStr>(std::move(child),
+                                                     std::move(children)...);
+    }
+  }
+
+  ExpressionResult evaluate(EvaluationContext* context) const override {
+    return impl_->evaluate(context);
+  }
+
+  std::string getCacheKey(const VariableToColumnMap& varColMap) const override {
+    return impl_->getCacheKey(varColMap);
+  }
+
+ private:
+  std::span<SparqlExpression::Ptr> childrenImpl() override {
+    return impl_->children();
   }
 };
 
-using SubstrExpression =
-    StringExpressionImpl<3, SubstrImpl, NumericValueGetter, NumericValueGetter>;
+using SubstrExpression = SubstrExpressionImpl;
 
 // STRSTARTS
 [[maybe_unused]] auto strStartsImpl = [](std::string_view text,