Skip to content

Commit

Permalink
Merge remote and local QgsEptPointCloudIndex classes
Browse files Browse the repository at this point in the history
  • Loading branch information
@dvdkon
dvdkon committed Oct 30, 2024
1 parent f0a7b63 commit dcc243d
Show file tree
Hide file tree
Showing 9 changed files with 199 additions and 446 deletions.
2 changes: 0 additions & 2 deletions src/core/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2234,13 +2234,11 @@ if (WITH_EPT)
set(QGIS_CORE_SRCS ${QGIS_CORE_SRCS}
providers/ept/qgseptprovider.cpp
pointcloud/qgseptpointcloudindex.cpp
pointcloud/qgsremoteeptpointcloudindex.cpp
pointcloud/qgseptpointcloudblockrequest.cpp
)
set(QGIS_CORE_HDRS ${QGIS_CORE_HDRS}
providers/ept/qgseptprovider.h
pointcloud/qgseptpointcloudindex.h
pointcloud/qgsremoteeptpointcloudindex.h
pointcloud/qgseptpointcloudblockrequest.h
)

Expand Down
255 changes: 184 additions & 71 deletions src/core/pointcloud/qgseptpointcloudindex.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,21 +26,31 @@
#include <QTime>
#include <QtDebug>
#include <QQueue>
#include <qnetworkrequest.h>

#include "qgsapplication.h"
#include "qgsblockingnetworkrequest.h"
#include "qgscachedpointcloudblockrequest.h"
#include "qgseptdecoder.h"
#include "qgseptpointcloudblockrequest.h"
#include "qgslazdecoder.h"
#include "qgscoordinatereferencesystem.h"
#include "qgspointcloudblockrequest.h"
#include "qgspointcloudrequest.h"
#include "qgspointcloudattribute.h"
#include "qgslogger.h"
#include "qgspointcloudexpression.h"
#include "qgssetrequestinitiator_p.h"

///@cond PRIVATE

#define PROVIDER_KEY QStringLiteral( "ept" )
#define PROVIDER_DESCRIPTION QStringLiteral( "EPT point cloud provider" )

QgsEptPointCloudIndex::QgsEptPointCloudIndex() = default;
QgsEptPointCloudIndex::QgsEptPointCloudIndex()
{
mHierarchyNodes.insert( IndexedPointCloudNode( 0, 0, 0, 0 ) );
}

QgsEptPointCloudIndex::~QgsEptPointCloudIndex() = default;

Expand All @@ -52,39 +62,70 @@ std::unique_ptr<QgsPointCloudIndex> QgsEptPointCloudIndex::clone() const
return std::unique_ptr<QgsPointCloudIndex>( clone );
}

void QgsEptPointCloudIndex::load( const QString &fileName )
QList<IndexedPointCloudNode> QgsEptPointCloudIndex::nodeChildren( const IndexedPointCloudNode &n ) const
{
QList<IndexedPointCloudNode> lst;
if ( !loadNodeHierarchy( n ) )
return lst;

const int d = n.d() + 1;
const int x = n.x() * 2;
const int y = n.y() * 2;
const int z = n.z() * 2;

lst.reserve( 8 );
for ( int i = 0; i < 8; ++i )
{
int dx = i & 1, dy = !!( i & 2 ), dz = !!( i & 4 );
const IndexedPointCloudNode n2( d, x + dx, y + dy, z + dz );
if ( loadNodeHierarchy( n2 ) )
lst.append( n2 );
}
return lst;
}

void QgsEptPointCloudIndex::load( const QString &urlString )
{
mUri = fileName;
QFile f( fileName );
if ( !f.open( QIODevice::ReadOnly ) )
QUrl url = urlString;
// Treat non-URLs as local files
if ( !url.isValid() || url.scheme().isEmpty() )
url = QUrl::fromLocalFile( urlString );
mUri = url.toString();

QStringList splitUrl = mUri.split( '/' );

mUrlFileNamePart = splitUrl.back();
splitUrl.pop_back();
mUrlDirectoryPart = splitUrl.join( '/' );

QNetworkRequest nr = QNetworkRequest( QUrl( mUri ) );

QgsBlockingNetworkRequest req;
if ( req.get( nr ) != QgsBlockingNetworkRequest::NoError )
{
mError = tr( "Unable to open %1 for reading" ).arg( fileName );
QgsDebugError( QStringLiteral( "Request failed: " ) + mUri );
mIsValid = false;
mError = req.errorMessage();
return;
}

const QDir directory = QFileInfo( fileName ).absoluteDir();
mDirectory = directory.absolutePath();

const QByteArray dataJson = f.readAll();
bool success = loadSchema( dataJson );
const QgsNetworkReplyContent reply = req.reply();
bool success = loadSchema( reply.content() );

if ( success )
{
// try to import the metadata too!
QFile manifestFile( mDirectory + QStringLiteral( "/ept-sources/manifest.json" ) );
if ( manifestFile.open( QIODevice::ReadOnly ) )
QUrl manifestUrl( mUrlDirectoryPart + QStringLiteral( "/ept-sources/manifest.json" ) );

QNetworkRequest nr = QNetworkRequest( QUrl( manifestUrl ) );
QgsBlockingNetworkRequest req;
if ( req.get( nr ) == QgsBlockingNetworkRequest::NoError )
{
const QByteArray manifestJson = manifestFile.readAll();
const QByteArray manifestJson = req.reply().content();
loadManifest( manifestJson );
}
}

if ( success )
{
success = loadHierarchy();
}

mIsValid = success;
}

Expand All @@ -101,10 +142,12 @@ void QgsEptPointCloudIndex::loadManifest( const QByteArray &manifestJson )
{
const QJsonObject sourceObject = manifestArray.at( 0 ).toObject();
const QString metadataPath = sourceObject.value( QStringLiteral( "metadataPath" ) ).toString();
QFile metadataFile( mDirectory + QStringLiteral( "/ept-sources/" ) + metadataPath );
if ( metadataFile.open( QIODevice::ReadOnly ) )
QUrl metadataUrl( mUrlDirectoryPart + QStringLiteral( "/ept-sources/" ) + metadataPath );
QNetworkRequest nr = QNetworkRequest( QUrl( metadataUrl ) );
QgsBlockingNetworkRequest req;
if ( req.get( nr ) == QgsBlockingNetworkRequest::NoError )
{
const QByteArray metadataJson = metadataFile.readAll();
const QByteArray metadataJson = req.reply().content();
const QJsonDocument metadataDoc = QJsonDocument::fromJson( metadataJson, &err );
if ( err.error == QJsonParseError::NoError )
{
Expand Down Expand Up @@ -315,47 +358,65 @@ std::unique_ptr<QgsPointCloudBlock> QgsEptPointCloudIndex::nodeData( const Index
return std::unique_ptr<QgsPointCloudBlock>( cached );
}

mHierarchyMutex.lock();
const bool found = mHierarchy.contains( n );
mHierarchyMutex.unlock();
if ( !found )
std::unique_ptr<QgsPointCloudBlockRequest> blockRequest( asyncNodeData( n, request ) );
if ( !blockRequest )
return nullptr;

// we need to create a copy of the expression to pass to the decoder
// as the same QgsPointCloudExpression object mighgt be concurrently
// used on another thread, for example in a 3d view
QgsPointCloudExpression filterExpression = mFilterExpression;
QgsPointCloudAttributeCollection requestAttributes = request.attributes();
requestAttributes.extend( attributes(), filterExpression.referencedAttributes() );
QgsRectangle filterRect = request.filterRect();
QEventLoop loop;
connect( blockRequest.get(), &QgsPointCloudBlockRequest::finished, &loop, &QEventLoop::quit );
loop.exec();

std::unique_ptr<QgsPointCloudBlock> block = blockRequest->takeBlock();
if ( !block )
{
QgsDebugError( QStringLiteral( "Error downloading node %1 data, error : %2 " ).arg( n.toString(), blockRequest->errorStr() ) );
}

storeNodeDataToCache( block.get(), n, request );
return block;
}

QgsPointCloudBlockRequest *QgsEptPointCloudIndex::asyncNodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request )
{
if ( QgsPointCloudBlock *cached = getNodeDataFromCache( n, request ) )
{
return new QgsCachedPointCloudBlockRequest( cached, n, mUri, attributes(), request.attributes(),
scale(), offset(), mFilterExpression, request.filterRect() );
}

if ( !loadNodeHierarchy( n ) )
return nullptr;

std::unique_ptr<QgsPointCloudBlock> decoded;
QString fileUrl;
if ( mDataType == QLatin1String( "binary" ) )
{
const QString filename = QStringLiteral( "%1/ept-data/%2.bin" ).arg( mDirectory, n.toString() );
decoded = QgsEptDecoder::decompressBinary( filename, attributes(), requestAttributes, scale(), offset(), filterExpression, filterRect );
fileUrl = QStringLiteral( "%1/ept-data/%2.bin" ).arg( mUrlDirectoryPart, n.toString() );
}
else if ( mDataType == QLatin1String( "zstandard" ) )
{
const QString filename = QStringLiteral( "%1/ept-data/%2.zst" ).arg( mDirectory, n.toString() );
decoded = QgsEptDecoder::decompressZStandard( filename, attributes(), request.attributes(), scale(), offset(), filterExpression, filterRect );
fileUrl = QStringLiteral( "%1/ept-data/%2.zst" ).arg( mUrlDirectoryPart, n.toString() );
}
else if ( mDataType == QLatin1String( "laszip" ) )
{
const QString filename = QStringLiteral( "%1/ept-data/%2.laz" ).arg( mDirectory, n.toString() );
decoded = QgsLazDecoder::decompressLaz( filename, requestAttributes, filterExpression, filterRect );
fileUrl = QStringLiteral( "%1/ept-data/%2.laz" ).arg( mUrlDirectoryPart, n.toString() );
}
else
{
return nullptr;
}

storeNodeDataToCache( decoded.get(), n, request );
return decoded;
// we need to create a copy of the expression to pass to the decoder
// as the same QgsPointCloudExpression object might be concurrently
// used on another thread, for example in a 3d view
QgsPointCloudExpression filterExpression = mFilterExpression;
QgsPointCloudAttributeCollection requestAttributes = request.attributes();
requestAttributes.extend( attributes(), filterExpression.referencedAttributes() );
return new QgsEptPointCloudBlockRequest( n, fileUrl, mDataType, attributes(), requestAttributes, scale(), offset(), filterExpression, request.filterRect() );
}

QgsPointCloudBlockRequest *QgsEptPointCloudIndex::asyncNodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request )
bool QgsEptPointCloudIndex::hasNode( const IndexedPointCloudNode &n ) const
{
Q_UNUSED( n );
Q_UNUSED( request );
Q_ASSERT( false );
return nullptr; // unsupported
return loadNodeHierarchy( n );
}

QgsCoordinateReferenceSystem QgsEptPointCloudIndex::crs() const
Expand All @@ -368,6 +429,14 @@ qint64 QgsEptPointCloudIndex::pointCount() const
return mPointCount;
}

int QgsEptPointCloudIndex::nodePointCount( const IndexedPointCloudNode &n )
{
// First load the node, so the superclass method can read it
if ( !loadNodeHierarchy( n ) )
return -1;
return QgsPointCloudIndex::nodePointCount( n );
}

bool QgsEptPointCloudIndex::hasStatisticsMetadata() const
{
return !mMetadataStats.isEmpty();
Expand Down Expand Up @@ -439,28 +508,63 @@ QVariant QgsEptPointCloudIndex::metadataClassStatistic( const QString &attribute
return values.value( value.toInt() );
}

bool QgsEptPointCloudIndex::loadHierarchy()
bool QgsEptPointCloudIndex::loadNodeHierarchy( const IndexedPointCloudNode &nodeId ) const
{
QQueue<QString> queue;
queue.enqueue( QStringLiteral( "0-0-0-0" ) );
while ( !queue.isEmpty() )
mHierarchyMutex.lock();
bool found = mHierarchy.contains( nodeId );
mHierarchyMutex.unlock();
if ( found )
return true;

QVector<IndexedPointCloudNode> nodePathToRoot;
{
const QString filename = QStringLiteral( "%1/ept-hierarchy/%2.json" ).arg( mDirectory, queue.dequeue() );
QFile fH( filename );
if ( !fH.open( QIODevice::ReadOnly ) )
IndexedPointCloudNode currentNode = nodeId;
do
{
QgsDebugMsgLevel( QStringLiteral( "unable to read hierarchy from file %1" ).arg( filename ), 2 );
mError = QStringLiteral( "unable to read hierarchy from file %1" ).arg( filename );
nodePathToRoot.push_back( currentNode );
currentNode = currentNode.parentNode();
}
while ( currentNode.d() >= 0 );
}

for ( int i = nodePathToRoot.size() - 1; i >= 0 && !mHierarchy.contains( nodeId ); --i )
{
const IndexedPointCloudNode node = nodePathToRoot[i];
//! The hierarchy of the node is found => No need to load its file
mHierarchyMutex.lock();
const bool foundInHierarchy = mHierarchy.contains( node );
const bool foundInHierarchyNodes = mHierarchyNodes.contains( node );
mHierarchyMutex.unlock();
if ( foundInHierarchy )
continue;

if ( !foundInHierarchyNodes )
continue;

const QString fileUrl = QStringLiteral( "%1/ept-hierarchy/%2.json" ).arg( mUrlDirectoryPart, node.toString() );
QNetworkRequest nr( fileUrl );
QgsSetRequestInitiatorClass( nr, QStringLiteral( "QgsEptPointCloudIndex" ) );
nr.setAttribute( QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache );
nr.setAttribute( QNetworkRequest::CacheSaveControlAttribute, true );

std::unique_ptr<QgsTileDownloadManagerReply> reply( QgsApplication::tileDownloadManager()->get( nr ) );

QEventLoop loop;
connect( reply.get(), &QgsTileDownloadManagerReply::finished, &loop, &QEventLoop::quit );
loop.exec();

if ( reply->error() != QNetworkReply::NoError )
{
QgsDebugError( QStringLiteral( "Request failed: " ) + mUri );
return false;
}

const QByteArray dataJsonH = fH.readAll();
const QByteArray dataJsonH = reply->data();
QJsonParseError errH;
const QJsonDocument docH = QJsonDocument::fromJson( dataJsonH, &errH );
if ( errH.error != QJsonParseError::NoError )
{
QgsDebugMsgLevel( QStringLiteral( "QJsonParseError when reading hierarchy from file %1" ).arg( filename ), 2 );
mError = QStringLiteral( "QJsonParseError when reading hierarchy from file %1" ).arg( filename );
QgsDebugMsgLevel( QStringLiteral( "QJsonParseError when reading hierarchy from file %1" ).arg( fileUrl ), 2 );
return false;
}

Expand All @@ -469,36 +573,45 @@ bool QgsEptPointCloudIndex::loadHierarchy()
{
const QString nodeIdStr = it.key();
const int nodePointCount = it.value().toInt();
if ( nodePointCount < 0 )
{
queue.enqueue( nodeIdStr );
}
else
{
const IndexedPointCloudNode nodeId = IndexedPointCloudNode::fromString( nodeIdStr );
mHierarchyMutex.lock();
const IndexedPointCloudNode nodeId = IndexedPointCloudNode::fromString( nodeIdStr );
mHierarchyMutex.lock();
if ( nodePointCount >= 0 )
mHierarchy[nodeId] = nodePointCount;
mHierarchyMutex.unlock();
}
else if ( nodePointCount == -1 )
mHierarchyNodes.insert( nodeId );
mHierarchyMutex.unlock();
}
}
return true;

mHierarchyMutex.lock();
found = mHierarchy.contains( nodeId );
mHierarchyMutex.unlock();

return found;
}


bool QgsEptPointCloudIndex::isValid() const
{
return mIsValid;
}

QgsPointCloudIndex::AccessType QgsEptPointCloudIndex::accessType() const
{
return QUrl( mUri ).scheme() == "file" ? AccessType::Local : AccessType::Remote;
}

void QgsEptPointCloudIndex::copyCommonProperties( QgsEptPointCloudIndex *destination ) const
{
QgsPointCloudIndex::copyCommonProperties( destination );

// QgsEptPointCloudIndex specific fields
destination->mIsValid = mIsValid;
destination->mDataType = mDataType;
destination->mDirectory = mDirectory;
destination->mUrlDirectoryPart = mUrlDirectoryPart;
destination->mUrlFileNamePart = mUrlFileNamePart;
destination->mWkt = mWkt;
destination->mHierarchyNodes = mHierarchyNodes;
destination->mPointCount = mPointCount;
destination->mMetadataStats = mMetadataStats;
destination->mAttributeClasses = mAttributeClasses;
Expand Down
Loading

0 comments on commit dcc243d

Please sign in to comment.