Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ENH: Virtualize DICOM database #1154

Merged
merged 10 commits into from
Dec 13, 2023
Merged

ENH: Virtualize DICOM database #1154

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
3b8182e
ENH: allow URL "filenames" in database
pieper Oct 25, 2023
fbae32f
ENH: Always use upper case hex for tags
pieper Nov 5, 2023
03e6ecc
ENH: applying DICOM display updates even if some fail
pieper Nov 5, 2023
ff515f9
ENH: use fileName in dicom db only for files
pieper Nov 13, 2023
5200de9
ENH: Updates dicom db schema to support urls
pieper Nov 13, 2023
8f9f14a
BUG: Fix dicom db insert with URL field
pieper Nov 14, 2023
0c47c6a
ENH: Allow file or url to be null in dicom-schema.sql
pieper Nov 14, 2023
d6be887
ENH: update dicom db schema and api for files and urls
pieper Nov 14, 2023
2bfc42a
Merge branch 'master' into virtualize-database
pieper Nov 16, 2023
df63274
ENH: Add index for URL to dicom database schema
pieper Dec 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions Libs/DICOM/Core/Resources/dicom-schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,12 @@ DROP INDEX IF EXISTS 'SeriesStudyIndex' ;
DROP INDEX IF EXISTS 'StudiesPatientIndex' ;

CREATE TABLE 'SchemaInfo' ( 'Version' VARCHAR(1024) NOT NULL );
INSERT INTO 'SchemaInfo' VALUES('0.7.0');
INSERT INTO 'SchemaInfo' VALUES('0.8.0');

CREATE TABLE 'Images' (
'SOPInstanceUID' VARCHAR(64) NOT NULL,
'Filename' VARCHAR(1024) NOT NULL ,
'Filename' VARCHAR(1024) NULL,
'URL' VARCHAR(2048) NULL,
'SeriesInstanceUID' VARCHAR(64) NOT NULL ,
'InsertTimestamp' VARCHAR(20) NOT NULL ,
'DisplayedFieldsUpdatedTimestamp' DATETIME NULL ,
Expand Down Expand Up @@ -84,7 +85,7 @@ CREATE TABLE 'Series' (
'DisplayedFieldsUpdatedTimestamp' DATETIME NULL ,
PRIMARY KEY ('SeriesInstanceUID') );

CREATE UNIQUE INDEX IF NOT EXISTS 'ImagesFilenameIndex' ON 'Images' ('Filename');
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should the URL be indexed as well ?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmmm, maybe, since that is used in the fileValue method to find the instanceUID so it can be used to look up values in the tag cache. Bypassing the tag cache is still probably better, but an index on this might have addressed some of the performance issues I was seeing.

CREATE INDEX IF NOT EXISTS 'ImagesFilenameIndex' ON 'Images' ('Filename');
CREATE INDEX IF NOT EXISTS 'ImagesSeriesIndex' ON 'Images' ('SeriesInstanceUID');
CREATE INDEX IF NOT EXISTS 'SeriesStudyIndex' ON 'Series' ('StudyInstanceUID');
CREATE INDEX IF NOT EXISTS 'StudiesPatientIndex' ON 'Studies' ('PatientsUID');
Expand Down
142 changes: 112 additions & 30 deletions Libs/DICOM/Core/ctkDICOMDatabase.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ ctkDICOMDatabasePrivate::ctkDICOMDatabasePrivate(ctkDICOMDatabase& o)
, UseShortStoragePath(true)
, ThumbnailGenerator(nullptr)
, TagCacheVerified(false)
, SchemaVersion("0.7.0")
, SchemaVersion("0.8.0")
{
this->resetLastInsertedValues();
this->DisplayedFieldGenerator = new ctkDICOMDisplayedFieldGenerator(q_ptr);
Expand Down Expand Up @@ -235,11 +235,12 @@ QString ctkDICOMDatabasePrivate::readValueFromFile(const QString& fileName, cons
return "";
}

QString upperTag = tag.toUpper();
QString value;
unsigned short group, element;
q->tagToGroupElement(tag, group, element);
q->tagToGroupElement(upperTag, group, element);
DcmTagKey tagKey(group, element);
if (this->TagsToExcludeFromStorage.contains(tag))
if (this->TagsToExcludeFromStorage.contains(upperTag))
{
if (dataset.TagExists(tagKey))
{
Expand All @@ -256,7 +257,7 @@ QString ctkDICOMDatabasePrivate::readValueFromFile(const QString& fileName, cons
}

// Store result in cache
q->cacheTag(sopInstanceUID, tag, value);
q->cacheTag(sopInstanceUID, upperTag, value);

return value;
}
Expand Down Expand Up @@ -562,11 +563,12 @@ void ctkDICOMDatabasePrivate::precacheTags(const ctkDICOMItem& dataset, const QS
QStringList sopInstanceUIDs, tags, values;
foreach (const QString &tag, this->TagsToPrecache)
{
QString upperTag = tag.toUpper();
unsigned short group, element;
q->tagToGroupElement(tag, group, element);
q->tagToGroupElement(upperTag, group, element);
DcmTagKey tagKey(group, element);
QString value;
if (this->TagsToExcludeFromStorage.contains(tag))
if (this->TagsToExcludeFromStorage.contains(upperTag))
{
if (dataset.TagExists(tagKey))
{
Expand All @@ -582,7 +584,7 @@ void ctkDICOMDatabasePrivate::precacheTags(const ctkDICOMItem& dataset, const QS
value = dataset.GetAllElementValuesAsString(tagKey);
}
sopInstanceUIDs << sopInstanceUID;
tags << tag;
tags << upperTag;
values << value;
}

Expand Down Expand Up @@ -963,9 +965,10 @@ void ctkDICOMDatabasePrivate::insert(const ctkDICOMItem& dataset, const QString&
}

QSqlQuery insertImageStatement(Database);
insertImageStatement.prepare("INSERT INTO Images ( 'SOPInstanceUID', 'Filename', 'SeriesInstanceUID', 'InsertTimestamp' ) VALUES ( ?, ?, ?, ? )");
insertImageStatement.prepare("INSERT INTO Images ( 'SOPInstanceUID', 'Filename', 'URL', 'SeriesInstanceUID', 'InsertTimestamp' ) VALUES ( ?, ?, ?, ?, ? )");
insertImageStatement.addBindValue(sopInstanceUID);
insertImageStatement.addBindValue(storedFilePathInDatabase);
insertImageStatement.addBindValue(QString(""));
insertImageStatement.addBindValue(seriesInstanceUID);
insertImageStatement.addBindValue(QDateTime::currentDateTime());
insertImageStatement.exec();
Expand Down Expand Up @@ -1076,11 +1079,12 @@ void ctkDICOMDatabase::insert(const QList<ctkDICOMDatabase::IndexingResult>& ind
insertTags.bindValue(0, sopInstanceUID);
foreach(const QString & tag, d->TagsToPrecache)
{
QString upperTag = tag.toUpper();
unsigned short group, element;
this->tagToGroupElement(tag, group, element);
this->tagToGroupElement(upperTag, group, element);
DcmTagKey tagKey(group, element);
QString value;
if (d->TagsToExcludeFromStorage.contains(tag))
if (d->TagsToExcludeFromStorage.contains(upperTag))
{
if (dataset.TagExists(tagKey))
{
Expand All @@ -1095,7 +1099,7 @@ void ctkDICOMDatabase::insert(const QList<ctkDICOMDatabase::IndexingResult>& ind
{
value = dataset.GetAllElementValuesAsString(tagKey);
}
insertTags.bindValue(1, tag);
insertTags.bindValue(1, upperTag);
if (value.isEmpty())
{
insertTags.bindValue(2, TagNotInInstance);
Expand All @@ -1109,9 +1113,10 @@ void ctkDICOMDatabase::insert(const QList<ctkDICOMDatabase::IndexingResult>& ind

// Insert image files
QSqlQuery insertImageStatement(d->Database);
insertImageStatement.prepare("INSERT INTO Images ( 'SOPInstanceUID', 'Filename', 'SeriesInstanceUID', 'InsertTimestamp' ) VALUES ( ?, ?, ?, ? )");
insertImageStatement.prepare("INSERT INTO Images ( 'SOPInstanceUID', 'Filename', 'URL', 'SeriesInstanceUID', 'InsertTimestamp' ) VALUES ( ?, ?, ?, ?, ? )");
insertImageStatement.addBindValue(sopInstanceUID);
insertImageStatement.addBindValue(d->internalPathFromAbsolute(storedFilePath));
insertImageStatement.addBindValue(QString(""));
insertImageStatement.addBindValue(seriesInstanceUID);
insertImageStatement.addBindValue(QDateTime::currentDateTime());
insertImageStatement.exec();
Expand Down Expand Up @@ -1259,7 +1264,7 @@ QString ctkDICOMDatabasePrivate::getDisplaySeriesFieldsKey(QString seriesInstanc
return seriesInstanceUID;
}

logger.error("Failed to find series with SeriesInstanceUID=" + seriesInstanceUID);
logger.error("in getDisplaySeriesFieldsKey: Failed to find series with SeriesInstanceUID=" + seriesInstanceUID);
return QString();
}

Expand Down Expand Up @@ -1375,8 +1380,8 @@ bool ctkDICOMDatabasePrivate::applyDisplayedFieldsChanges( QMap<QString, QMap<QS
}
else
{
logger.error("Failed to find study with StudyInstanceUID=" + currentStudyInstanceUid);
return false;
logger.error("in applyDisplayedFieldsChanges: Failed to find study with StudyInstanceUID=" + currentStudyInstanceUid);
continue;
}
} // For each study in displayedFieldsMapStudy

Expand Down Expand Up @@ -1421,8 +1426,8 @@ bool ctkDICOMDatabasePrivate::applyDisplayedFieldsChanges( QMap<QString, QMap<QS
}
else
{
logger.error("Failed to find series with SeriesInstanceUID=" + currentSeriesInstanceUid);
return false;
logger.error("in applyDisplayedFieldsChanges: Failed to find series with SeriesInstanceUID=" + currentSeriesInstanceUid);
continue;
}
} // For each series in displayedFieldsMapSeries

Expand Down Expand Up @@ -1459,7 +1464,7 @@ ctkDICOMDatabase::ctkDICOMDatabase(QString databaseFile)
: d_ptr(new ctkDICOMDatabasePrivate(*this))
{
Q_D(ctkDICOMDatabase);
d->TagsToExcludeFromStorage << groupElementToTag(0x7fe0,0x0010); // pixel data
d->TagsToExcludeFromStorage << groupElementToTag(0x7FE0,0x0010); // pixel data
d->registerCompressionLibraries();
d->init(databaseFile);
}
Expand All @@ -1469,7 +1474,7 @@ ctkDICOMDatabase::ctkDICOMDatabase(QObject* parent)
{
Q_UNUSED(parent);
Q_D(ctkDICOMDatabase);
d->TagsToExcludeFromStorage << groupElementToTag(0x7fe0, 0x0010); // pixel data
d->TagsToExcludeFromStorage << groupElementToTag(0x7FE0, 0x0010); // pixel data
d->registerCompressionLibraries();
}

Expand Down Expand Up @@ -2094,7 +2099,9 @@ QStringList ctkDICOMDatabase::filesForSeries(QString seriesUID, int hits/*=-1*/)
QStringList allFileNames;
while (query.next())
{
allFileNames << d->absolutePathFromInternal(query.value(0).toString());
QString fileName = query.value(0).toString();
fileName = d->absolutePathFromInternal(fileName);
allFileNames << fileName;
if (hits > 0 && allFileNames.size() >= hits)
{
// reached the number of requested files
Expand All @@ -2104,6 +2111,28 @@ QStringList ctkDICOMDatabase::filesForSeries(QString seriesUID, int hits/*=-1*/)
return allFileNames;
}

//------------------------------------------------------------------------------
QStringList ctkDICOMDatabase::urlsForSeries(QString seriesUID, int hits/*=-1*/)
{
Q_D(ctkDICOMDatabase);
QSqlQuery query(d->Database);
query.prepare("SELECT URL FROM Images WHERE SeriesInstanceUID=?");
query.addBindValue(seriesUID);
query.exec();
QStringList allURLs;
while (query.next())
{
QString url = query.value(0).toString();
allURLs << url;
if (hits > 0 && allURLs.size() >= hits)
{
// reached the number of requested files
break;
}
}
return allURLs;
}

//------------------------------------------------------------------------------
QString ctkDICOMDatabase::fileForInstance(QString sopInstanceUID)
{
Expand All @@ -2120,6 +2149,22 @@ QString ctkDICOMDatabase::fileForInstance(QString sopInstanceUID)
return result;
}

//------------------------------------------------------------------------------
QString ctkDICOMDatabase::urlForInstance(QString sopInstanceUID)
{
Q_D(ctkDICOMDatabase);
QSqlQuery query(d->Database);
query.prepare("SELECT URL FROM Images WHERE SOPInstanceUID=?");
query.addBindValue(sopInstanceUID);
query.exec();
QString result;
if (query.next())
{
result = query.value(0).toString();
}
return result;
}

//------------------------------------------------------------------------------
QString ctkDICOMDatabase::seriesForFile(QString fileName)
{
Expand Down Expand Up @@ -2152,6 +2197,22 @@ QString ctkDICOMDatabase::instanceForFile(QString fileName)
return result;
}

//------------------------------------------------------------------------------
QString ctkDICOMDatabase::instanceForURL(QString url)
{
Q_D(ctkDICOMDatabase);
QSqlQuery query(d->Database);
query.prepare( "SELECT SOPInstanceUID FROM Images WHERE URL=?");
query.addBindValue(url);
query.exec();
QString result;
if (query.next())
{
result = query.value(0).toString();
}
return result;
}

//------------------------------------------------------------------------------
QDateTime ctkDICOMDatabase::insertDateTimeForInstance(QString sopInstanceUID)
{
Expand Down Expand Up @@ -2239,6 +2300,7 @@ void ctkDICOMDatabase::loadFileHeader(QString fileName)
if (dO)
{
QString tag = QString("%1,%2").arg(dO->getGTag(),4,16,QLatin1Char('0')).arg(dO->getETag(),4,16,QLatin1Char('0'));
tag = tag.toUpper();
std::ostringstream s;
dO->print(s);
d->LoadedHeader[tag] = QString(s.str().c_str());
Expand Down Expand Up @@ -2270,6 +2332,7 @@ QString ctkDICOMDatabase::headerValue (QString key)
QString ctkDICOMDatabase::instanceValue(QString sopInstanceUID, QString tag)
{
Q_D(ctkDICOMDatabase);
tag = tag.toUpper();
// Read from cache, if available
QString value = this->cachedTag(sopInstanceUID, tag);
if (value == TagNotInInstance || value == ValueIsEmptyString || value == ValueIsNotStored)
Expand Down Expand Up @@ -2304,7 +2367,18 @@ QString ctkDICOMDatabase::fileValue(const QString fileName, QString tag)
Q_D(ctkDICOMDatabase);

// Read from cache, if available

// first, try treating argument as filePath
QString sopInstanceUID = this->instanceForFile(fileName);

// second, try treating argument as a url
if (sopInstanceUID.isEmpty())
{
sopInstanceUID = this->instanceForURL(fileName);
}

// third, look for the value
tag = tag.toUpper();
QString value = this->cachedTag(sopInstanceUID, tag);
if (value == TagNotInInstance || value == ValueIsEmptyString || value == ValueIsNotStored)
{
Expand All @@ -2315,7 +2389,7 @@ QString ctkDICOMDatabase::fileValue(const QString fileName, QString tag)
return value;
}

// Read value from file
// Read value from file as a fallback (won't work if fileName is a URL)
value = d->readValueFromFile(fileName, sopInstanceUID, tag);
return value;
}
Expand All @@ -2332,7 +2406,8 @@ QString ctkDICOMDatabase::fileValue(const QString fileName, const unsigned short
bool ctkDICOMDatabase::instanceValueExists(const QString sopInstanceUID, const QString tag)
{
Q_D(ctkDICOMDatabase);
QString value = this->cachedTag(sopInstanceUID, tag);
QString upperTag = tag.toUpper();
QString value = this->cachedTag(sopInstanceUID, upperTag);
if (value == TagNotInInstance || value == ValueIsEmptyString)
{
return false;
Expand All @@ -2348,7 +2423,7 @@ bool ctkDICOMDatabase::instanceValueExists(const QString sopInstanceUID, const Q
{
return false;
}
value = d->readValueFromFile(filePath, sopInstanceUID, tag);
value = d->readValueFromFile(filePath, sopInstanceUID, upperTag);
return (value != TagNotInInstance && value != ValueIsEmptyString);
}

Expand All @@ -2363,6 +2438,7 @@ bool ctkDICOMDatabase::instanceValueExists(const QString sopInstanceUID, const u
bool ctkDICOMDatabase::fileValueExists(const QString fileName, QString tag)
{
Q_D(ctkDICOMDatabase);
tag = tag.toUpper();
QString sopInstanceUID = this->instanceForFile(fileName);
QString value = this->cachedTag(sopInstanceUID, tag);
if (value == TagNotInInstance || value == ValueIsEmptyString)
Expand Down Expand Up @@ -2405,7 +2481,8 @@ bool ctkDICOMDatabase::tagToGroupElement(const QString tag, unsigned short& grou
//------------------------------------------------------------------------------
QString ctkDICOMDatabase::groupElementToTag(const unsigned short& group, const unsigned short& element)
{
return QString("%1,%2").arg(group,4,16,QLatin1Char('0')).arg(element,4,16,QLatin1Char('0'));
QString groupElement = QString("%1,%2").arg(group,4,16,QLatin1Char('0')).arg(element,4,16,QLatin1Char('0'));
return groupElement.toUpper();
}

//
Expand Down Expand Up @@ -2507,11 +2584,16 @@ const QStringList ctkDICOMDatabase::tagsToPrecache()
void ctkDICOMDatabase::setTagsToExcludeFromStorage(const QStringList tags)
{
Q_D(ctkDICOMDatabase);
if (d->TagsToExcludeFromStorage == tags)
QStringList upperTags;
foreach (const QString &tag, tags)
{
upperTags << tag.toUpper();
}
if (d->TagsToExcludeFromStorage == upperTags)
{
return;
}
d->TagsToExcludeFromStorage = tags;
d->TagsToExcludeFromStorage = upperTags;
emit tagsToExcludeFromStorageChanged();
}

Expand Down Expand Up @@ -2839,7 +2921,7 @@ QString ctkDICOMDatabase::cachedTag(const QString sopInstanceUID, const QString
QSqlQuery selectValue( d->TagCacheDatabase );
selectValue.prepare( "SELECT Value FROM TagCache WHERE SOPInstanceUID = :sopInstanceUID AND Tag = :tag" );
selectValue.bindValue(":sopInstanceUID",sopInstanceUID);
selectValue.bindValue(":tag",tag);
selectValue.bindValue(":tag",tag.toUpper());
d->loggedExec(selectValue);
QString result("");
if (selectValue.next())
Expand Down Expand Up @@ -2874,7 +2956,7 @@ void ctkDICOMDatabase::getCachedTags(const QString sopInstanceUID, QMap<QString,
QString value;
while (selectValue.next())
{
tag = selectValue.value(0).toString();
tag = selectValue.value(0).toString().toUpper();
value = selectValue.value(1).toString();
if (value == TagNotInInstance || value == ValueIsEmptyString || value == ValueIsNotStored)
{
Expand All @@ -2889,7 +2971,7 @@ bool ctkDICOMDatabase::cacheTag(const QString sopInstanceUID, const QString tag,
{
QStringList sopInstanceUIDs, tags, values;
sopInstanceUIDs << sopInstanceUID;
tags << tag;
tags << tag.toUpper();
values << value;
return this->cacheTags(sopInstanceUIDs, tags, values);
}
Expand Down Expand Up @@ -2925,7 +3007,7 @@ bool ctkDICOMDatabase::cacheTags(const QStringList sopInstanceUIDs, const QStrin
for (int i = 0; i<itemCount; ++i)
{
insertTags.bindValue(0, *sopInstanceUIDsIt);
insertTags.bindValue(1, *tagsIt);
insertTags.bindValue(1, (*tagsIt).toUpper());
if (valuesIt->isEmpty())
{
// replace empty strings with special flag string
Expand Down
Loading
Loading