Skip to content

Gradienthealth/file streaming of tar files #10

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Adithyan-Dinesh-Trenser wants to merge 3 commits into
base: gradienthealth/load-dicom-from-tar-file-using-offsets
Choose a base branch
from
Draft

Gradienthealth/file streaming of tar files #10

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
73398a0
Added support for dicomtar scheme
Adithyan-Dinesh-Trenser Oct 21, 2024
459807d
Remove the previous study tar files when another study is loaded and …
Adithyan-Dinesh-Trenser Nov 1, 2024
9a9951f
Changing the request type of precaching next studies to PreCache
Adithyan-Dinesh-Trenser Nov 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion extensions/ohif-gradienthealth-extension/src/DicomJSONDataSource/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,9 @@ const getMetaDataByURL = url => {
const getInstanceUrl = (url, prefix, bucket, bucketPrefix) => {
let modifiedUrl = url;

const schemaPresent = !!url.match(/^(dicomweb:|dicomzip:|wadouri:)/)
const schemaPresent = !!url.match(
/^(dicomweb:|dicomzip:|wadouri:|dicomtar:)/
);
if (!schemaPresent) {
const filePath = url.split('studies/')[1];
modifiedUrl = `dicomweb:https://storage.googleapis.com/${bucket}/${
Expand All @@ -65,6 +67,10 @@ const getInstanceUrl = (url, prefix, bucket, bucketPrefix) => {
? modifiedUrl.replace(dicomwebRegex, 'dicomzip:')
: modifiedUrl;

modifiedUrl = modifiedUrl.includes('.tar://')
? modifiedUrl.replace(dicomwebRegex, 'dicomtar:')
: modifiedUrl;

return modifiedUrl;
}

Expand Down
14 changes: 11 additions & 3 deletions extensions/ohif-gradienthealth-extension/src/services/CacheAPIService/CacheAPIService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ export default class CacheAPIService {
(serie) => !segSOPClassUIDs.includes(serie.instances[0].SOPClassUID)
)
.flatMap((serie) => utils.getImageIdsFromInstances(serie.instances));
await Promise.all([
return await Promise.all([
this.cacheImageIds(imageIds),
this.cacheSegFiles(StudyInstanceUID),
]);
Expand Down Expand Up @@ -172,7 +172,7 @@ export default class CacheAPIService {
}

const priority = 0;
const requestType = Enums.RequestType.Prefetch;
const requestType = Enums.RequestType.PreCache;
const options = {
preScale: {
enabled: true,
Expand All @@ -190,7 +190,15 @@ export default class CacheAPIService {
);
});

await Promise.all(promises)
return new Promise<void>((resolve, reject) => {
const id = setInterval(async () => {
if (promises.length === imageIds.length) {
clearInterval(id);
await Promise.all(promises).catch((error) => reject(error));
resolve();
}
}, 1000);
});
}

public async cacheSegFiles(studyInstanceUID) {
Expand Down
50 changes: 39 additions & 11 deletions ...ons/ohif-gradienthealth-extension/src/services/GoogleSheetsService/GoogleSheetsService.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { eventTarget, Enums, cache } from '@cornerstonejs/core';
import { utilities as csToolsUtils } from '@cornerstonejs/tools';
import { DicomMetadataStore, pubSubServiceInterface, utils } from '@ohif/core';
import { alphabet } from './utils';
import { removeStudyFilesFromCache } from '../utils';

const MAX_ROWS = 100000;

Expand Down Expand Up @@ -50,7 +51,8 @@ export default class GoogleSheetsService {
}

cacheNearbyStudyInstanceUIDs(id, bufferBack, bufferFront) {
const { CacheAPIService } = this.serviceManager.services;
const { CacheAPIService, uiNotificationService } =
this.serviceManager.services;
const index = this.studyUIDToIndex[id];
const min = index - bufferBack < 2 ? 2 : index - bufferBack;
const max = index + bufferFront;
Expand All @@ -64,16 +66,35 @@ export default class GoogleSheetsService {
const element = rowsToCache.splice(indexOfCurrentId, 1);
rowsToCache.unshift(element[0]); // making the current studyid as first element

let hadMaxSizeError = false;
rowsToCache.reduce((promise, row) => {
return promise.then(() => {
const url = row[urlIndex];
const params = new URLSearchParams('?' + url.split('?')[1]);
const StudyInstanceUID = getStudyInstanceUIDFromParams(params);
return CacheAPIService.cacheStudy(
StudyInstanceUID,
params.getAll('bucket')
);
});
return promise
.then(() => {
if (hadMaxSizeError) return Promise.resolve();

const url = row[urlIndex];
const params = new URLSearchParams('?' + url.split('?')[1]);
const StudyInstanceUID = getStudyInstanceUIDFromParams(params);
return CacheAPIService.cacheStudy(
StudyInstanceUID,
params.getAll('bucket')
);
})
.catch((error) => {
if (error.message?.includes('Maximum size')) {
hadMaxSizeError = true;
uiNotificationService.show({
title: 'Maximum size has reached',
message:
error.message ||
'You have reached the maximum size of fetching files for this study.',
type: 'error',
duration: 10000,
});
}

return;
});
}, Promise.resolve());
}

Expand Down Expand Up @@ -320,6 +341,7 @@ export default class GoogleSheetsService {
);

const nextParams = new URLSearchParams(window.location.search);
const prevStudyUID = getStudyInstanceUIDFromParams(nextParams);
if (nextParams.get('StudyInstanceUIDs'))
nextParams.set('StudyInstanceUIDs', StudyInstanceUID);
else {
Expand All @@ -330,6 +352,12 @@ export default class GoogleSheetsService {
nextParams.append('bucket', bucket);
});

if (prevStudyUID !== StudyInstanceUID) {
// Remove the file arraybuffers( currently TAR files ) of the previous
// study when switching to another study.
removeStudyFilesFromCache(prevStudyUID, this.serviceManager);
}

const nextURL =
window.location.href.split('?')[0] + '?' + nextParams.toString();
window.history.replaceState({}, null, nextURL);
Expand Down Expand Up @@ -440,7 +468,7 @@ function loadSegFiles(serviceManager) {
segmentationsOfLoadedImage[0].displaySetInstanceUID
);
});

unsubscribe?.();
}
};
Expand Down
42 changes: 42 additions & 0 deletions extensions/ohif-gradienthealth-extension/src/services/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
// @ts-ignore
import dicomImageLoader from '@cornerstonejs/dicom-image-loader';

export const getSegDisplaysetsOfReferencedImagesIds = (
imageIds: string[] = [],
displaySetService: any
Expand All @@ -11,3 +14,42 @@ export const getSegDisplaysetsOfReferencedImagesIds = (
(ds) => ds.referencedSeriesInstanceUID === referencedSeriesInstanceUID
);
};

export const removeStudyFilesFromCache = (
studyInstanceUID: string,
servicesManager: Record<string, any>
) => {
const { displaySetService } = servicesManager.services;
const studyDisplaySets = displaySetService.getDisplaySetsBy(
(ds) => ds.StudyInstanceUID === studyInstanceUID
);
const urls = studyDisplaySets.flatMap((displaySet) =>
displaySet.instances.reduce((imageIds, instance) => {
const instanceUrl = instance.imageId.split(
/dicomweb:|dicomtar:|dicomzip:/
)[1];
return [...imageIds, ...(instanceUrl ? [instanceUrl] : [])];
}, [])
);

const fileUrls = new Set<string>();
for (const url of urls) {
// Handles .tar files
const urlParts = url.split('.tar');

if (urlParts.length > 1) {
// Adding the '.tar' to the part since spliting with it removes it from the parts.
fileUrls.add(urlParts[0] + '.tar');
}
}

fileUrls.forEach((fileUrl) => {
if (fileUrl.includes('.tar')) {
try {
dicomImageLoader.wadors.tarFileManager.remove(fileUrl);
} catch (error) {
console.warn(error);
}
}
});
};