Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add public datasets under "Hub" #3129

Merged
merged 39 commits into from
Jan 22, 2025
Merged
Show file tree
Hide file tree
Changes from 31 commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
553ce82
update
GspikeHalo Nov 30, 2024
79b9292
update
GspikeHalo Nov 30, 2024
a74ff54
update
GspikeHalo Nov 30, 2024
ded8e02
update
GspikeHalo Nov 30, 2024
715128d
update
GspikeHalo Dec 1, 2024
9b06756
update
GspikeHalo Dec 1, 2024
39f1d44
Merge branch 'master' into add-public-dataset
GspikeHalo Dec 14, 2024
2ea84d9
Add dynamic navigation for list items. Restrict non-logged-in users f…
GspikeHalo Dec 14, 2024
710d07e
Clean up and simplify the code
GspikeHalo Dec 15, 2024
e952e60
Merge branch 'master' into add-public-dataset
GspikeHalo Dec 16, 2024
72877f4
Merge branch 'master' into add-public-dataset
GspikeHalo Dec 17, 2024
217e4d4
update dataset
GspikeHalo Dec 18, 2024
5503407
quick fix
GspikeHalo Dec 18, 2024
2c5c79a
quick fix
GspikeHalo Dec 18, 2024
129d25e
Merge branch 'master' into add-public-dataset
GspikeHalo Dec 18, 2024
32ed48d
Merge branch 'master' into add-public-dataset
GspikeHalo Dec 18, 2024
83728ff
Merge branch 'master' into add-public-dataset
GspikeHalo Jan 4, 2025
428069b
Merge branch 'master' into add-public-dataset
GspikeHalo Jan 7, 2025
fc0e057
Merge branch 'master' into add-public-dataset
GspikeHalo Jan 8, 2025
2687c56
Merge branch 'master' into add-public-dataset
GspikeHalo Jan 9, 2025
fa71a10
Merge branch 'master' into add-public-dataset
GspikeHalo Jan 9, 2025
7873603
move "workflowUserAccess" from DashboardResource.scala to WorkflowRes…
GspikeHalo Jan 10, 2025
20c57aa
Merge branch 'master' into add-public-dataset
GspikeHalo Jan 11, 2025
b27e746
Use Jiadong's dataset program
GspikeHalo Jan 17, 2025
d225763
Merge branch 'master' into add-public-dataset
GspikeHalo Jan 17, 2025
c012f54
finish merging
GspikeHalo Jan 17, 2025
fe4dbf2
Clean up and simplify the code
GspikeHalo Jan 17, 2025
00550fc
Resolve some minor issues
GspikeHalo Jan 17, 2025
170c127
Rename hub-dataset-result.component.ts to hub-dataset-search.componen…
GspikeHalo Jan 17, 2025
fd1aa66
quick fix
GspikeHalo Jan 17, 2025
6343238
Merge branch 'master' into add-public-dataset
GspikeHalo Jan 19, 2025
5bf9c01
quick fix
GspikeHalo Jan 19, 2025
603a883
rename user-dataset-explorer.component.ts to dataset-detail.component.ts
GspikeHalo Jan 19, 2025
986d752
Remove the extra router-outlet
GspikeHalo Jan 19, 2025
1b1dd68
merge public dataset and public workflow search result
GspikeHalo Jan 19, 2025
73e28e9
Resolve some minor issues
GspikeHalo Jan 19, 2025
a054922
quick fix
GspikeHalo Jan 19, 2025
71cbbf5
Merge branch 'master' into add-public-dataset
GspikeHalo Jan 20, 2025
862456c
Merge branch 'master' into add-public-dataset
GspikeHalo Jan 22, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -202,18 +202,4 @@ class DashboardResource {

userIdToInfoMap
}

@GET
@Path("/workflowUserAccess")
def workflowUserAccess(
GspikeHalo marked this conversation as resolved.
Show resolved Hide resolved
@QueryParam("wid") wid: UInteger
): util.List[UInteger] = {
val records = context
.select(WORKFLOW_USER_ACCESS.UID)
.from(WORKFLOW_USER_ACCESS)
.where(WORKFLOW_USER_ACCESS.WID.eq(wid))
.fetch()

records.getValues(WORKFLOW_USER_ACCESS.UID)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,15 @@ object DatasetAccessResource {
.getInstance(StorageConfig.jdbcUrl, StorageConfig.jdbcUsername, StorageConfig.jdbcPassword)
.createDSLContext()

def userHasReadAccess(ctx: DSLContext, did: UInteger, uid: UInteger): Boolean = {
def isDatasetPublic(ctx: DSLContext, did: UInteger): Boolean = {
val datasetDao = new DatasetDao(ctx.configuration())
val isDatasetPublic = Option(datasetDao.fetchOneByDid(did))
Option(datasetDao.fetchOneByDid(did))
.flatMap(dataset => Option(dataset.getIsPublic))
.contains(1.toByte)
}

isDatasetPublic ||
def userHasReadAccess(ctx: DSLContext, did: UInteger, uid: UInteger): Boolean = {
isDatasetPublic(ctx, did) ||
userHasWriteAccess(ctx, did, uid) ||
getDatasetUserAccessPrivilege(ctx, did, uid) == DatasetUserAccessPrivilege.READ
}
Expand Down Expand Up @@ -92,7 +94,7 @@ class DatasetAccessResource {
@GET
@Path("/owner/{did}")
def getOwnerEmailOfDataset(@PathParam("did") did: UInteger): String = {
var email = "";
var email = ""
withTransaction(context) { ctx =>
val owner = getOwner(ctx, did)
if (owner != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,6 @@ object DatasetResource {
}

@Produces(Array(MediaType.APPLICATION_JSON, "image/jpeg", "application/pdf"))
@RolesAllowed(Array("REGULAR", "ADMIN"))
@Path("/dataset")
class DatasetResource {
private val ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE = "User has no read access to this dataset"
Expand All @@ -359,20 +358,27 @@ class DatasetResource {
private def getDashboardDataset(
ctx: DSLContext,
did: UInteger,
uid: UInteger
uid: Option[UInteger],
isPublic: Boolean = false
): DashboardDataset = {
if (!userHasReadAccess(ctx, did, uid)) {
if (
(isPublic && !isDatasetPublic(ctx, did)) ||
(!isPublic && (!userHasReadAccess(ctx, did, uid.get)))
) {
throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE)
}

val targetDataset = getDatasetByID(ctx, did)
val userAccessPrivilege = getDatasetUserAccessPrivilege(ctx, did, uid)
val userAccessPrivilege =
if (isPublic) DatasetUserAccessPrivilege.NONE
else getDatasetUserAccessPrivilege(ctx, did, uid.get)
val isOwner = !isPublic && (targetDataset.getOwnerUid == uid.get)

DashboardDataset(
targetDataset,
getOwner(ctx, did).getEmail,
userAccessPrivilege,
targetDataset.getOwnerUid == uid,
isOwner,
List(),
calculateDatasetVersionSize(did)
)
Expand Down Expand Up @@ -401,6 +407,7 @@ class DatasetResource {
}

@POST
@RolesAllowed(Array("REGULAR", "ADMIN"))
@Path("/create")
@Consumes(Array(MediaType.MULTIPART_FORM_DATA))
def createDataset(
Expand Down Expand Up @@ -477,6 +484,7 @@ class DatasetResource {
}

@POST
@RolesAllowed(Array("REGULAR", "ADMIN"))
@Path("/delete")
def deleteDataset(datasetIDs: DatasetIDs, @Auth user: SessionUser): Response = {
val uid = user.getUid
Expand All @@ -501,6 +509,7 @@ class DatasetResource {
@POST
@Consumes(Array(MediaType.APPLICATION_JSON))
@Produces(Array(MediaType.APPLICATION_JSON))
@RolesAllowed(Array("REGULAR", "ADMIN"))
@Path("/update/name")
def updateDatasetName(
modificator: DatasetNameModification,
Expand All @@ -525,6 +534,7 @@ class DatasetResource {
@POST
@Consumes(Array(MediaType.APPLICATION_JSON))
@Produces(Array(MediaType.APPLICATION_JSON))
@RolesAllowed(Array("REGULAR", "ADMIN"))
@Path("/update/description")
def updateDatasetDescription(
modificator: DatasetDescriptionModification,
Expand All @@ -548,6 +558,7 @@ class DatasetResource {
}

@POST
@RolesAllowed(Array("REGULAR", "ADMIN"))
@Path("/{did}/update/publicity")
def toggleDatasetPublicity(
@PathParam("did") did: UInteger,
Expand All @@ -574,6 +585,7 @@ class DatasetResource {
}

@POST
@RolesAllowed(Array("REGULAR", "ADMIN"))
@Path("/{did}/version/create")
@Consumes(Array(MediaType.MULTIPART_FORM_DATA))
def createDatasetVersion(
Expand Down Expand Up @@ -607,6 +619,7 @@ class DatasetResource {
* @return list of user accessible DashboardDataset objects
*/
@GET
@RolesAllowed(Array("REGULAR", "ADMIN"))
@Path("")
def listDatasets(
@Auth user: SessionUser
Expand Down Expand Up @@ -684,28 +697,36 @@ class DatasetResource {
}

@GET
@RolesAllowed(Array("REGULAR", "ADMIN"))
@Path("/{did}/version/list")
def getDatasetVersionList(
@PathParam("did") did: UInteger,
@Auth user: SessionUser
): List[DatasetVersion] = {
val uid = user.getUid
withTransaction(context)(ctx => {

if (!userHasReadAccess(ctx, did, uid)) {
throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE)
}
val result: java.util.List[DatasetVersion] = ctx
.selectFrom(DATASET_VERSION)
.where(DATASET_VERSION.DID.eq(did))
.orderBy(DATASET_VERSION.CREATION_TIME.desc()) // or .asc() for ascending
.fetchInto(classOf[DatasetVersion])
fetchDatasetVersions(ctx, did)
})
}

result.asScala.toList
@GET
@Path("/{did}/publicVersion/list")
def getPublicDatasetVersionList(
@PathParam("did") did: UInteger
): List[DatasetVersion] = {
withTransaction(context)(ctx => {
if (!isDatasetPublic(ctx, did)) {
throw new ForbiddenException(ERR_USER_HAS_NO_ACCESS_TO_DATASET_MESSAGE)
}
fetchDatasetVersions(ctx, did)
})
}

@GET
@RolesAllowed(Array("REGULAR", "ADMIN"))
@Path("/{did}/version/latest")
def retrieveLatestDatasetVersion(
@PathParam("did") did: UInteger,
Expand Down Expand Up @@ -753,65 +774,53 @@ class DatasetResource {
}

@GET
@RolesAllowed(Array("REGULAR", "ADMIN"))
@Path("/{did}/version/{dvid}/rootFileNodes")
def retrieveDatasetVersionRootFileNodes(
@PathParam("did") did: UInteger,
@PathParam("dvid") dvid: UInteger,
@Auth user: SessionUser
): DatasetVersionRootFileNodesResponse = {
val uid = user.getUid
withTransaction(context)(ctx =>
fetchDatasetVersionRootFileNodes(ctx, did, dvid, Some(uid), isPublic = false)
)
}

withTransaction(context)(ctx => {
val dataset = getDashboardDataset(ctx, did, uid)
val targetDatasetPath = PathUtils.getDatasetPath(did)
val datasetVersion = getDatasetVersionByID(ctx, dvid)
val datasetName = dataset.dataset.getName
val fileNodes = GitVersionControlLocalFileStorage.retrieveRootFileNodesOfVersion(
targetDatasetPath,
datasetVersion.getVersionHash
)
val versionHash = getDatasetVersionByID(ctx, dvid).getVersionHash
val size = calculateDatasetVersionSize(did, Some(versionHash))
val ownerFileNode = DatasetFileNode
.fromPhysicalFileNodes(
Map((dataset.ownerEmail, datasetName, datasetVersion.getName) -> fileNodes.asScala.toList)
)
.head

DatasetVersionRootFileNodesResponse(
ownerFileNode.children.get
.find(_.getName == datasetName)
.head
.children
.get
.find(_.getName == datasetVersion.getName)
.head
.children
.get,
size
)
})
@GET
@Path("/{did}/publicVersion/{dvid}/rootFileNodes")
def retrievePublicDatasetVersionRootFileNodes(
@PathParam("did") did: UInteger,
@PathParam("dvid") dvid: UInteger
): DatasetVersionRootFileNodesResponse = {
withTransaction(context)(ctx =>
fetchDatasetVersionRootFileNodes(ctx, did, dvid, None, isPublic = true)
)
}

@GET
@RolesAllowed(Array("REGULAR", "ADMIN"))
@Path("/{did}")
def getDataset(
@PathParam("did") did: UInteger,
@Auth user: SessionUser
): DashboardDataset = {
val uid = user.getUid
withTransaction(context)(ctx => {
val dashboardDataset = getDashboardDataset(ctx, did, uid)
val size = calculateDatasetVersionSize(did)
dashboardDataset.copy(size = size)
})
withTransaction(context)(ctx => fetchDataset(ctx, did, Some(uid), isPublic = false))
}

@GET
@Path("/public/{did}")
def getPublicDataset(
@PathParam("did") did: UInteger
): DashboardDataset = {
withTransaction(context)(ctx => fetchDataset(ctx, did, None, isPublic = true))
}

@GET
@Path("/file")
def retrieveDatasetSingleFile(
@QueryParam("path") pathStr: String,
@Auth user: SessionUser
@QueryParam("path") pathStr: String
): Response = {
val decodedPathStr = URLDecoder.decode(pathStr, StandardCharsets.UTF_8.name())

Expand Down Expand Up @@ -863,6 +872,7 @@ class DatasetResource {
* @return A Response containing the dataset version as a ZIP file.
*/
@GET
@RolesAllowed(Array("REGULAR", "ADMIN"))
@Path("/version-zip")
def retrieveDatasetVersionZip(
@QueryParam("did") did: UInteger,
Expand Down Expand Up @@ -935,4 +945,77 @@ class DatasetResource {
.`type`("application/zip")
.build()
}

@GET
@Path("/datasetUserAccess")
def workflowUserAccess(
GspikeHalo marked this conversation as resolved.
Show resolved Hide resolved
@QueryParam("did") did: UInteger
): java.util.List[UInteger] = {
val records = context
.select(DATASET_USER_ACCESS.UID)
.from(DATASET_USER_ACCESS)
.where(DATASET_USER_ACCESS.DID.eq(did))
.fetch()

records.getValues(DATASET_USER_ACCESS.UID)
}

private def fetchDatasetVersions(ctx: DSLContext, did: UInteger): List[DatasetVersion] = {
ctx
.selectFrom(DATASET_VERSION)
.where(DATASET_VERSION.DID.eq(did))
.orderBy(DATASET_VERSION.CREATION_TIME.desc()) // Change to .asc() for ascending order
.fetchInto(classOf[DatasetVersion])
.asScala
.toList
}

private def fetchDatasetVersionRootFileNodes(
ctx: DSLContext,
did: UInteger,
dvid: UInteger,
uid: Option[UInteger],
isPublic: Boolean
): DatasetVersionRootFileNodesResponse = {
val dataset = getDashboardDataset(ctx, did, uid, isPublic)
val targetDatasetPath = PathUtils.getDatasetPath(did)
val datasetVersion = getDatasetVersionByID(ctx, dvid)
val datasetName = dataset.dataset.getName
val fileNodes = GitVersionControlLocalFileStorage.retrieveRootFileNodesOfVersion(
targetDatasetPath,
datasetVersion.getVersionHash
)
val versionHash = datasetVersion.getVersionHash
val size = calculateDatasetVersionSize(did, Some(versionHash))

val ownerFileNode = DatasetFileNode
.fromPhysicalFileNodes(
Map((dataset.ownerEmail, datasetName, datasetVersion.getName) -> fileNodes.asScala.toList)
)
.head

DatasetVersionRootFileNodesResponse(
ownerFileNode.children.get
.find(_.getName == datasetName)
.head
.children
.get
.find(_.getName == datasetVersion.getName)
.head
.children
.get,
size
)
}

private def fetchDataset(
ctx: DSLContext,
did: UInteger,
uid: Option[UInteger],
isPublic: Boolean
): DashboardDataset = {
val dashboardDataset = getDashboardDataset(ctx, did, uid, isPublic)
val size = calculateDatasetVersionSize(did)
dashboardDataset.copy(size = size)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import edu.uci.ics.texera.dao.jooq.generated.tables.daos.{
WorkflowUserAccessDao
}
import edu.uci.ics.texera.dao.jooq.generated.tables.pojos._
import edu.uci.ics.texera.web.resource.dashboard.SearchQueryBuilder.context
import edu.uci.ics.texera.web.resource.dashboard.hub.workflow.HubWorkflowResource.recordUserActivity
import edu.uci.ics.texera.web.resource.dashboard.user.workflow.WorkflowAccessResource.hasReadAccess
import edu.uci.ics.texera.web.resource.dashboard.user.workflow.WorkflowResource._
Expand Down Expand Up @@ -590,4 +591,18 @@ class WorkflowResource extends LazyLogging {
"Private"
}
}

@GET
@Path("/workflowUserAccess")
def workflowUserAccess(
@QueryParam("wid") wid: UInteger
): util.List[UInteger] = {
val records = context
.select(WORKFLOW_USER_ACCESS.UID)
.from(WORKFLOW_USER_ACCESS)
.where(WORKFLOW_USER_ACCESS.WID.eq(wid))
.fetch()

records.getValues(WORKFLOW_USER_ACCESS.UID)
}
}
Loading
Loading