From c5df40ea4a2e60454fd206adcfb7fedb960c20cf Mon Sep 17 00:00:00 2001 From: Dmytro Polityka Date: Mon, 2 Sep 2024 16:54:12 +0200 Subject: [PATCH 001/113] add cleanup service skeleton --- .../cleanup/OldDataCleanUpRepository.java | 20 +++ .../cleanup/OldDataCleanUpRepositoryImpl.java | 150 ++++++++++++++++++ .../cleanup/OldDataCleanupService.java | 39 +++++ .../admin/AdminOldDataCleanUpResource.java | 52 ++++++ 4 files changed, 261 insertions(+) create mode 100644 src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepository.java create mode 100644 src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepositoryImpl.java create mode 100644 src/main/java/de/tum/in/www1/artemis/service/cleanup/OldDataCleanupService.java create mode 100644 src/main/java/de/tum/in/www1/artemis/web/rest/admin/AdminOldDataCleanUpResource.java diff --git a/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepository.java b/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepository.java new file mode 100644 index 000000000000..4d4ce4738aab --- /dev/null +++ b/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepository.java @@ -0,0 +1,20 @@ +package de.tum.in.www1.artemis.repository.cleanup; + +import java.time.ZonedDateTime; + +public interface OldDataCleanUpRepository { + + void deleteOrphans(); + + void deletePlagiarismComparisons(ZonedDateTime keepAllLaterThan); + + void deleteNonRatedResults(ZonedDateTime keepAllLaterThan); + + void deleteOldRatedResults(ZonedDateTime keepAllLaterThan); + + void deleteOldSubmissionVersions(ZonedDateTime keepAllLaterThan); + + void deleteOldFeedback(ZonedDateTime keepAllLaterThan); + + boolean existsDataForCleanup(ZonedDateTime keepAllLaterThan); +} diff --git a/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepositoryImpl.java b/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepositoryImpl.java new file mode 100644 index 000000000000..b879de21d7f9 --- /dev/null +++ b/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepositoryImpl.java @@ -0,0 +1,150 @@ +package de.tum.in.www1.artemis.repository.cleanup; + +import java.time.ZonedDateTime; + +import jakarta.persistence.EntityManager; +import jakarta.persistence.PersistenceContext; + +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +@Repository +public class OldDataCleanUpRepositoryImpl implements OldDataCleanUpRepository { + + // transactinal ok, because of delete statements + + @PersistenceContext + private EntityManager entityManager; + + @Override + @Transactional + public void deleteOrphans() { + entityManager.createQuery("DELETE FROM Feedback f WHERE f.result IS NULL").executeUpdate(); + entityManager.createQuery("DELETE FROM LongFeedbackText lft WHERE lft.feedback IS NULL").executeUpdate(); + entityManager.createQuery("DELETE FROM ParticipantScore ps WHERE ps.team IS NULL AND ps.user IS NULL").executeUpdate(); + entityManager.createQuery("DELETE FROM ResultRating rr WHERE rr.result IS NULL").executeUpdate(); + entityManager.createQuery("DELETE FROM AssessmentNote an WHERE an.result IS NULL").executeUpdate(); + entityManager.createQuery("DELETE FROM Result r WHERE r.participation IS NULL AND r.submission IS NULL").executeUpdate(); + entityManager.createQuery("DELETE FROM PlagiarismComparison pc WHERE pc.submissionA IS NULL AND pc.submissionB IS NULL").executeUpdate(); + } + + @Override + @Transactional + public void deletePlagiarismComparisons(ZonedDateTime keepAllLaterThan) { + entityManager + .createQuery("DELETE FROM PlagiarismComparison pc " + "WHERE pc.id IN (" + " SELECT pc.id " + " FROM PlagiarismComparison pc " + " JOIN pc.submissionA s1 " + + " JOIN pc.submissionB s2 " + " JOIN s1.participation p1 " + " JOIN s2.participation p2 " + " JOIN p1.exercise e1 " + + " JOIN p2.exercise e2 " + " JOIN e1.course c1 " + " JOIN e2.course c2 " + " WHERE GREATEST(c1.endDate, c2.endDate) < :keepAllLaterThan" + ")") + .setParameter("keepAllLaterThan", keepAllLaterThan).executeUpdate(); + } + + @Override + @Transactional + public void deleteNonRatedResults(ZonedDateTime keepAllLaterThan) { + entityManager.createQuery("DELETE FROM Result r " + "WHERE r.rated = false " + "AND r.participation.exercise.course.endDate < :keepAllLaterThan") + .setParameter("keepAllLaterThan", keepAllLaterThan).executeUpdate(); + } + + @Override + @Transactional + public void deleteOldRatedResults(ZonedDateTime keepAllLaterThan) { + entityManager + .createQuery("DELETE FROM Result r " + "WHERE r.id IN (" + " SELECT r.id " + " FROM Result r " + " WHERE r.rated = true " + + " AND r.participation.exercise.course.endDate < :keepAllLaterThan " + " AND r.id NOT IN (" + " SELECT r1.id " + " FROM Result r1 " + + " WHERE r1.participation = r.participation " + " ORDER BY r1.completionDate DESC " + " LIMIT 1" + " )" + ")") + .setParameter("keepAllLaterThan", keepAllLaterThan).executeUpdate(); + } + + @Override + @Transactional + public void deleteOldSubmissionVersions(ZonedDateTime keepAllLaterThan) { + entityManager.createQuery("DELETE FROM SubmissionVersion sv " + "WHERE sv.submission.id IN (" + " SELECT s.id " + " FROM Submission s " + + " JOIN s.participation p " + " JOIN p.exercise e " + " JOIN e.exerciseGroup eg " + " JOIN eg.exam ex " + " JOIN ex.course c " + + " WHERE c.endDate < :keepAllLaterThan" + ")").setParameter("keepAllLaterThan", keepAllLaterThan).executeUpdate(); + } + + @Override + @Transactional + public void deleteOldFeedback(ZonedDateTime keepAllLaterThan) { + entityManager + .createQuery("DELETE FROM Feedback f " + "WHERE f.result.id NOT IN (" + " SELECT MAX(r.id) " + " FROM Result r " + + " WHERE r.participation = f.result.participation" + ") " + "AND f.result.participation.exercise.course.endDate < :keepAllLaterThan") + .setParameter("keepAllLaterThan", keepAllLaterThan).executeUpdate(); + } + + @Override + @Transactional + public boolean existsDataForCleanup(ZonedDateTime keepAllLaterThan) { + Long count = (Long) entityManager.createQuery(""" + SELECT + CASE + WHEN (SELECT COUNT(f) FROM Feedback f WHERE f.result IS NULL) > 0 THEN 1 + WHEN (SELECT COUNT(lft) FROM LongFeedbackText lft WHERE lft.feedback IS NULL) > 0 THEN 1 + WHEN (SELECT COUNT(ps) FROM ParticipantScore ps WHERE ps.team IS NULL AND ps.user IS NULL) > 0 THEN 1 + WHEN (SELECT COUNT(rr) FROM ResultRating rr WHERE rr.result IS NULL) > 0 THEN 1 + WHEN (SELECT COUNT(an) FROM AssessmentNote an WHERE an.result IS NULL) > 0 THEN 1 + WHEN (SELECT COUNT(r) FROM Result r WHERE r.participation IS NULL AND r.submission IS NULL) > 0 THEN 1 + WHEN (SELECT COUNT(pc) FROM PlagiarismComparison pc WHERE pc.submissionA IS NULL AND pc.submissionB IS NULL) > 0 THEN 1 + WHEN ( + SELECT COUNT(pc) + FROM PlagiarismComparison pc + JOIN pc.submissionA s1 + JOIN pc.submissionB s2 + JOIN s1.participation p1 + JOIN s2.participation p2 + JOIN p1.exercise e1 + JOIN p2.exercise e2 + JOIN e1.course c1 + JOIN e2.course c2 + WHERE GREATEST(c1.endDate, c2.endDate) < :keepAllLaterThan + ) > 0 THEN 1 + WHEN ( + SELECT COUNT(r) + FROM Result r + WHERE r.rated = false + AND r.participation.exercise.course.endDate < :keepAllLaterThan + ) > 0 THEN 1 + WHEN ( + SELECT COUNT(r.id) + FROM Result r + WHERE r.rated = true + AND r.participation.exercise.course.endDate < :keepAllLaterThan + AND r.id NOT IN ( + SELECT r1.id + FROM Result r1 + WHERE r1.participation = r.participation + ORDER BY r1.completionDate DESC + LIMIT 1 + ) + ) > 0 THEN 1 + WHEN ( + SELECT COUNT(sv) + FROM SubmissionVersion sv + WHERE sv.submission.id IN ( + SELECT s.id + FROM Submission s + JOIN s.participation p + JOIN p.exercise e + JOIN e.exerciseGroup eg + JOIN eg.exam ex + JOIN ex.course c + WHERE c.endDate < :keepAllLaterThan + ) + ) > 0 THEN 1 + WHEN ( + SELECT COUNT(f) + FROM Feedback f + WHERE f.result.id NOT IN ( + SELECT MAX(r.id) + FROM Result r + WHERE r.participation = f.result.participation + ) + AND f.result.participation.exercise.course.endDate < :keepAllLaterThan + ) > 0 THEN 1 + ELSE 0 + END + """).setParameter("keepAllLaterThan", keepAllLaterThan).getSingleResult(); + return count != null && count > 0; + } + +} diff --git a/src/main/java/de/tum/in/www1/artemis/service/cleanup/OldDataCleanupService.java b/src/main/java/de/tum/in/www1/artemis/service/cleanup/OldDataCleanupService.java new file mode 100644 index 000000000000..1cdb8317699a --- /dev/null +++ b/src/main/java/de/tum/in/www1/artemis/service/cleanup/OldDataCleanupService.java @@ -0,0 +1,39 @@ +package de.tum.in.www1.artemis.service.cleanup; + +import java.time.ZonedDateTime; + +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import de.tum.in.www1.artemis.repository.cleanup.OldDataCleanUpRepository; + +@Service +public class OldDataCleanupService { + + private final OldDataCleanUpRepository oldDataCleanUpRepository; + + public OldDataCleanupService(OldDataCleanUpRepository oldDataCleanUpRepository) { + this.oldDataCleanUpRepository = oldDataCleanUpRepository; + } + + public void cleanupOldData(ZonedDateTime keepAllLaterThan) { + oldDataCleanUpRepository.deleteOrphans(); + oldDataCleanUpRepository.deletePlagiarismComparisons(keepAllLaterThan); + oldDataCleanUpRepository.deleteNonRatedResults(keepAllLaterThan); + oldDataCleanUpRepository.deleteOldRatedResults(keepAllLaterThan); + oldDataCleanUpRepository.deleteOldSubmissionVersions(keepAllLaterThan); + oldDataCleanUpRepository.deleteOldFeedback(keepAllLaterThan); + } + + @Scheduled(fixedRateString = "#{T(java.time.Duration).ofDays(182).toMillis()}", initialDelayString = "#{T(java.time.Duration).ofHours(1).toMillis()}") + public void scheduleDataCleanup() { + if (dataNeedsCleanup()) { + cleanupOldData(ZonedDateTime.now().minusDays(182)); + } + } + + private boolean dataNeedsCleanup() { + return true; + } + +} diff --git a/src/main/java/de/tum/in/www1/artemis/web/rest/admin/AdminOldDataCleanUpResource.java b/src/main/java/de/tum/in/www1/artemis/web/rest/admin/AdminOldDataCleanUpResource.java new file mode 100644 index 000000000000..c7c37c38446e --- /dev/null +++ b/src/main/java/de/tum/in/www1/artemis/web/rest/admin/AdminOldDataCleanUpResource.java @@ -0,0 +1,52 @@ +package de.tum.in.www1.artemis.web.rest.admin; + +import static de.tum.in.www1.artemis.config.Constants.PROFILE_CORE; + +import java.time.ZonedDateTime; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Profile; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import de.tum.in.www1.artemis.security.annotations.EnforceAdmin; +import de.tum.in.www1.artemis.service.cleanup.OldDataCleanupService; + +/** + * REST controller for managing old data cleanup operations in Artemis. + * Provides an endpoint for administrators to clean up old or orphaned data in the database. + */ +@RestController +@RequestMapping("api/admin/") +@Profile(PROFILE_CORE) +public class AdminOldDataCleanUpResource { + + private static final Logger log = LoggerFactory.getLogger(AdminOldDataCleanUpResource.class); + + private final OldDataCleanupService oldDataCleanupService; + + public AdminOldDataCleanUpResource(OldDataCleanupService oldDataCleanupService) { + this.oldDataCleanupService = oldDataCleanupService; + } + + /** + * REST endpoint to trigger the cleanup of old data in the Artemis database based on a user-provided date. + * This operation will remove old or orphaned data that is older than the specified date. + * This method is restricted to admin users only. + * + * @param cleanupDate the date before which data should be cleaned up. Data older than this date will be deleted. + * @return an empty HTTP 200 response if the cleanup was successful + */ + @PostMapping("deleteOldData") + @EnforceAdmin + public ResponseEntity deleteOldData(@RequestParam("cleanupDate") ZonedDateTime cleanupDate) { + log.debug("REST request to clean up Artemis database for data before {}", cleanupDate); + this.oldDataCleanupService.cleanupOldData(cleanupDate); + return ResponseEntity.ok().build(); + } + +} From a817dbe3879b561915631922bdd9a0c8f9776fa3 Mon Sep 17 00:00:00 2001 From: Michal Kawka Date: Thu, 5 Sep 2024 14:25:21 +0200 Subject: [PATCH 002/113] update AdminOldDataCleanUpResource.java --- .../admin/AdminOldDataCleanUpResource.java | 65 +++++++++++++++---- 1 file changed, 52 insertions(+), 13 deletions(-) diff --git a/src/main/java/de/tum/in/www1/artemis/web/rest/admin/AdminOldDataCleanUpResource.java b/src/main/java/de/tum/in/www1/artemis/web/rest/admin/AdminOldDataCleanUpResource.java index c7c37c38446e..0c9149b68655 100644 --- a/src/main/java/de/tum/in/www1/artemis/web/rest/admin/AdminOldDataCleanUpResource.java +++ b/src/main/java/de/tum/in/www1/artemis/web/rest/admin/AdminOldDataCleanUpResource.java @@ -18,7 +18,7 @@ /** * REST controller for managing old data cleanup operations in Artemis. - * Provides an endpoint for administrators to clean up old or orphaned data in the database. + * Provides endpoints for administrators to clean up old or orphaned data in the database. */ @RestController @RequestMapping("api/admin/") @@ -33,20 +33,59 @@ public AdminOldDataCleanUpResource(OldDataCleanupService oldDataCleanupService) this.oldDataCleanupService = oldDataCleanupService; } - /** - * REST endpoint to trigger the cleanup of old data in the Artemis database based on a user-provided date. - * This operation will remove old or orphaned data that is older than the specified date. - * This method is restricted to admin users only. - * - * @param cleanupDate the date before which data should be cleaned up. Data older than this date will be deleted. - * @return an empty HTTP 200 response if the cleanup was successful - */ - @PostMapping("deleteOldData") + @PostMapping("delete-orphans") @EnforceAdmin - public ResponseEntity deleteOldData(@RequestParam("cleanupDate") ZonedDateTime cleanupDate) { - log.debug("REST request to clean up Artemis database for data before {}", cleanupDate); - this.oldDataCleanupService.cleanupOldData(cleanupDate); + public ResponseEntity deleteOrphans(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, @RequestParam("deleteTo") ZonedDateTime deleteTo) { + log.debug("REST request to delete orphaned data in Artemis database"); + // oldDataCleanupService.deleteOrphans(deleteFrom, deleteTo); return ResponseEntity.ok().build(); } + @PostMapping("delete-plagiarism-comparisons") + @EnforceAdmin + public ResponseEntity deletePlagiarismComparisons(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, @RequestParam("deleteTo") ZonedDateTime deleteTo) { + log.debug("REST request to delete plagiarism comparisons between {} and {}", deleteFrom, deleteTo); + // oldDataCleanupService.deletePlagiarismComparisons(deleteFrom, deleteTo); + return ResponseEntity.ok().build(); + } + + @PostMapping("delete-non-rated-results") + @EnforceAdmin + public ResponseEntity deleteNonRatedResults(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, @RequestParam("deleteTo") ZonedDateTime deleteTo) { + log.debug("REST request to delete non-rated results between {} and {}", deleteFrom, deleteTo); + // oldDataCleanupService.deleteNonRatedResults(deleteFrom, deleteTo); + return ResponseEntity.ok().build(); + } + + @PostMapping("delete-old-rated-results") + @EnforceAdmin + public ResponseEntity deleteOldRatedResults(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, @RequestParam("deleteTo") ZonedDateTime deleteTo) { + log.debug("REST request to delete old rated results between {} and {}", deleteFrom, deleteTo); + // oldDataCleanupService.deleteOldRatedResults(deleteFrom, deleteTo); + return ResponseEntity.ok().build(); + } + + @PostMapping("delete-old-submission-versions") + @EnforceAdmin + public ResponseEntity deleteOldSubmissionVersions(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, @RequestParam("deleteTo") ZonedDateTime deleteTo) { + log.debug("REST request to delete old submission versions between {} and {}", deleteFrom, deleteTo); + // oldDataCleanupService.deleteOldSubmissionVersions(deleteFrom, deleteTo); + return ResponseEntity.ok().build(); + } + + @PostMapping("delete-old-feedback") + @EnforceAdmin + public ResponseEntity deleteOldFeedback(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, @RequestParam("deleteTo") ZonedDateTime deleteTo) { + log.debug("REST request to delete old feedback between {} and {}", deleteFrom, deleteTo); + // oldDataCleanupService.deleteOldFeedback(deleteFrom, deleteTo); + return ResponseEntity.ok().build(); + } + + @PostMapping("delete-all-old-data") + @EnforceAdmin + public ResponseEntity deleteOldData(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, @RequestParam("deleteTo") ZonedDateTime deleteTo) { + log.debug("REST request to clean up Artemis database for data between {} and {}", deleteFrom, deleteTo); + // oldDataCleanupService.cleanupOldData(deleteFrom, deleteTo); + return ResponseEntity.ok().build(); + } } From c099bb147805adbfbebb7468c65ee8c63f45466d Mon Sep 17 00:00:00 2001 From: Michal Kawka Date: Thu, 5 Sep 2024 14:26:04 +0200 Subject: [PATCH 003/113] update OldDataCleanUpRepository --- .../cleanup/OldDataCleanUpRepository.java | 14 +++--- .../cleanup/OldDataCleanUpRepositoryImpl.java | 50 ++++++++++--------- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepository.java b/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepository.java index 4d4ce4738aab..e8cd773a557c 100644 --- a/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepository.java +++ b/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepository.java @@ -4,17 +4,17 @@ public interface OldDataCleanUpRepository { - void deleteOrphans(); + void deleteOrphans(ZonedDateTime deleteFrom, ZonedDateTime deleteTo); - void deletePlagiarismComparisons(ZonedDateTime keepAllLaterThan); + void deletePlagiarismComparisons(ZonedDateTime deleteFrom, ZonedDateTime deleteTo); - void deleteNonRatedResults(ZonedDateTime keepAllLaterThan); + void deleteNonRatedResults(ZonedDateTime deleteFrom, ZonedDateTime deleteTo); - void deleteOldRatedResults(ZonedDateTime keepAllLaterThan); + void deleteOldRatedResults(ZonedDateTime deleteFrom, ZonedDateTime deleteTo); - void deleteOldSubmissionVersions(ZonedDateTime keepAllLaterThan); + void deleteOldSubmissionVersions(ZonedDateTime deleteFrom, ZonedDateTime deleteTo); - void deleteOldFeedback(ZonedDateTime keepAllLaterThan); + void deleteOldFeedback(ZonedDateTime deleteFrom, ZonedDateTime deleteTo); - boolean existsDataForCleanup(ZonedDateTime keepAllLaterThan); + boolean existsDataForCleanup(ZonedDateTime deleteFrom, ZonedDateTime deleteTo); } diff --git a/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepositoryImpl.java b/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepositoryImpl.java index b879de21d7f9..607a3b818ae9 100644 --- a/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepositoryImpl.java +++ b/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepositoryImpl.java @@ -8,6 +8,7 @@ import org.springframework.stereotype.Repository; import org.springframework.transaction.annotation.Transactional; +// TODO Dmytro: modify queries to use deleteFrom and deleteTo @Repository public class OldDataCleanUpRepositoryImpl implements OldDataCleanUpRepository { @@ -18,7 +19,7 @@ public class OldDataCleanUpRepositoryImpl implements OldDataCleanUpRepository { @Override @Transactional - public void deleteOrphans() { + public void deleteOrphans(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { entityManager.createQuery("DELETE FROM Feedback f WHERE f.result IS NULL").executeUpdate(); entityManager.createQuery("DELETE FROM LongFeedbackText lft WHERE lft.feedback IS NULL").executeUpdate(); entityManager.createQuery("DELETE FROM ParticipantScore ps WHERE ps.team IS NULL AND ps.user IS NULL").executeUpdate(); @@ -30,51 +31,52 @@ public void deleteOrphans() { @Override @Transactional - public void deletePlagiarismComparisons(ZonedDateTime keepAllLaterThan) { + public void deletePlagiarismComparisons(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { entityManager .createQuery("DELETE FROM PlagiarismComparison pc " + "WHERE pc.id IN (" + " SELECT pc.id " + " FROM PlagiarismComparison pc " + " JOIN pc.submissionA s1 " + " JOIN pc.submissionB s2 " + " JOIN s1.participation p1 " + " JOIN s2.participation p2 " + " JOIN p1.exercise e1 " - + " JOIN p2.exercise e2 " + " JOIN e1.course c1 " + " JOIN e2.course c2 " + " WHERE GREATEST(c1.endDate, c2.endDate) < :keepAllLaterThan" + ")") - .setParameter("keepAllLaterThan", keepAllLaterThan).executeUpdate(); + + " JOIN p2.exercise e2 " + " JOIN e1.course c1 " + " JOIN e2.course c2 " + " WHERE GREATEST(c1.endDate, c2.endDate) < :deleteFrom" + ")") + .setParameter("deleteFrom", deleteFrom).executeUpdate(); } @Override @Transactional - public void deleteNonRatedResults(ZonedDateTime keepAllLaterThan) { - entityManager.createQuery("DELETE FROM Result r " + "WHERE r.rated = false " + "AND r.participation.exercise.course.endDate < :keepAllLaterThan") - .setParameter("keepAllLaterThan", keepAllLaterThan).executeUpdate(); + public void deleteNonRatedResults(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { + entityManager.createQuery("DELETE FROM Result r " + "WHERE r.rated = false " + "AND r.participation.exercise.course.endDate < :deleteFrom") + .setParameter("deleteFrom", deleteFrom).executeUpdate(); } @Override @Transactional - public void deleteOldRatedResults(ZonedDateTime keepAllLaterThan) { + public void deleteOldRatedResults(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { entityManager .createQuery("DELETE FROM Result r " + "WHERE r.id IN (" + " SELECT r.id " + " FROM Result r " + " WHERE r.rated = true " - + " AND r.participation.exercise.course.endDate < :keepAllLaterThan " + " AND r.id NOT IN (" + " SELECT r1.id " + " FROM Result r1 " + + " AND r.participation.exercise.course.endDate < :deleteFrom " + " AND r.id NOT IN (" + " SELECT r1.id " + " FROM Result r1 " + " WHERE r1.participation = r.participation " + " ORDER BY r1.completionDate DESC " + " LIMIT 1" + " )" + ")") - .setParameter("keepAllLaterThan", keepAllLaterThan).executeUpdate(); + .setParameter("deleteFrom", deleteFrom).executeUpdate(); } @Override @Transactional - public void deleteOldSubmissionVersions(ZonedDateTime keepAllLaterThan) { - entityManager.createQuery("DELETE FROM SubmissionVersion sv " + "WHERE sv.submission.id IN (" + " SELECT s.id " + " FROM Submission s " - + " JOIN s.participation p " + " JOIN p.exercise e " + " JOIN e.exerciseGroup eg " + " JOIN eg.exam ex " + " JOIN ex.course c " - + " WHERE c.endDate < :keepAllLaterThan" + ")").setParameter("keepAllLaterThan", keepAllLaterThan).executeUpdate(); + public void deleteOldSubmissionVersions(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { + entityManager + .createQuery("DELETE FROM SubmissionVersion sv " + "WHERE sv.submission.id IN (" + " SELECT s.id " + " FROM Submission s " + " JOIN s.participation p " + + " JOIN p.exercise e " + " JOIN e.exerciseGroup eg " + " JOIN eg.exam ex " + " JOIN ex.course c " + " WHERE c.endDate < :deleteFrom" + ")") + .setParameter("deleteFrom", deleteFrom).executeUpdate(); } @Override @Transactional - public void deleteOldFeedback(ZonedDateTime keepAllLaterThan) { + public void deleteOldFeedback(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { entityManager .createQuery("DELETE FROM Feedback f " + "WHERE f.result.id NOT IN (" + " SELECT MAX(r.id) " + " FROM Result r " - + " WHERE r.participation = f.result.participation" + ") " + "AND f.result.participation.exercise.course.endDate < :keepAllLaterThan") - .setParameter("keepAllLaterThan", keepAllLaterThan).executeUpdate(); + + " WHERE r.participation = f.result.participation" + ") " + "AND f.result.participation.exercise.course.endDate < :deleteFrom") + .setParameter("deleteFrom", deleteFrom).executeUpdate(); } @Override @Transactional - public boolean existsDataForCleanup(ZonedDateTime keepAllLaterThan) { + public boolean existsDataForCleanup(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { Long count = (Long) entityManager.createQuery(""" SELECT CASE @@ -96,19 +98,19 @@ SELECT COUNT(pc) JOIN p2.exercise e2 JOIN e1.course c1 JOIN e2.course c2 - WHERE GREATEST(c1.endDate, c2.endDate) < :keepAllLaterThan + WHERE GREATEST(c1.endDate, c2.endDate) < :deleteFrom ) > 0 THEN 1 WHEN ( SELECT COUNT(r) FROM Result r WHERE r.rated = false - AND r.participation.exercise.course.endDate < :keepAllLaterThan + AND r.participation.exercise.course.endDate < :deleteFrom ) > 0 THEN 1 WHEN ( SELECT COUNT(r.id) FROM Result r WHERE r.rated = true - AND r.participation.exercise.course.endDate < :keepAllLaterThan + AND r.participation.exercise.course.endDate < :deleteFrom AND r.id NOT IN ( SELECT r1.id FROM Result r1 @@ -128,7 +130,7 @@ WHERE sv.submission.id IN ( JOIN e.exerciseGroup eg JOIN eg.exam ex JOIN ex.course c - WHERE c.endDate < :keepAllLaterThan + WHERE c.endDate < :deleteFrom ) ) > 0 THEN 1 WHEN ( @@ -139,11 +141,11 @@ SELECT MAX(r.id) FROM Result r WHERE r.participation = f.result.participation ) - AND f.result.participation.exercise.course.endDate < :keepAllLaterThan + AND f.result.participation.exercise.course.endDate < :deleteFrom ) > 0 THEN 1 ELSE 0 END - """).setParameter("keepAllLaterThan", keepAllLaterThan).getSingleResult(); + """).setParameter("deleteFrom", deleteFrom).getSingleResult(); return count != null && count > 0; } From 1e1a790f67a9301a25507795e842c085189ae502 Mon Sep 17 00:00:00 2001 From: Michal Kawka Date: Thu, 5 Sep 2024 14:26:29 +0200 Subject: [PATCH 004/113] update OldDataCleanupService --- .../cleanup/OldDataCleanupService.java | 43 +++++++++++++++---- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/src/main/java/de/tum/in/www1/artemis/service/cleanup/OldDataCleanupService.java b/src/main/java/de/tum/in/www1/artemis/service/cleanup/OldDataCleanupService.java index 1cdb8317699a..725d56ef965a 100644 --- a/src/main/java/de/tum/in/www1/artemis/service/cleanup/OldDataCleanupService.java +++ b/src/main/java/de/tum/in/www1/artemis/service/cleanup/OldDataCleanupService.java @@ -7,6 +7,7 @@ import de.tum.in.www1.artemis.repository.cleanup.OldDataCleanUpRepository; +// TODO Dmytro: when an operation is executed, update the lastExecuted in the operations table @Service public class OldDataCleanupService { @@ -16,24 +17,48 @@ public OldDataCleanupService(OldDataCleanUpRepository oldDataCleanUpRepository) this.oldDataCleanUpRepository = oldDataCleanUpRepository; } - public void cleanupOldData(ZonedDateTime keepAllLaterThan) { - oldDataCleanUpRepository.deleteOrphans(); - oldDataCleanUpRepository.deletePlagiarismComparisons(keepAllLaterThan); - oldDataCleanUpRepository.deleteNonRatedResults(keepAllLaterThan); - oldDataCleanUpRepository.deleteOldRatedResults(keepAllLaterThan); - oldDataCleanUpRepository.deleteOldSubmissionVersions(keepAllLaterThan); - oldDataCleanUpRepository.deleteOldFeedback(keepAllLaterThan); + public void deleteOrphans(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { + oldDataCleanUpRepository.deleteOrphans(deleteFrom, deleteTo); + } + + public void deletePlagiarismComparisons(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { + oldDataCleanUpRepository.deletePlagiarismComparisons(deleteFrom, deleteTo); + } + + public void deleteNonRatedResults(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { + oldDataCleanUpRepository.deleteNonRatedResults(deleteFrom, deleteTo); + } + + public void deleteOldRatedResults(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { + oldDataCleanUpRepository.deleteOldRatedResults(deleteFrom, deleteTo); + } + + public void deleteOldSubmissionVersions(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { + oldDataCleanUpRepository.deleteOldSubmissionVersions(deleteFrom, deleteTo); + } + + public void deleteOldFeedback(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { + oldDataCleanUpRepository.deleteOldFeedback(deleteFrom, deleteTo); } @Scheduled(fixedRateString = "#{T(java.time.Duration).ofDays(182).toMillis()}", initialDelayString = "#{T(java.time.Duration).ofHours(1).toMillis()}") public void scheduleDataCleanup() { if (dataNeedsCleanup()) { - cleanupOldData(ZonedDateTime.now().minusDays(182)); + ZonedDateTime cutoffDate = ZonedDateTime.now().minusDays(182); + // cleanupOldData(cutoffDate); } } + public void cleanupOldData(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { + deleteOrphans(deleteFrom, deleteTo); + deletePlagiarismComparisons(deleteFrom, deleteTo); + deleteNonRatedResults(deleteFrom, deleteTo); + deleteOldRatedResults(deleteFrom, deleteTo); + deleteOldSubmissionVersions(deleteFrom, deleteTo); + deleteOldFeedback(deleteFrom, deleteTo); + } + private boolean dataNeedsCleanup() { return true; } - } From d941086012c63d9e60a05de0e87af617e6315f36 Mon Sep 17 00:00:00 2001 From: Michal Kawka Date: Thu, 5 Sep 2024 14:27:02 +0200 Subject: [PATCH 005/113] create cleanup-service components --- .../cleanup-service.component.html | 56 +++++++++ .../cleanup-service.component.ts | 117 ++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 src/main/webapp/app/admin/cleanup-service/cleanup-service.component.html create mode 100644 src/main/webapp/app/admin/cleanup-service/cleanup-service.component.ts diff --git a/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.html b/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.html new file mode 100644 index 000000000000..f6487cdafeb2 --- /dev/null +++ b/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.html @@ -0,0 +1,56 @@ +
+

+ Cleanup Service Operations + +

+
+ + + + + + + + + + + + @for (operation of cleanupOperations; track operation) { + + + + + + + + } + +
OperationDelete FromDelete ToLast Executed
+ {{ 'cleanupService.operation.' + operation.name | artemisTranslate }} + +
+ +
+ 		+
+ +
+ 		+ {{ operation.lastExecuted | artemisDate }} + + +
+
+
diff --git a/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.ts b/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.ts new file mode 100644 index 000000000000..dc476671929e --- /dev/null +++ b/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.ts @@ -0,0 +1,117 @@ +import { Component, OnInit } from '@angular/core'; +import { faSync, faTrash } from '@fortawesome/free-solid-svg-icons'; +import dayjs from 'dayjs/esm'; +import { CleanupOperation } from 'app/admin/cleanup-service/cleanup-operation.model'; +import { DataCleanupService } from 'app/admin/cleanup-service/cleanup-service.service'; +import { convertDateFromClient } from 'app/utils/date.utils'; +import { Subject } from 'rxjs'; +import { EventManager } from 'app/core/util/event-manager.service'; +import { HttpErrorResponse } from '@angular/common/http'; + +@Component({ + selector: 'jhi-cleanup-service', + templateUrl: './cleanup-service.component.html', +}) +export class CleanupServiceComponent implements OnInit { + faSync = faSync; + faTrash = faTrash; + + private dialogErrorSource = new Subject(); + dialogError = this.dialogErrorSource.asObservable(); + + constructor( + private cleanupService: DataCleanupService, + private eventManager: EventManager, + ) {} + + cleanupOperations: CleanupOperation[] = [ + { + name: 'deleteOrphans', + deleteFrom: dayjs().subtract(30, 'days'), // 30 days ago + deleteTo: dayjs().subtract(10, 'days'), // 10 days ago + lastExecuted: dayjs().subtract(1, 'days'), // 1 day ago + }, + { + name: 'deletePlagiarismComparisons', + deleteFrom: dayjs().subtract(60, 'days'), // 60 days ago + deleteTo: dayjs().subtract(20, 'days'), // 20 days ago + lastExecuted: dayjs().subtract(2, 'days'), // 2 days ago + }, + { + name: 'deleteNonRatedResults', + deleteFrom: dayjs().subtract(90, 'days'), // 90 days ago + deleteTo: dayjs().subtract(30, 'days'), // 30 days ago + lastExecuted: dayjs().subtract(3, 'days'), // 3 days ago + }, + { + name: 'deleteOldRatedResults', + deleteFrom: dayjs().subtract(120, 'days'), // 120 days ago + deleteTo: dayjs().subtract(40, 'days'), // 40 days ago + lastExecuted: dayjs().subtract(4, 'days'), // 4 days ago + }, + { + name: 'deleteOldSubmissionVersions', + deleteFrom: dayjs().subtract(150, 'days'), // 150 days ago + deleteTo: dayjs().subtract(50, 'days'), // 50 days ago + lastExecuted: dayjs().subtract(5, 'days'), // 5 days ago + }, + { + name: 'deleteOldFeedback', + deleteFrom: dayjs().subtract(180, 'days'), // 180 days ago + deleteTo: dayjs().subtract(60, 'days'), // 60 days ago + lastExecuted: dayjs().subtract(6, 'days'), // 6 days ago + }, + ]; + + // constructor(private cleanupService: CleanupService) {} + + ngOnInit(): void { + this.refresh(); + } + + refresh(): void { + // Implement logic to refresh the data, possibly fetching from a service + } + + executeCleanupOperation(operation: CleanupOperation): void { + console.log(`Executing cleanup operation: ${operation.name}`); + const deleteFrom = convertDateFromClient(operation.deleteFrom)!; + const deleteTo = convertDateFromClient(operation.deleteTo)!; + + const subscriptionHandler = this.handleResponse(); + + switch (operation.name) { + case 'deleteOrphans': + this.cleanupService.deleteOrphans(deleteFrom, deleteTo).subscribe(subscriptionHandler); + break; + case 'deletePlagiarismComparisons': + this.cleanupService.deletePlagiarismComparisons(deleteFrom, deleteTo).subscribe(subscriptionHandler); + break; + case 'deleteNonRatedResults': + this.cleanupService.deleteNonRatedResults(deleteFrom, deleteTo).subscribe(subscriptionHandler); + break; + case 'deleteOldRatedResults': + this.cleanupService.deleteOldRatedResults(deleteFrom, deleteTo).subscribe(subscriptionHandler); + break; + case 'deleteOldSubmissionVersions': + this.cleanupService.deleteOldSubmissionVersions(deleteFrom, deleteTo).subscribe(subscriptionHandler); + break; + case 'deleteOldFeedback': + this.cleanupService.deleteOldFeedback(deleteFrom, deleteTo).subscribe(subscriptionHandler); + break; + default: + console.warn(`Unknown operation: ${operation.name}`); + } + } + + private handleResponse() { + return { + next: () => { + this.dialogErrorSource.next(''); + }, + error: (error: HttpErrorResponse) => { + this.dialogErrorSource.next(error.message); + }, + }; + } +} From b1188de6df73959203249e736448835223514f2e Mon Sep 17 00:00:00 2001 From: Michal Kawka Date: Thu, 5 Sep 2024 14:27:20 +0200 Subject: [PATCH 006/113] update admin routes --- src/main/webapp/app/admin/admin.route.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/webapp/app/admin/admin.route.ts b/src/main/webapp/app/admin/admin.route.ts index 0c2099494d4a..5275197d5785 100644 --- a/src/main/webapp/app/admin/admin.route.ts +++ b/src/main/webapp/app/admin/admin.route.ts @@ -20,6 +20,7 @@ import { BuildAgentSummaryComponent } from 'app/localci/build-agents/build-agent import { StandardizedCompetencyManagementComponent } from 'app/admin/standardized-competencies/standardized-competency-management.component'; import { BuildAgentDetailsComponent } from 'app/localci/build-agents/build-agent-details/build-agent-details/build-agent-details.component'; import { AdminImportStandardizedCompetenciesComponent } from 'app/admin/standardized-competencies/import/admin-import-standardized-competencies.component'; +import { CleanupServiceComponent } from 'app/admin/cleanup-service/cleanup-service.component'; export const adminState: Routes = [ { @@ -148,6 +149,13 @@ export const adminState: Routes = [ (module) => module.IrisGlobalSettingsUpdateRoutingModule, ), }, + { + path: 'cleanup-service', + component: CleanupServiceComponent, + data: { + pageTitle: 'cleanupService.title', + }, + }, ...organizationMgmtRoute, ...userManagementRoute, ...systemNotificationManagementRoute, From 0d31a8c0a1bec07e94a2fa2bc0899deddad277df Mon Sep 17 00:00:00 2001 From: Michal Kawka Date: Thu, 5 Sep 2024 14:27:36 +0200 Subject: [PATCH 007/113] add cleanup operation model --- .../cleanup-service/cleanup-operation.model.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 src/main/webapp/app/admin/cleanup-service/cleanup-operation.model.ts diff --git a/src/main/webapp/app/admin/cleanup-service/cleanup-operation.model.ts b/src/main/webapp/app/admin/cleanup-service/cleanup-operation.model.ts new file mode 100644 index 000000000000..e37634182475 --- /dev/null +++ b/src/main/webapp/app/admin/cleanup-service/cleanup-operation.model.ts @@ -0,0 +1,16 @@ +import dayjs from 'dayjs/esm'; + +export type OperationName = + | 'deleteOrphans' + | 'deletePlagiarismComparisons' + | 'deleteNonRatedResults' + | 'deleteOldRatedResults' + | 'deleteOldSubmissionVersions' + | 'deleteOldFeedback'; + +export interface CleanupOperation { + name: OperationName; + deleteFrom: dayjs.Dayjs; + deleteTo: dayjs.Dayjs; + lastExecuted: dayjs.Dayjs; +} From c9b81d1ccb052c22acacd15918e58078a005fad6 Mon Sep 17 00:00:00 2001 From: Michal Kawka Date: Thu, 5 Sep 2024 14:28:01 +0200 Subject: [PATCH 008/113] add declaration in the admin module --- src/main/webapp/app/admin/admin.module.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/webapp/app/admin/admin.module.ts b/src/main/webapp/app/admin/admin.module.ts index 9f8b86712c93..bd701a734bea 100644 --- a/src/main/webapp/app/admin/admin.module.ts +++ b/src/main/webapp/app/admin/admin.module.ts @@ -47,6 +47,7 @@ import { AdminImportStandardizedCompetenciesComponent } from 'app/admin/standard import { KnowledgeAreaTreeComponent } from 'app/shared/standardized-competencies/knowledge-area-tree.component'; import { StandardizedCompetencyFilterComponent } from 'app/shared/standardized-competencies/standardized-competency-filter.component'; import { StandardizedCompetencyDetailComponent } from 'app/shared/standardized-competencies/standardized-competency-detail.component'; +import { CleanupServiceComponent } from 'app/admin/cleanup-service/cleanup-service.component'; const ENTITY_STATES = [...adminState]; @@ -101,6 +102,7 @@ const ENTITY_STATES = [...adminState]; KnowledgeAreaEditComponent, StandardizedCompetencyManagementComponent, AdminImportStandardizedCompetenciesComponent, + CleanupServiceComponent, ], }) export class ArtemisAdminModule {} From 508a2fe5be35f823def8bd3f18aabbe0de97698a Mon Sep 17 00:00:00 2001 From: Michal Kawka Date: Thu, 5 Sep 2024 14:28:26 +0200 Subject: [PATCH 009/113] add cleanup requests --- .../cleanup-service.service.ts | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 src/main/webapp/app/admin/cleanup-service/cleanup-service.service.ts diff --git a/src/main/webapp/app/admin/cleanup-service/cleanup-service.service.ts b/src/main/webapp/app/admin/cleanup-service/cleanup-service.service.ts new file mode 100644 index 000000000000..0fc96b1b2cb0 --- /dev/null +++ b/src/main/webapp/app/admin/cleanup-service/cleanup-service.service.ts @@ -0,0 +1,94 @@ +import { Injectable } from '@angular/core'; +import { HttpClient, HttpResponse } from '@angular/common/http'; +import { Observable } from 'rxjs'; + +@Injectable({ providedIn: 'root' }) +export class DataCleanupService { + private readonly adminResourceUrl = 'api/admin'; + + constructor(private http: HttpClient) {} + + /** + * Send POST request to delete orphaned data within a specific date range. + * @param deleteFrom the start date from which orphaned data should be deleted + * @param deleteTo the end date until which orphaned data should be deleted + */ + deleteOrphans(deleteFrom: string, deleteTo: string): Observable> { + return this.http.post(`${this.adminResourceUrl}/delete-orphans`, null, { + params: { deleteFrom, deleteTo }, + observe: 'response', + }); + } + + /** + * Send POST request to delete plagiarism comparisons within a specific date range. + * @param deleteFrom the start date from which plagiarism comparisons should be deleted + * @param deleteTo the end date until which plagiarism comparisons should be deleted + */ + deletePlagiarismComparisons(deleteFrom: string, deleteTo: string): Observable> { + return this.http.post(`${this.adminResourceUrl}/delete-plagiarism-comparisons`, null, { + params: { deleteFrom, deleteTo }, + observe: 'response', + }); + } + + /** + * Send POST request to delete non-rated results within a specific date range. + * @param deleteFrom the start date from which non-rated results should be deleted + * @param deleteTo the end date until which non-rated results should be deleted + */ + deleteNonRatedResults(deleteFrom: string, deleteTo: string): Observable> { + return this.http.post(`${this.adminResourceUrl}/delete-non-rated-results`, null, { + params: { deleteFrom, deleteTo }, + observe: 'response', + }); + } + + /** + * Send POST request to delete old rated results within a specific date range. + * @param deleteFrom the start date from which old rated results should be deleted + * @param deleteTo the end date until which old rated results should be deleted + */ + deleteOldRatedResults(deleteFrom: string, deleteTo: string): Observable> { + return this.http.post(`${this.adminResourceUrl}/delete-old-rated-results`, null, { + params: { deleteFrom, deleteTo }, + observe: 'response', + }); + } + + /** + * Send POST request to delete old submission versions within a specific date range. + * @param deleteFrom the start date from which old submission versions should be deleted + * @param deleteTo the end date until which old submission versions should be deleted + */ + deleteOldSubmissionVersions(deleteFrom: string, deleteTo: string): Observable> { + return this.http.post(`${this.adminResourceUrl}/delete-old-submission-versions`, null, { + params: { deleteFrom, deleteTo }, + observe: 'response', + }); + } + + /** + * Send POST request to delete old feedback within a specific date range. + * @param deleteFrom the start date from which old feedback should be deleted + * @param deleteTo the end date until which old feedback should be deleted + */ + deleteOldFeedback(deleteFrom: string, deleteTo: string): Observable> { + return this.http.post(`${this.adminResourceUrl}/delete-old-feedback`, null, { + params: { deleteFrom, deleteTo }, + observe: 'response', + }); + } + + /** + * Send POST request to clean up all old data within a specific date range. + * @param deleteFrom the start date from which all old data should be cleaned up + * @param deleteTo the end date until which all old data should be cleaned up + */ + deleteOldData(deleteFrom: string, deleteTo: string): Observable> { + return this.http.post(`${this.adminResourceUrl}/delete-all-old-data`, null, { + params: { deleteFrom, deleteTo }, + observe: 'response', + }); + } +} From 7b8527cc6825e398f2590166048c858adcb8f5fb Mon Sep 17 00:00:00 2001 From: Michal Kawka Date: Thu, 5 Sep 2024 14:28:41 +0200 Subject: [PATCH 010/113] add translations --- src/main/webapp/i18n/de/cleanupService.json | 23 +++++++++++++++++++++ src/main/webapp/i18n/en/cleanupService.json | 23 +++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 src/main/webapp/i18n/de/cleanupService.json create mode 100644 src/main/webapp/i18n/en/cleanupService.json diff --git a/src/main/webapp/i18n/de/cleanupService.json b/src/main/webapp/i18n/de/cleanupService.json new file mode 100644 index 000000000000..f1d496e513cb --- /dev/null +++ b/src/main/webapp/i18n/de/cleanupService.json @@ -0,0 +1,23 @@ +{ + "cleanupService": { + "title": "Cleanup service", + "table": { + "operation": "Operationsname", + "deleteFrom": "Löschen von", + "deleteTo": "Löschen bis", + "lastExecuted": "Zuletzt ausgeführt" + }, + "operation": { + "deleteOrphans": "Orphans löschen", + "deletePlagiarismComparisons": "Plagiatsvergleiche löschen", + "deleteNonRatedResults": "Nicht bewertete Ergebnisse löschen", + "deleteOldRatedResults": "Alte bewertete Ergebnisse löschen", + "deleteOldSubmissionVersions": "Alte Teilnahmeversionen löschen", + "deleteOldFeedback": "Altes Feedback löschen" + }, + "execute": { + "question": "Bist du sicher, dass du die Operation {{ title }} ausführen möchtest? Diese Aktion kann NICHT rückgängig gemacht werden!", + "typeOperationNameToConfirm": "Bitte gib den Namen der Operation ein, um zu bestätigen." + } + } +} diff --git a/src/main/webapp/i18n/en/cleanupService.json b/src/main/webapp/i18n/en/cleanupService.json new file mode 100644 index 000000000000..9f3b0bbaa28d --- /dev/null +++ b/src/main/webapp/i18n/en/cleanupService.json @@ -0,0 +1,23 @@ +{ + "cleanupService": { + "title": "Cleanup Service", + "table": { + "operation": "Operation name", + "deleteFrom": "Delete from", + "deleteTo": "Delete to", + "lastExecuted": "Last executed" + }, + "operation": { + "deleteOrphans": "Delete orphans", + "deletePlagiarismComparisons": "Delete plagiarism comparisons", + "deleteNonRatedResults": "Delete non-rated results", + "deleteOldRatedResults": "Delete old rated results", + "deleteOldSubmissionVersions": "Delete old submission versions", + "deleteOldFeedback": "Delete old feedback versions" + }, + "execute": { + "question": "Are you sure you want to execute the operation {{ title }}? This action can NOT be undone!", + "typeOperationNameToConfirm": "Please type in the name of the operation to confirm." + } + } +} From b0aa22c173eac3e42a1a2f5c6e64d06bd201fead Mon Sep 17 00:00:00 2001 From: Michal Kawka Date: Thu, 5 Sep 2024 14:28:52 +0200 Subject: [PATCH 011/113] update navbar --- .../webapp/app/shared/layouts/navbar/navbar.component.html | 7 +++++++ .../webapp/app/shared/layouts/navbar/navbar.component.ts | 2 ++ 2 files changed, 9 insertions(+) diff --git a/src/main/webapp/app/shared/layouts/navbar/navbar.component.html b/src/main/webapp/app/shared/layouts/navbar/navbar.component.html index 556aee6ce6bb..473774bd8047 100644 --- a/src/main/webapp/app/shared/layouts/navbar/navbar.component.html +++ b/src/main/webapp/app/shared/layouts/navbar/navbar.component.html @@ -300,6 +300,13 @@
} +
  • + + + + Cleanup Service + +
    • diff --git a/src/main/webapp/app/shared/layouts/navbar/navbar.component.ts b/src/main/webapp/app/shared/layouts/navbar/navbar.component.ts index b24b25a4b64d..bbf8ab1918ed 100644 --- a/src/main/webapp/app/shared/layouts/navbar/navbar.component.ts +++ b/src/main/webapp/app/shared/layouts/navbar/navbar.component.ts @@ -23,6 +23,7 @@ import { faBell, faBook, faBookOpen, + faBroom, faChevronRight, faCog, faEye, @@ -119,6 +120,7 @@ export class NavbarComponent implements OnInit, OnDestroy { faGears = faGears; faPuzzlePiece = faPuzzlePiece; faChevronRight = faChevronRight; + faBroom = faBroom; private standardizedCompetencySubscription: Subscription; private authStateSubscription: Subscription; From 36aefe6fa29e9f3a7dfc989ef2ac283ab93667bd Mon Sep 17 00:00:00 2001 From: Michal Kawka Date: Thu, 5 Sep 2024 14:49:03 +0200 Subject: [PATCH 012/113] add TODO --- .../cleanup-service.component.ts | 46 +++++++++---------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.ts b/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.ts index dc476671929e..4da5d6af907c 100644 --- a/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.ts +++ b/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.ts @@ -24,47 +24,46 @@ export class CleanupServiceComponent implements OnInit { private eventManager: EventManager, ) {} + // TODO Michal Kawka replace with API call fetching operations from the DB cleanupOperations: CleanupOperation[] = [ { name: 'deleteOrphans', - deleteFrom: dayjs().subtract(30, 'days'), // 30 days ago - deleteTo: dayjs().subtract(10, 'days'), // 10 days ago - lastExecuted: dayjs().subtract(1, 'days'), // 1 day ago + deleteFrom: dayjs().subtract(6, 'months'), + deleteTo: dayjs(), + lastExecuted: dayjs().subtract(1, 'days'), }, { name: 'deletePlagiarismComparisons', - deleteFrom: dayjs().subtract(60, 'days'), // 60 days ago - deleteTo: dayjs().subtract(20, 'days'), // 20 days ago - lastExecuted: dayjs().subtract(2, 'days'), // 2 days ago + deleteFrom: dayjs().subtract(6, 'months'), + deleteTo: dayjs(), + lastExecuted: dayjs().subtract(2, 'days'), }, { name: 'deleteNonRatedResults', - deleteFrom: dayjs().subtract(90, 'days'), // 90 days ago - deleteTo: dayjs().subtract(30, 'days'), // 30 days ago - lastExecuted: dayjs().subtract(3, 'days'), // 3 days ago + deleteFrom: dayjs().subtract(6, 'months'), + deleteTo: dayjs(), + lastExecuted: dayjs().subtract(3, 'days'), }, { name: 'deleteOldRatedResults', - deleteFrom: dayjs().subtract(120, 'days'), // 120 days ago - deleteTo: dayjs().subtract(40, 'days'), // 40 days ago - lastExecuted: dayjs().subtract(4, 'days'), // 4 days ago + deleteFrom: dayjs().subtract(6, 'months'), + deleteTo: dayjs(), + lastExecuted: dayjs().subtract(4, 'days'), }, { name: 'deleteOldSubmissionVersions', - deleteFrom: dayjs().subtract(150, 'days'), // 150 days ago - deleteTo: dayjs().subtract(50, 'days'), // 50 days ago - lastExecuted: dayjs().subtract(5, 'days'), // 5 days ago + deleteFrom: dayjs().subtract(6, 'months'), + deleteTo: dayjs(), + lastExecuted: dayjs().subtract(5, 'days'), }, { name: 'deleteOldFeedback', - deleteFrom: dayjs().subtract(180, 'days'), // 180 days ago - deleteTo: dayjs().subtract(60, 'days'), // 60 days ago - lastExecuted: dayjs().subtract(6, 'days'), // 6 days ago + deleteFrom: dayjs().subtract(6, 'months'), + deleteTo: dayjs(), + lastExecuted: dayjs().subtract(6, 'days'), }, ]; - // constructor(private cleanupService: CleanupService) {} - ngOnInit(): void { this.refresh(); } @@ -78,7 +77,7 @@ export class CleanupServiceComponent implements OnInit { const deleteFrom = convertDateFromClient(operation.deleteFrom)!; const deleteTo = convertDateFromClient(operation.deleteTo)!; - const subscriptionHandler = this.handleResponse(); + const subscriptionHandler = this.handleResponse(operation); switch (operation.name) { case 'deleteOrphans': @@ -99,15 +98,14 @@ export class CleanupServiceComponent implements OnInit { case 'deleteOldFeedback': this.cleanupService.deleteOldFeedback(deleteFrom, deleteTo).subscribe(subscriptionHandler); break; - default: - console.warn(`Unknown operation: ${operation.name}`); } } - private handleResponse() { + private handleResponse(operation: CleanupOperation) { return { next: () => { this.dialogErrorSource.next(''); + operation.lastExecuted = dayjs(); // Update lastExecuted to now }, error: (error: HttpErrorResponse) => { this.dialogErrorSource.next(error.message); From 28b050df309a123fa6d9ab6e77ccf6a4cab9abc9 Mon Sep 17 00:00:00 2001 From: Michal Kawka Date: Thu, 5 Sep 2024 14:49:21 +0200 Subject: [PATCH 013/113] add TODO --- src/main/webapp/i18n/de/global.json | 3 ++- src/main/webapp/i18n/en/global.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/i18n/de/global.json b/src/main/webapp/i18n/de/global.json index e687aaaf72fb..a04122d3737b 100644 --- a/src/main/webapp/i18n/de/global.json +++ b/src/main/webapp/i18n/de/global.json @@ -112,7 +112,8 @@ "iris": "Iris Einstellungen", "jhipster-needle-menu-add-admin-element": "JHipster will add additional menu entries here (do not translate!)", "lti": "LTI Konfiguration", - "standardizedCompetencies": "Standardisierte Kompetenzen" + "standardizedCompetencies": "Standardisierte Kompetenzen", + "cleanupService": "Cleanup Service" }, "language": "Sprache", "settings": "Einstellungen", diff --git a/src/main/webapp/i18n/en/global.json b/src/main/webapp/i18n/en/global.json index 7e59d4c46f22..f50809b1974d 100644 --- a/src/main/webapp/i18n/en/global.json +++ b/src/main/webapp/i18n/en/global.json @@ -113,7 +113,8 @@ "iris": "Iris Settings", "jhipster-needle-menu-add-admin-element": "JHipster will add additional menu entries here (do not translate!)", "lti": "LTI Configuration", - "standardizedCompetencies": "Standardized Competencies" + "standardizedCompetencies": "Standardized Competencies", + "cleanupService": "Cleanup Service" }, "language": "Language", "settings": "Settings", From 9d2e83ee87cd61e5e70a5978ed9ff4d222885785 Mon Sep 17 00:00:00 2001 From: Michal Kawka Date: Thu, 5 Sep 2024 14:49:33 +0200 Subject: [PATCH 014/113] remove button --- .../admin/cleanup-service/cleanup-service.component.html | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.html b/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.html index f6487cdafeb2..ac53ac2a24e9 100644 --- a/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.html +++ b/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.html @@ -1,10 +1,6 @@

    Cleanup Service Operations -

    @@ -25,12 +21,12 @@

    - +
    - +
    		From 6b4ce8208396e68af00b9b9b31ba96836d3f68d1 Mon Sep 17 00:00:00 2001 From: Michal Kawka Date: Thu, 5 Sep 2024 14:53:23 +0200 Subject: [PATCH 015/113] update navbar.component.html --- .../webapp/app/shared/layouts/navbar/navbar.component.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/webapp/app/shared/layouts/navbar/navbar.component.html b/src/main/webapp/app/shared/layouts/navbar/navbar.component.html index 473774bd8047..08ff795789ae 100644 --- a/src/main/webapp/app/shared/layouts/navbar/navbar.component.html +++ b/src/main/webapp/app/shared/layouts/navbar/navbar.component.html @@ -303,8 +303,7 @@
  • - - Cleanup Service +
    • From a7704d49cbaafb067613cd554756cd0bbacbd599 Mon Sep 17 00:00:00 2001 From: Michal Kawka Date: Thu, 5 Sep 2024 21:56:38 +0200 Subject: [PATCH 016/113] add validation --- .../cleanup-service.component.html | 24 +++++++++++++++---- .../cleanup-service.component.ts | 18 +++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.html b/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.html index ac53ac2a24e9..1e94dffcb536 100644 --- a/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.html +++ b/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.html @@ -20,13 +20,28 @@

    {{ 'cleanupService.operation.' + operation.name | artemisTranslate }}

    		-
    - +
    +
    		-
    - +
    + + @if (!areDatesValid(operation)) { + + }
    		@@ -36,6 +51,7 @@

    		-
    - -
    + @if (operation.name !== 'deleteOrphans') { +
    + +
    + }     		-
    - - @if (!areDatesValid(operation)) { - - } -
    + @if (operation.name !== 'deleteOrphans') { +
    + + @if (!areDatesValid(operation)) { + + } +
    + }     		{{ operation.lastExecuted | artemisDate }} diff --git a/src/main/webapp/app/admin/cleanup-service/cleanup-service.service.ts b/src/main/webapp/app/admin/cleanup-service/cleanup-service.service.ts index 0fc96b1b2cb0..af47518ef6ce 100644 --- a/src/main/webapp/app/admin/cleanup-service/cleanup-service.service.ts +++ b/src/main/webapp/app/admin/cleanup-service/cleanup-service.service.ts @@ -10,12 +10,9 @@ export class DataCleanupService { /** * Send POST request to delete orphaned data within a specific date range. - * @param deleteFrom the start date from which orphaned data should be deleted - * @param deleteTo the end date until which orphaned data should be deleted */ - deleteOrphans(deleteFrom: string, deleteTo: string): Observable> { + deleteOrphans(): Observable> { return this.http.post(`${this.adminResourceUrl}/delete-orphans`, null, { - params: { deleteFrom, deleteTo }, observe: 'response', }); } From 5fdc5d67210aba18adf9954153c262281b4c0af8 Mon Sep 17 00:00:00 2001 From: Dmytro Polityka Date: Sat, 7 Sep 2024 09:30:36 +0200 Subject: [PATCH 019/113] implement last executed date, refactor queries --- .../domain/cleanup/CleanupJobExecution.java | 10 +- .../CleanupJobExecutionRepository.java | 4 + ...sitory.java => DataCleanupRepository.java} | 2 +- ...pl.java => DataCleanupRepositoryImpl.java} | 59 ++++++----- .../service/cleanup/DataCleanupService.java | 86 ++++++++++++++++ .../cleanup/OldDataCleanupService.java | 45 --------- .../admin/AdminOldDataCleanUpResource.java | 83 ---------------- .../admin/AdminOldDataCleanupResource.java | 97 +++++++++++++++++++ .../dto/CleanupServiceExecutionRecordDTO.java | 12 +++ .../cleanup-operation.model.ts | 2 +- .../cleanup-service.component.html | 20 ++-- .../cleanup-service.component.ts | 44 ++++++--- .../cleanup-service.service.ts | 42 ++++---- src/main/webapp/i18n/de/cleanupService.json | 3 +- src/main/webapp/i18n/en/cleanupService.json | 3 +- 15 files changed, 306 insertions(+), 206 deletions(-) rename src/main/java/de/tum/in/www1/artemis/repository/cleanup/{OldDataCleanUpRepository.java => DataCleanupRepository.java} (92%) rename src/main/java/de/tum/in/www1/artemis/repository/cleanup/{OldDataCleanUpRepositoryImpl.java => DataCleanupRepositoryImpl.java} (73%) create mode 100644 src/main/java/de/tum/in/www1/artemis/service/cleanup/DataCleanupService.java delete mode 100644 src/main/java/de/tum/in/www1/artemis/service/cleanup/OldDataCleanupService.java delete mode 100644 src/main/java/de/tum/in/www1/artemis/web/rest/admin/AdminOldDataCleanUpResource.java create mode 100644 src/main/java/de/tum/in/www1/artemis/web/rest/admin/AdminOldDataCleanupResource.java create mode 100644 src/main/java/de/tum/in/www1/artemis/web/rest/dto/CleanupServiceExecutionRecordDTO.java diff --git a/src/main/java/de/tum/in/www1/artemis/domain/cleanup/CleanupJobExecution.java b/src/main/java/de/tum/in/www1/artemis/domain/cleanup/CleanupJobExecution.java index 86cb28f6a245..540110dee002 100644 --- a/src/main/java/de/tum/in/www1/artemis/domain/cleanup/CleanupJobExecution.java +++ b/src/main/java/de/tum/in/www1/artemis/domain/cleanup/CleanupJobExecution.java @@ -27,7 +27,7 @@ public class CleanupJobExecution extends DomainObject { @Column(name = "deleted_at") @NotNull - private ZonedDateTime deletion_timestamp; + private ZonedDateTime deletionTimestamp; @Enumerated(EnumType.STRING) @Column(name = "operation_type") @@ -52,12 +52,12 @@ public void setDeleteTo(ZonedDateTime deleteTo) { this.deleteTo = deleteTo; } - public ZonedDateTime getDeletion_timestamp() { - return deletion_timestamp; + public ZonedDateTime getDeletionTimestamp() { + return deletionTimestamp; } - public void setDeletion_timestamp(ZonedDateTime deletion_timestamp) { - this.deletion_timestamp = deletion_timestamp; + public void setDeletionTimestamp(ZonedDateTime deletionTimestamp) { + this.deletionTimestamp = deletionTimestamp; } public CleanupJobType getCleanupJobType() { diff --git a/src/main/java/de/tum/in/www1/artemis/repository/cleanup/CleanupJobExecutionRepository.java b/src/main/java/de/tum/in/www1/artemis/repository/cleanup/CleanupJobExecutionRepository.java index e541afbaa6fe..a750be07324f 100644 --- a/src/main/java/de/tum/in/www1/artemis/repository/cleanup/CleanupJobExecutionRepository.java +++ b/src/main/java/de/tum/in/www1/artemis/repository/cleanup/CleanupJobExecutionRepository.java @@ -6,6 +6,7 @@ import org.springframework.stereotype.Repository; import de.tum.in.www1.artemis.domain.cleanup.CleanupJobExecution; +import de.tum.in.www1.artemis.domain.enumeration.CleanupJobType; import de.tum.in.www1.artemis.repository.base.ArtemisJpaRepository; /** @@ -14,4 +15,7 @@ @Profile(PROFILE_CORE) @Repository public interface CleanupJobExecutionRepository extends ArtemisJpaRepository { + + CleanupJobExecution findTopByCleanupJobTypeOrderByDeletionTimestampDesc(CleanupJobType jobType); + } diff --git a/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepository.java b/src/main/java/de/tum/in/www1/artemis/repository/cleanup/DataCleanupRepository.java similarity index 92% rename from src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepository.java rename to src/main/java/de/tum/in/www1/artemis/repository/cleanup/DataCleanupRepository.java index da802fe557c6..8868c3c910f0 100644 --- a/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepository.java +++ b/src/main/java/de/tum/in/www1/artemis/repository/cleanup/DataCleanupRepository.java @@ -2,7 +2,7 @@ import java.time.ZonedDateTime; -public interface OldDataCleanUpRepository { +public interface DataCleanupRepository { void deleteOrphans(); diff --git a/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepositoryImpl.java b/src/main/java/de/tum/in/www1/artemis/repository/cleanup/DataCleanupRepositoryImpl.java similarity index 73% rename from src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepositoryImpl.java rename to src/main/java/de/tum/in/www1/artemis/repository/cleanup/DataCleanupRepositoryImpl.java index 496f1bee1b6b..d1bdaa8077bc 100644 --- a/src/main/java/de/tum/in/www1/artemis/repository/cleanup/OldDataCleanUpRepositoryImpl.java +++ b/src/main/java/de/tum/in/www1/artemis/repository/cleanup/DataCleanupRepositoryImpl.java @@ -9,7 +9,7 @@ import org.springframework.transaction.annotation.Transactional; @Repository -public class OldDataCleanUpRepositoryImpl implements OldDataCleanUpRepository { +public class DataCleanupRepositoryImpl implements DataCleanupRepository { // transactinal ok, because of delete statements @@ -54,17 +54,19 @@ WHERE LEAST(c1.endDate, c2.endDate) < :deleteTo AND @Override @Transactional public void deleteNonRatedResults(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { - // delete all non-related results for all courses that were conducted withing a specific date range entityManager.createQuery(""" - DELETE FROM Result r - WHERE r.rated = false - AND r.participation IS NOT NULL - AND r.participation.exercise IS NOT NULL - AND r.participation.exercise.course IS NOT NULL - AND r.participation.exercise.course.endDate IS NOT NULL - AND r.participation.exercise.course.endDate < :deleteTo - AND r.participation.exercise.course.startDate > :deleteFrom - """).setParameter("deleteTo", deleteTo).setParameter("deleteFrom", deleteFrom).executeUpdate(); + DELETE FROM Result r + WHERE r.rated = false + AND r.participation IS NOT NULL + AND EXISTS ( + SELECT 1 + FROM Course c + JOIN Exercise e ON e.course = c + WHERE e = r.participation.exercise + AND c.endDate < :deleteTo + AND c.startDate > :deleteFrom + ) + """).setParameter("deleteTo", deleteTo).setParameter("deleteFrom", deleteFrom).executeUpdate(); } @Override @@ -72,38 +74,43 @@ public void deleteNonRatedResults(ZonedDateTime deleteFrom, ZonedDateTime delete public void deleteRatedResults(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { // delete all rated results that are not latest rated for a participation for courses conducted within a specific date range entityManager.createQuery(""" - DELETE FROM Result r + DELETE FROM Result r WHERE r.rated = true AND r.participation IS NOT NULL AND r.participation.exercise IS NOT NULL - AND r.participation.exercise.course IS NOT NULL - AND r.participation.exercise.course.endDate IS NOT NULL - AND r.participation.exercise.course.endDate < :deleteTo - AND r.participation.exercise.course.startDate > :deleteFrom + AND EXISTS ( + SELECT 1 + FROM Course c + JOIN Exercise e ON e.course = c + WHERE e = r.participation.exercise + AND c.endDate < :deleteTo + AND c.startDate > :deleteFrom + ) AND r.id NOT IN ( SELECT MAX(r2.id) FROM Result r2 WHERE r2.participation = r.participation AND r2.rated = true ) - """).setParameter("deleteFrom", deleteFrom).setParameter("deleteTo", deleteTo).executeUpdate(); + """).setParameter("deleteFrom", deleteFrom).setParameter("deleteTo", deleteTo).executeUpdate(); } @Override @Transactional public void deleteSubmissionVersions(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { - // delete old submission versions for exams entityManager.createQuery(""" DELETE FROM SubmissionVersion sv WHERE sv.submission.id IS NOT NULL - AND sv.submission.participation.id IS NOT NULL - AND sv.submission.participation.exercise.id IS NOT NULL - AND sv.submission.participation.exercise.exerciseGroup.id IS NOT NULL - AND sv.submission.participation.exercise.exerciseGroup.exam.id IS NOT NULL - AND sv.submission.participation.exercise.exerciseGroup.exam.course.id IS NOT NULL - AND sv.submission.participation.exercise.exerciseGroup.exam.course.endDate IS NOT NULL - AND sv.submission.participation.exercise.exerciseGroup.exam.course.endDate < :deleteTo - AND sv.submission.participation.exercise.exerciseGroup.exam.course.endDate > :deleteFrom + AND EXISTS ( + SELECT 1 + FROM Exam ex + JOIN Course c ON ex.course = c + JOIN ExerciseGroup eg ON eg.exam = ex + JOIN Exercise e ON e.exerciseGroup = eg + WHERE e = sv.submission.participation.exercise + AND c.endDate < :deleteTo + AND c.endDate > :deleteFrom + ) """).setParameter("deleteFrom", deleteFrom).setParameter("deleteTo", deleteTo).executeUpdate(); } diff --git a/src/main/java/de/tum/in/www1/artemis/service/cleanup/DataCleanupService.java b/src/main/java/de/tum/in/www1/artemis/service/cleanup/DataCleanupService.java new file mode 100644 index 000000000000..ce5f2ff43537 --- /dev/null +++ b/src/main/java/de/tum/in/www1/artemis/service/cleanup/DataCleanupService.java @@ -0,0 +1,86 @@ +package de.tum.in.www1.artemis.service.cleanup; + +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; + +import org.springframework.stereotype.Service; + +import de.tum.in.www1.artemis.domain.cleanup.CleanupJobExecution; +import de.tum.in.www1.artemis.domain.enumeration.CleanupJobType; +import de.tum.in.www1.artemis.repository.cleanup.CleanupJobExecutionRepository; +import de.tum.in.www1.artemis.repository.cleanup.DataCleanupRepository; +import de.tum.in.www1.artemis.web.rest.dto.CleanupServiceExecutionRecordDTO; + +@Service +public class DataCleanupService { + + private final DataCleanupRepository dataCleanUpRepository; + + private final CleanupJobExecutionRepository cleanupJobExecutionRepository; + + public DataCleanupService(DataCleanupRepository dataCleanUpRepository, CleanupJobExecutionRepository cleanupJobExecutionRepository) { + this.dataCleanUpRepository = dataCleanUpRepository; + this.cleanupJobExecutionRepository = cleanupJobExecutionRepository; + } + + public CleanupServiceExecutionRecordDTO deleteOrphans() { + dataCleanUpRepository.deleteOrphans(); + return new CleanupServiceExecutionRecordDTO(this.createDBEntry(CleanupJobType.ORPHANS, null, null).getDeletionTimestamp()); + } + + public CleanupServiceExecutionRecordDTO deletePlagiarismComparisons(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { + dataCleanUpRepository.deletePlagiarismComparisons(deleteFrom, deleteTo); + return new CleanupServiceExecutionRecordDTO(this.createDBEntry(CleanupJobType.PLAGIARISM_COMPARISONS, deleteFrom, deleteTo).getDeletionTimestamp()); + } + + public CleanupServiceExecutionRecordDTO deleteNonRatedResults(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { + dataCleanUpRepository.deleteNonRatedResults(deleteFrom, deleteTo); + return new CleanupServiceExecutionRecordDTO(this.createDBEntry(CleanupJobType.NON_RATED_RESULTS, deleteFrom, deleteTo).getDeletionTimestamp()); + } + + public CleanupServiceExecutionRecordDTO deleteRatedResults(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { + dataCleanUpRepository.deleteRatedResults(deleteFrom, deleteTo); + return new CleanupServiceExecutionRecordDTO(this.createDBEntry(CleanupJobType.RATED_RESULTS, deleteFrom, deleteTo).getDeletionTimestamp()); + } + + public CleanupServiceExecutionRecordDTO deleteSubmissionVersions(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { + dataCleanUpRepository.deleteSubmissionVersions(deleteFrom, deleteTo); + return new CleanupServiceExecutionRecordDTO(this.createDBEntry(CleanupJobType.SUBMISSION_VERSIONS, deleteFrom, deleteTo).getDeletionTimestamp()); + } + + public CleanupServiceExecutionRecordDTO deleteFeedback(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { + dataCleanUpRepository.deleteFeedback(deleteFrom, deleteTo); + return new CleanupServiceExecutionRecordDTO(this.createDBEntry(CleanupJobType.FEEDBACK, deleteFrom, deleteTo).getDeletionTimestamp()); + } + + public List getLastExecutions() { + List executionRecords = new ArrayList<>(); + + for (CleanupJobType jobType : CleanupJobType.values()) { + CleanupJobExecution lastExecution = cleanupJobExecutionRepository.findTopByCleanupJobTypeOrderByDeletionTimestampDesc(jobType); + if (lastExecution != null) { + executionRecords.add(new CleanupServiceExecutionRecordDTO(lastExecution.getDeletionTimestamp())); + } + else { + // Если задание еще ни разу не выполнялось, нужно добавить null + executionRecords.add(new CleanupServiceExecutionRecordDTO(null)); + } + } + + return executionRecords; + } + + private CleanupJobExecution createDBEntry(CleanupJobType cleanupJobType, ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { + var entry = new CleanupJobExecution(); + entry.setCleanupJobType(cleanupJobType); + if (deleteFrom != null) { + entry.setDeleteFrom(deleteFrom); + } + if (deleteTo != null) { + entry.setDeleteTo(deleteTo); + } + entry.setDeletionTimestamp(ZonedDateTime.now()); + return this.cleanupJobExecutionRepository.save(entry); + } +} diff --git a/src/main/java/de/tum/in/www1/artemis/service/cleanup/OldDataCleanupService.java b/src/main/java/de/tum/in/www1/artemis/service/cleanup/OldDataCleanupService.java deleted file mode 100644 index a37acdbead36..000000000000 --- a/src/main/java/de/tum/in/www1/artemis/service/cleanup/OldDataCleanupService.java +++ /dev/null @@ -1,45 +0,0 @@ -package de.tum.in.www1.artemis.service.cleanup; - -import java.time.ZonedDateTime; - -import org.springframework.stereotype.Service; - -import de.tum.in.www1.artemis.repository.cleanup.CleanupJobExecutionRepository; -import de.tum.in.www1.artemis.repository.cleanup.OldDataCleanUpRepository; - -@Service -public class OldDataCleanupService { - - private final OldDataCleanUpRepository oldDataCleanUpRepository; - - private final CleanupJobExecutionRepository cleanupJobExecutionRepository; - - public OldDataCleanupService(OldDataCleanUpRepository oldDataCleanUpRepository, CleanupJobExecutionRepository cleanupJobExecutionRepository) { - this.oldDataCleanUpRepository = oldDataCleanUpRepository; - this.cleanupJobExecutionRepository = cleanupJobExecutionRepository; - } - - public void deleteOrphans() { - oldDataCleanUpRepository.deleteOrphans(); - } - - public void deletePlagiarismComparisons(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { - oldDataCleanUpRepository.deletePlagiarismComparisons(deleteFrom, deleteTo); - } - - public void deleteNonRatedResults(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { - oldDataCleanUpRepository.deleteNonRatedResults(deleteFrom, deleteTo); - } - - public void deleteOldRatedResults(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { - oldDataCleanUpRepository.deleteRatedResults(deleteFrom, deleteTo); - } - - public void deleteSubmissionVersions(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { - oldDataCleanUpRepository.deleteSubmissionVersions(deleteFrom, deleteTo); - } - - public void deleteOldFeedback(ZonedDateTime deleteFrom, ZonedDateTime deleteTo) { - oldDataCleanUpRepository.deleteFeedback(deleteFrom, deleteTo); - } -} diff --git a/src/main/java/de/tum/in/www1/artemis/web/rest/admin/AdminOldDataCleanUpResource.java b/src/main/java/de/tum/in/www1/artemis/web/rest/admin/AdminOldDataCleanUpResource.java deleted file mode 100644 index e8d4ca30a0c0..000000000000 --- a/src/main/java/de/tum/in/www1/artemis/web/rest/admin/AdminOldDataCleanUpResource.java +++ /dev/null @@ -1,83 +0,0 @@ -package de.tum.in.www1.artemis.web.rest.admin; - -import static de.tum.in.www1.artemis.config.Constants.PROFILE_CORE; - -import java.time.ZonedDateTime; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.context.annotation.Profile; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; - -import de.tum.in.www1.artemis.security.annotations.EnforceAdmin; -import de.tum.in.www1.artemis.service.cleanup.OldDataCleanupService; - -/** - * REST controller for managing old data cleanup operations in Artemis. - * Provides endpoints for administrators to clean up old or orphaned data in the database. - */ -@RestController -@RequestMapping("api/admin/") -@Profile(PROFILE_CORE) -public class AdminOldDataCleanUpResource { - - private static final Logger log = LoggerFactory.getLogger(AdminOldDataCleanUpResource.class); - - private final OldDataCleanupService oldDataCleanupService; - - public AdminOldDataCleanUpResource(OldDataCleanupService oldDataCleanupService) { - this.oldDataCleanupService = oldDataCleanupService; - } - - @PostMapping("delete-orphans") - @EnforceAdmin - public ResponseEntity deleteOrphans() { - log.debug("REST request to delete orphaned data in Artemis database"); - oldDataCleanupService.deleteOrphans(); - return ResponseEntity.ok().build(); - } - - @PostMapping("delete-plagiarism-comparisons") - @EnforceAdmin - public ResponseEntity deletePlagiarismComparisons(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, @RequestParam("deleteTo") ZonedDateTime deleteTo) { - log.debug("REST request to delete plagiarism comparisons between {} and {}", deleteFrom, deleteTo); - oldDataCleanupService.deletePlagiarismComparisons(deleteFrom, deleteTo); - return ResponseEntity.ok().build(); - } - - @PostMapping("delete-non-rated-results") - @EnforceAdmin - public ResponseEntity deleteNonRatedResults(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, @RequestParam("deleteTo") ZonedDateTime deleteTo) { - log.debug("REST request to delete non-rated results between {} and {}", deleteFrom, deleteTo); - oldDataCleanupService.deleteNonRatedResults(deleteFrom, deleteTo); - return ResponseEntity.ok().build(); - } - - @PostMapping("delete-old-rated-results") - @EnforceAdmin - public ResponseEntity deleteOldRatedResults(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, @RequestParam("deleteTo") ZonedDateTime deleteTo) { - log.debug("REST request to delete old rated results between {} and {}", deleteFrom, deleteTo); - oldDataCleanupService.deleteOldRatedResults(deleteFrom, deleteTo); - return ResponseEntity.ok().build(); - } - - @PostMapping("delete-old-submission-versions") - @EnforceAdmin - public ResponseEntity deleteOldSubmissionVersions(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, @RequestParam("deleteTo") ZonedDateTime deleteTo) { - log.debug("REST request to delete old submission versions between {} and {}", deleteFrom, deleteTo); - oldDataCleanupService.deleteSubmissionVersions(deleteFrom, deleteTo); - return ResponseEntity.ok().build(); - } - - @PostMapping("delete-old-feedback") - @EnforceAdmin - public ResponseEntity deleteOldFeedback(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, @RequestParam("deleteTo") ZonedDateTime deleteTo) { - log.debug("REST request to delete old feedback between {} and {}", deleteFrom, deleteTo); - oldDataCleanupService.deleteOldFeedback(deleteFrom, deleteTo); - return ResponseEntity.ok().build(); - } -} diff --git a/src/main/java/de/tum/in/www1/artemis/web/rest/admin/AdminOldDataCleanupResource.java b/src/main/java/de/tum/in/www1/artemis/web/rest/admin/AdminOldDataCleanupResource.java new file mode 100644 index 000000000000..7da97bdbe16b --- /dev/null +++ b/src/main/java/de/tum/in/www1/artemis/web/rest/admin/AdminOldDataCleanupResource.java @@ -0,0 +1,97 @@ +package de.tum.in.www1.artemis.web.rest.admin; + +import static de.tum.in.www1.artemis.config.Constants.PROFILE_CORE; + +import java.time.ZonedDateTime; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Profile; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import de.tum.in.www1.artemis.security.annotations.EnforceAdmin; +import de.tum.in.www1.artemis.service.cleanup.DataCleanupService; +import de.tum.in.www1.artemis.web.rest.dto.CleanupServiceExecutionRecordDTO; + +/** + * REST controller for managing old data cleanup operations in Artemis. + * Provides endpoints for administrators to clean up old or orphaned data in the database. + */ +@RestController +@RequestMapping("api/admin/") +@Profile(PROFILE_CORE) +public class AdminOldDataCleanupResource { + + private static final Logger log = LoggerFactory.getLogger(AdminOldDataCleanupResource.class); + + private final DataCleanupService dataCleanupService; + + public AdminOldDataCleanupResource(DataCleanupService dataCleanupService) { + this.dataCleanupService = dataCleanupService; + } + + @PostMapping("delete-orphans") + @EnforceAdmin + public ResponseEntity deleteOrphans() { + log.debug("REST request to delete orphaned data in Artemis database"); + CleanupServiceExecutionRecordDTO result = dataCleanupService.deleteOrphans(); + return ResponseEntity.ok().body(result); + } + + @PostMapping("delete-plagiarism-comparisons") + @EnforceAdmin + public ResponseEntity deletePlagiarismComparisons(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, + @RequestParam("deleteTo") ZonedDateTime deleteTo) { + log.debug("REST request to delete plagiarism comparisons between {} and {}", deleteFrom, deleteTo); + CleanupServiceExecutionRecordDTO result = dataCleanupService.deletePlagiarismComparisons(deleteFrom, deleteTo); + return ResponseEntity.ok().body(result); + } + + @PostMapping("delete-non-rated-results") + @EnforceAdmin + public ResponseEntity deleteNonRatedResults(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, + @RequestParam("deleteTo") ZonedDateTime deleteTo) { + log.debug("REST request to delete non-rated results between {} and {}", deleteFrom, deleteTo); + CleanupServiceExecutionRecordDTO result = dataCleanupService.deleteNonRatedResults(deleteFrom, deleteTo); + return ResponseEntity.ok().body(result); + } + + @PostMapping("delete-old-rated-results") + @EnforceAdmin + public ResponseEntity deleteOldRatedResults(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, + @RequestParam("deleteTo") ZonedDateTime deleteTo) { + log.debug("REST request to delete old rated results between {} and {}", deleteFrom, deleteTo); + CleanupServiceExecutionRecordDTO result = dataCleanupService.deleteRatedResults(deleteFrom, deleteTo); + return ResponseEntity.ok().body(result); + } + + @PostMapping("delete-old-submission-versions") + @EnforceAdmin + public ResponseEntity deleteOldSubmissionVersions(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, + @RequestParam("deleteTo") ZonedDateTime deleteTo) { + log.debug("REST request to delete old submission versions between {} and {}", deleteFrom, deleteTo); + CleanupServiceExecutionRecordDTO result = dataCleanupService.deleteSubmissionVersions(deleteFrom, deleteTo); + return ResponseEntity.ok().body(result); + } + + @PostMapping("delete-old-feedback") + @EnforceAdmin + public ResponseEntity deleteOldFeedback(@RequestParam("deleteFrom") ZonedDateTime deleteFrom, + @RequestParam("deleteTo") ZonedDateTime deleteTo) { + log.debug("REST request to delete old feedback between {} and {}", deleteFrom, deleteTo); + CleanupServiceExecutionRecordDTO result = dataCleanupService.deleteFeedback(deleteFrom, deleteTo); + return ResponseEntity.ok().body(result); + } + + @PostMapping("get-last-executions") + @EnforceAdmin + public ResponseEntity> getLastExecutions() { + List result = dataCleanupService.getLastExecutions(); + return ResponseEntity.ok().body(result); + } +} diff --git a/src/main/java/de/tum/in/www1/artemis/web/rest/dto/CleanupServiceExecutionRecordDTO.java b/src/main/java/de/tum/in/www1/artemis/web/rest/dto/CleanupServiceExecutionRecordDTO.java new file mode 100644 index 000000000000..cd8c90c6998d --- /dev/null +++ b/src/main/java/de/tum/in/www1/artemis/web/rest/dto/CleanupServiceExecutionRecordDTO.java @@ -0,0 +1,12 @@ +package de.tum.in.www1.artemis.web.rest.dto; + +import java.time.ZonedDateTime; + +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * todo + */ +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public record CleanupServiceExecutionRecordDTO(ZonedDateTime executionDate) { +} diff --git a/src/main/webapp/app/admin/cleanup-service/cleanup-operation.model.ts b/src/main/webapp/app/admin/cleanup-service/cleanup-operation.model.ts index e37634182475..287fdb8bb156 100644 --- a/src/main/webapp/app/admin/cleanup-service/cleanup-operation.model.ts +++ b/src/main/webapp/app/admin/cleanup-service/cleanup-operation.model.ts @@ -12,5 +12,5 @@ export interface CleanupOperation { name: OperationName; deleteFrom: dayjs.Dayjs; deleteTo: dayjs.Dayjs; - lastExecuted: dayjs.Dayjs; + lastExecuted: dayjs.Dayjs | undefined; } diff --git a/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.html b/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.html index fc2bc6b2e30c..89a386554c5d 100644 --- a/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.html +++ b/src/main/webapp/app/admin/cleanup-service/cleanup-service.component.html @@ -5,13 +5,13 @@

    - - - - - - - + + + + + + + @for (operation of cleanupOperations; track operation) { @@ -49,7 +49,11 @@

    }

    OperationDelete FromDelete ToLast Executed
    OperationDelete FromDelete ToLast Executed
    - {{ operation.lastExecuted | artemisDate }} + @if (operation.lastExecuted === undefined) { + {{ 'cleanupService.notRunYet' | artemisTranslate }} + } @else { + {{ operation.lastExecuted | artemisDate }} + }