From f3810d82d10531bcf61307a8fdded4fe3551b95c Mon Sep 17 00:00:00 2001 From: jo-elimu <1451036+jo-elimu@users.noreply.github.com> Date: Sat, 26 Oct 2024 11:12:12 +0700 Subject: [PATCH 1/3] feat(analytics): add video learning events to dashboard #1894 --- src/main/webapp/WEB-INF/jsp/analytics/layout.jsp | 1 + src/main/webapp/WEB-INF/jsp/analytics/main.jsp | 11 +++++++++++ .../jsp/analytics/video-learning-event/list.jsp | 0 3 files changed, 12 insertions(+) create mode 100644 src/main/webapp/WEB-INF/jsp/analytics/video-learning-event/list.jsp diff --git a/src/main/webapp/WEB-INF/jsp/analytics/layout.jsp b/src/main/webapp/WEB-INF/jsp/analytics/layout.jsp index 56c03bfcf..a0b07cbba 100644 --- a/src/main/webapp/WEB-INF/jsp/analytics/layout.jsp +++ b/src/main/webapp/WEB-INF/jsp/analytics/layout.jsp @@ -53,6 +53,7 @@
  • text_format
  • sms
  • book
    • +
  • movieVideos
    • dehaze diff --git a/src/main/webapp/WEB-INF/jsp/analytics/main.jsp b/src/main/webapp/WEB-INF/jsp/analytics/main.jsp index 1ed8b62d5..5a6e8e7c9 100644 --- a/src/main/webapp/WEB-INF/jsp/analytics/main.jsp +++ b/src/main/webapp/WEB-INF/jsp/analytics/main.jsp @@ -40,6 +40,17 @@ + +
    +
    +
    + movie Videos +
    +
    + View list (${storyBookLearningEventCount}) +
    +
    +
    diff --git a/src/main/webapp/WEB-INF/jsp/analytics/video-learning-event/list.jsp b/src/main/webapp/WEB-INF/jsp/analytics/video-learning-event/list.jsp new file mode 100644 index 000000000..e69de29bb From 8916fc096dbbe911f9a2df9f99d54cd561bae87a Mon Sep 17 00:00:00 2001 From: jo-elimu <1451036+jo-elimu@users.noreply.github.com> Date: Wed, 30 Oct 2024 16:25:37 +0700 Subject: [PATCH 2/3] fix(analytics): use correct timezone when extracting timestamp from csv Resolves #1924 --- .../java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java b/src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java index 9de7f4520..96278034d 100644 --- a/src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java +++ b/src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java @@ -21,6 +21,8 @@ import java.util.Arrays; import java.util.Calendar; import java.util.List; +import java.util.TimeZone; + import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; @@ -137,7 +139,7 @@ public static List extractVideoLearningEvents(File csvFile) VideoLearningEvent videoLearningEvent = new VideoLearningEvent(); long timestampInMillis = Long.valueOf(csvRecord.get("timestamp")); - Calendar timestamp = Calendar.getInstance(); + Calendar timestamp = Calendar.getInstance(TimeZone.getTimeZone("UTC")); timestamp.setTimeInMillis(timestampInMillis); videoLearningEvent.setTimestamp(timestamp); From d4bcc46a3570bcf3010d969a01bf3b639259a5fc Mon Sep 17 00:00:00 2001 From: jo-elimu <1451036+jo-elimu@users.noreply.github.com> Date: Wed, 30 Oct 2024 18:43:54 +0700 Subject: [PATCH 3/3] feat(analytics): dashboard #1894 --- .../java/ai/elimu/util/AnalyticsHelper.java | 10 +++ .../analytics/MainAnalyticsController.java | 5 ++ ...VideoLearningEventCsvExportController.java | 90 +++++++++++++++++++ .../VideoLearningEventListController.java | 73 +++++++++++++++ .../webapp/WEB-INF/jsp/analytics/main.jsp | 2 +- .../analytics/video-learning-event/list.jsp | 88 ++++++++++++++++++ src/main/webapp/static/css/styles.css | 5 ++ .../ai/elimu/util/AnalyticsHelperTest.java | 6 ++ 8 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 src/main/java/ai/elimu/web/analytics/VideoLearningEventCsvExportController.java create mode 100644 src/main/java/ai/elimu/web/analytics/VideoLearningEventListController.java diff --git a/src/main/java/ai/elimu/util/AnalyticsHelper.java b/src/main/java/ai/elimu/util/AnalyticsHelper.java index ebfcfbaaf..acc3e31f8 100644 --- a/src/main/java/ai/elimu/util/AnalyticsHelper.java +++ b/src/main/java/ai/elimu/util/AnalyticsHelper.java @@ -30,4 +30,14 @@ public static Integer extractVersionCodeFromCsvFilename(String filename) { Integer versionCode = Integer.valueOf(versionCodeAsString); return versionCode; } + + /** + * E.g. "7161a85a0e4751cd" --> "7161***51cd" + * + * @param androidId The Android ID + * @return The redacted version of the Android ID + */ + public static String redactAndroidId(String androidId) { + return androidId.substring(0, 4) + "***" + androidId.substring(12); + } } diff --git a/src/main/java/ai/elimu/web/analytics/MainAnalyticsController.java b/src/main/java/ai/elimu/web/analytics/MainAnalyticsController.java index 156d9546f..c722a65ce 100644 --- a/src/main/java/ai/elimu/web/analytics/MainAnalyticsController.java +++ b/src/main/java/ai/elimu/web/analytics/MainAnalyticsController.java @@ -2,6 +2,7 @@ import ai.elimu.dao.LetterLearningEventDao; import ai.elimu.dao.StoryBookLearningEventDao; +import ai.elimu.dao.VideoLearningEventDao; import ai.elimu.dao.WordLearningEventDao; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -26,6 +27,9 @@ public class MainAnalyticsController { @Autowired private StoryBookLearningEventDao storyBookLearningEventDao; + @Autowired + private VideoLearningEventDao videoLearningEventDao; + @RequestMapping(method = RequestMethod.GET) public String handleRequest(Model model) { logger.info("handleRequest"); @@ -33,6 +37,7 @@ public String handleRequest(Model model) { model.addAttribute("letterLearningEventCount", letterLearningEventDao.readCount()); model.addAttribute("wordLearningEventCount", wordLearningEventDao.readCount()); model.addAttribute("storyBookLearningEventCount", storyBookLearningEventDao.readCount()); + model.addAttribute("videoLearningEventCount", videoLearningEventDao.readCount()); return "analytics/main"; } diff --git a/src/main/java/ai/elimu/web/analytics/VideoLearningEventCsvExportController.java b/src/main/java/ai/elimu/web/analytics/VideoLearningEventCsvExportController.java new file mode 100644 index 000000000..f7c98d193 --- /dev/null +++ b/src/main/java/ai/elimu/web/analytics/VideoLearningEventCsvExportController.java @@ -0,0 +1,90 @@ +package ai.elimu.web.analytics; + +import ai.elimu.dao.VideoLearningEventDao; +import ai.elimu.model.analytics.VideoLearningEvent; +import ai.elimu.util.AnalyticsHelper; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.StringWriter; +import java.util.List; + +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.RequestMapping; + +import javax.servlet.http.HttpServletResponse; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.apache.logging.log4j.LogManager; +import org.springframework.web.bind.annotation.RequestMethod; + +@Controller +@RequestMapping("/analytics/video-learning-event/list") +public class VideoLearningEventCsvExportController { + + private final Logger logger = LogManager.getLogger(); + + @Autowired + private VideoLearningEventDao videoLearningEventDao; + + @RequestMapping(value="/video-learning-events.csv", method = RequestMethod.GET) + public void handleRequest( + HttpServletResponse response, + OutputStream outputStream + ) throws IOException { + logger.info("handleRequest"); + + List videoLearningEvents = videoLearningEventDao.readAll(); + logger.info("videoLearningEvents.size(): " + videoLearningEvents.size()); + + CSVFormat csvFormat = CSVFormat.DEFAULT.builder() + .setHeader( + "id", // The Room database ID + "timestamp", + "android_id", + "package_name", + "video_id", + "video_title", + "learning_event_type", + "additional_data" + ) + .build(); + + StringWriter stringWriter = new StringWriter(); + CSVPrinter csvPrinter = new CSVPrinter(stringWriter, csvFormat); + + for (VideoLearningEvent videoLearningEvent : videoLearningEvents) { + logger.info("videoLearningEvent.getId(): " + videoLearningEvent.getId()); + + videoLearningEvent.setAndroidId(AnalyticsHelper.redactAndroidId(videoLearningEvent.getAndroidId())); + + csvPrinter.printRecord( + videoLearningEvent.getId(), + videoLearningEvent.getTimestamp().getTimeInMillis(), + videoLearningEvent.getAndroidId(), + videoLearningEvent.getPackageName(), + videoLearningEvent.getVideoId(), + videoLearningEvent.getVideoTitle(), + videoLearningEvent.getLearningEventType(), + videoLearningEvent.getAdditionalData() + ); + csvPrinter.flush(); + } + csvPrinter.close(); + + String csvFileContent = stringWriter.toString(); + + response.setContentType("text/csv"); + byte[] bytes = csvFileContent.getBytes(); + response.setContentLength(bytes.length); + try { + outputStream.write(bytes); + outputStream.flush(); + outputStream.close(); + } catch (IOException ex) { + logger.error(ex); + } + } +} diff --git a/src/main/java/ai/elimu/web/analytics/VideoLearningEventListController.java b/src/main/java/ai/elimu/web/analytics/VideoLearningEventListController.java new file mode 100644 index 000000000..7dba17bbc --- /dev/null +++ b/src/main/java/ai/elimu/web/analytics/VideoLearningEventListController.java @@ -0,0 +1,73 @@ +package ai.elimu.web.analytics; + +import ai.elimu.dao.VideoLearningEventDao; +import ai.elimu.model.analytics.VideoLearningEvent; +import ai.elimu.util.AnalyticsHelper; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Controller; +import org.springframework.ui.Model; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; + +@Controller +@RequestMapping("/analytics/video-learning-event/list") +public class VideoLearningEventListController { + + private final Logger logger = LogManager.getLogger(); + + @Autowired + private VideoLearningEventDao videoLearningEventDao; + + @RequestMapping(method = RequestMethod.GET) + public String handleRequest(Model model) { + logger.info("handleRequest"); + + List videoLearningEvents = videoLearningEventDao.readAll(); + for (VideoLearningEvent videoLearningEvent : videoLearningEvents) { + videoLearningEvent.setAndroidId(AnalyticsHelper.redactAndroidId(videoLearningEvent.getAndroidId())); + } + model.addAttribute("videoLearningEvents", videoLearningEvents); + + // Prepare chart data + List monthList = new ArrayList<>(); + List eventCountList = new ArrayList<>(); + if (!videoLearningEvents.isEmpty()) { + // Group event count by month (e.g. "Aug-2024", "Sep-2024") + Map eventCountByMonthMap = new HashMap<>(); + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("MMM-yyyy"); + for (VideoLearningEvent event : videoLearningEvents) { + String eventMonth = simpleDateFormat.format(event.getTimestamp().getTime()); + eventCountByMonthMap.put(eventMonth, eventCountByMonthMap.getOrDefault(eventMonth, 0) + 1); + } + + // Iterate each month from 4 years ago until now + Calendar calendar4YearsAgo = Calendar.getInstance(); + calendar4YearsAgo.add(Calendar.YEAR, -4); + Calendar calendarNow = Calendar.getInstance(); + Calendar month = calendar4YearsAgo; + while (!month.after(calendarNow)) { + String monthAsString = simpleDateFormat.format(month.getTime()); + monthList.add(monthAsString); + + eventCountList.add(eventCountByMonthMap.getOrDefault(monthAsString, 0)); + + // Increase the date by 1 month + month.add(Calendar.MONTH, 1); + } + } + model.addAttribute("monthList", monthList); + model.addAttribute("eventCountList", eventCountList); + + return "analytics/video-learning-event/list"; + } +} diff --git a/src/main/webapp/WEB-INF/jsp/analytics/main.jsp b/src/main/webapp/WEB-INF/jsp/analytics/main.jsp index 5a6e8e7c9..2b54bb66b 100644 --- a/src/main/webapp/WEB-INF/jsp/analytics/main.jsp +++ b/src/main/webapp/WEB-INF/jsp/analytics/main.jsp @@ -47,7 +47,7 @@ movie Videos
    - View list (${storyBookLearningEventCount}) + View list (${videoLearningEventCount})
    diff --git a/src/main/webapp/WEB-INF/jsp/analytics/video-learning-event/list.jsp b/src/main/webapp/WEB-INF/jsp/analytics/video-learning-event/list.jsp index e69de29bb..f81c5584d 100644 --- a/src/main/webapp/WEB-INF/jsp/analytics/video-learning-event/list.jsp +++ b/src/main/webapp/WEB-INF/jsp/analytics/video-learning-event/list.jsp @@ -0,0 +1,88 @@ + + VideoLearningEvents (${fn:length(videoLearningEvents)}) + + + +
    +
    + + + +
    +
    + +
    + + vertical_align_bottom + + + + + + + + + + + + + + + + + + + + + + +
    timestampandroid_idpackage_namevideo_titlelearning_event_type
    + + + ${videoLearningEvent.androidId} + + ${videoLearningEvent.packageName} + + + + "${videoLearningEvent.video.title}" + + + "${videoLearningEvent.videoTitle}" + + + + ${videoLearningEvent.learningEventType} +
    +
    +     diff --git a/src/main/webapp/static/css/styles.css b/src/main/webapp/static/css/styles.css index 3c3441dc4..42a4c12bb 100644 --- a/src/main/webapp/static/css/styles.css +++ b/src/main/webapp/static/css/styles.css @@ -1,6 +1,11 @@ html { font-family: monospace, sans-serif; } +code { + background-color: rgba(103,58,183, 0.05); /* deep-purple */ + border-radius: 4px; + padding: 4px; +} body { background: #EEE; diff --git a/src/test/java/ai/elimu/util/AnalyticsHelperTest.java b/src/test/java/ai/elimu/util/AnalyticsHelperTest.java index 61481898d..d7ddfcc80 100644 --- a/src/test/java/ai/elimu/util/AnalyticsHelperTest.java +++ b/src/test/java/ai/elimu/util/AnalyticsHelperTest.java @@ -23,4 +23,10 @@ public void testExtractVersionCodeFromCsvFilename() { filename = "7161a85a0e4751cd_3001012_word-learning-events_2020-04-23.csv"; assertEquals(3001012, AnalyticsHelper.extractVersionCodeFromCsvFilename(filename)); } + + @Test + public void testRedactAndroidId() { + String androidId = "745f90e7aae26423"; + assertEquals("745f***6423", AnalyticsHelper.redactAndroidId(androidId)); + } }