diff --git a/pom.xml b/pom.xml index 5bc42335b..828e48646 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ 17 UTF-8 - 2.0.73 + 2.0.74 5.6.15.Final 10.0.22 5.3.18 diff --git a/src/main/java/ai/elimu/dao/VideoLearningEventDao.java b/src/main/java/ai/elimu/dao/VideoLearningEventDao.java new file mode 100644 index 000000000..7521f6bc8 --- /dev/null +++ b/src/main/java/ai/elimu/dao/VideoLearningEventDao.java @@ -0,0 +1,10 @@ +package ai.elimu.dao; + +import ai.elimu.model.analytics.VideoLearningEvent; +import java.util.Calendar; +import org.springframework.dao.DataAccessException; + +public interface VideoLearningEventDao extends GenericDao { + + VideoLearningEvent read(Calendar timestamp, String androidId, String packageName, String videoTitle) throws DataAccessException; +} diff --git a/src/main/java/ai/elimu/dao/jpa/VideoLearningEventDaoJpa.java b/src/main/java/ai/elimu/dao/jpa/VideoLearningEventDaoJpa.java new file mode 100644 index 000000000..aca77c09a --- /dev/null +++ b/src/main/java/ai/elimu/dao/jpa/VideoLearningEventDaoJpa.java @@ -0,0 +1,31 @@ +package ai.elimu.dao.jpa; + +import ai.elimu.dao.VideoLearningEventDao; +import ai.elimu.model.analytics.VideoLearningEvent; +import java.util.Calendar; +import javax.persistence.NoResultException; +import org.springframework.dao.DataAccessException; + +public class VideoLearningEventDaoJpa extends GenericDaoJpa implements VideoLearningEventDao { + + @Override + public VideoLearningEvent read(Calendar timestamp, String androidId, String packageName, String videoTitle) throws DataAccessException { + try { + return (VideoLearningEvent) em.createQuery( + "SELECT event " + + "FROM VideoLearningEvent event " + + "WHERE event.timestamp = :timestamp " + + "AND event.androidId = :androidId " + + "AND event.packageName = :packageName " + + "AND event.videoTitle = :videoTitle") + .setParameter("timestamp", timestamp) + .setParameter("androidId", androidId) + .setParameter("packageName", packageName) + .setParameter("videoTitle", videoTitle) + .getSingleResult(); + } catch (NoResultException e) { + logger.info("VideoLearningEvent (" + timestamp.getTimeInMillis() + ", " + androidId + ", " + packageName + ", \"" + videoTitle + "\") was not found"); + return null; + } + } +} diff --git a/src/main/java/ai/elimu/model/analytics/LearningEvent.java b/src/main/java/ai/elimu/model/analytics/LearningEvent.java index 758cf08fb..a48e207dd 100644 --- a/src/main/java/ai/elimu/model/analytics/LearningEvent.java +++ b/src/main/java/ai/elimu/model/analytics/LearningEvent.java @@ -65,12 +65,7 @@ public void setTimestamp(Calendar timestamp) { } public String getAndroidId() { - if (!androidId.contains("***")) { - // Hide parts of the Android ID, e.g. "7161a85a0e4751cd" --> "7161***51cd" - return androidId.substring(0, 4) + "***" + androidId.substring(12); - } else { - return androidId; - } + return androidId; } public void setAndroidId(String androidId) { diff --git a/src/main/java/ai/elimu/tasks/analytics/VideoLearningEventImportScheduler.java b/src/main/java/ai/elimu/tasks/analytics/VideoLearningEventImportScheduler.java new file mode 100644 index 000000000..4f0140d81 --- /dev/null +++ b/src/main/java/ai/elimu/tasks/analytics/VideoLearningEventImportScheduler.java @@ -0,0 +1,102 @@ +package ai.elimu.tasks.analytics; + +import java.io.File; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import ai.elimu.dao.VideoLearningEventDao; +import ai.elimu.model.analytics.VideoLearningEvent; +import ai.elimu.model.v2.enums.Language; +import ai.elimu.rest.v2.analytics.VideoLearningEventsRestController; +import ai.elimu.util.ConfigHelper; +import ai.elimu.util.csv.CsvAnalyticsExtractionHelper; + +/** + * Extracts learning events from CSV files previously received by the + * {@link VideoLearningEventsRestController}, and imports them into the database. + *

+ * + * Expected folder structure: + *

+ * ├── lang-ENG
+ * │   ├── analytics
+ * │   │   ├── android-id-e387e38700000001
+ * │   │   │   └── version-code-3001018
+ * │   │   │       └── video-learning-events
+ * │   │   │           ├── e387e38700000001_3001018_video-learning-events_2024-10-09.csv
+ * │   │   │           ├── e387e38700000001_3001018_video-learning-events_2024-10-10.csv
+ * │   │   │           ├── e387e38700000001_3001018_video-learning-events_2024-10-11.csv
+ * │   │   │           ├── e387e38700000001_3001018_video-learning-events_2024-10-14.csv
+ * │   │   │           ├── e387e38700000001_3001018_video-learning-events_2024-10-18.csv
+ * │   │   │           └── e387e38700000001_3001018_video-learning-events_2024-10-20.csv
+ * │   │   ├── android-id-e387e38700000002
+ * │   │   │   └── version-code-3001018
+ * │   │   │       └── video-learning-events
+ * │   │   │           ├── e387e38700000002_3001018_video-learning-events_2024-10-09.csv
+ * │   │   │           ├── e387e38700000002_3001018_video-learning-events_2024-10-10.csv
+ * │   │   │           ├── e387e38700000002_3001018_video-learning-events_2024-10-11.csv
+ * 
+ */ +@Service +public class VideoLearningEventImportScheduler { + + private Logger logger = LogManager.getLogger(); + + @Autowired + private VideoLearningEventDao videoLearningEventDao; + + @Scheduled(cron="00 30 * * * *") // Half past every hour + public synchronized void execute() { + logger.info("execute"); + + // Lookup CSV files stored on the filesystem + File elimuAiDir = new File(System.getProperty("user.home"), ".elimu-ai"); + File languageDir = new File(elimuAiDir, "lang-" + Language.valueOf(ConfigHelper.getProperty("content.language"))); + File analyticsDir = new File(languageDir, "analytics"); + logger.info("analyticsDir: " + analyticsDir); + analyticsDir.mkdirs(); + for (File analyticsDirFile : analyticsDir.listFiles()) { + if (analyticsDirFile.getName().startsWith("android-id-")) { + File androidIdDir = new File(analyticsDir, analyticsDirFile.getName()); + for (File androidIdDirFile : androidIdDir.listFiles()) { + if (androidIdDirFile.getName().startsWith("version-code-")) { + File versionCodeDir = new File(androidIdDir, androidIdDirFile.getName()); + for (File versionCodeDirFile : versionCodeDir.listFiles()) { + if (versionCodeDirFile.getName().equals("video-learning-events")) { + File videoLearningEventsDir = new File(versionCodeDir, versionCodeDirFile.getName()); + for (File csvFile : videoLearningEventsDir.listFiles()) { + logger.info("csvFile: " + csvFile); + + // Convert from CSV to Java + List events = CsvAnalyticsExtractionHelper.extractVideoLearningEvents(csvFile); + logger.info("events.size(): " + events.size()); + + // Store in database + for (VideoLearningEvent event : events) { + // Check if the event has already been stored in the database + VideoLearningEvent existingVideoLearningEvent = videoLearningEventDao.read(event.getTimestamp(), event.getAndroidId(), event.getPackageName(), event.getVideoTitle()); + if (existingVideoLearningEvent != null) { + logger.warn("The event has already been stored in the database. Skipping data import."); + continue; + } + + // Store the event in the database + videoLearningEventDao.create(event); + logger.info("Stored event in database with ID " + event.getId()); + } + } + } + } + } + } + } + } + + logger.info("execute complete"); + } +} diff --git a/src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java b/src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java index 2500d5a35..9de7f4520 100644 --- a/src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java +++ b/src/main/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelper.java @@ -4,7 +4,10 @@ import ai.elimu.dao.StoryBookDao; import ai.elimu.model.admin.Application; import ai.elimu.model.analytics.StoryBookLearningEvent; +import ai.elimu.model.analytics.VideoLearningEvent; +import ai.elimu.model.analytics.WordLearningEvent; import ai.elimu.model.content.StoryBook; +import ai.elimu.model.content.Word; import ai.elimu.model.v2.enums.analytics.LearningEventType; import ai.elimu.rest.v2.analytics.StoryBookLearningEventsRestController; import ai.elimu.web.analytics.StoryBookLearningEventCsvExportController; @@ -15,6 +18,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.Calendar; import java.util.List; import org.apache.commons.csv.CSVFormat; @@ -22,6 +26,7 @@ import org.apache.commons.csv.CSVRecord; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.springframework.http.HttpStatus; public class CsvAnalyticsExtractionHelper { @@ -99,4 +104,68 @@ public static List getStoryBookLearningEventsFromCsvBack return storyBookLearningEvents; } + + public static List extractVideoLearningEvents(File csvFile) { + logger.info("extractVideoLearningEvents"); + + List videoLearningEvents = new ArrayList<>(); + + // Iterate each row in the CSV file + Path csvFilePath = Paths.get(csvFile.toURI()); + logger.info("csvFilePath: " + csvFilePath); + try { + Reader reader = Files.newBufferedReader(csvFilePath); + CSVFormat csvFormat = CSVFormat.DEFAULT + .withHeader( + "id", // The Android database ID + "timestamp", + "android_id", + "package_name", + "video_id", + "video_title", + "learning_event_type", + "additional_data" + ) + .withSkipHeaderRecord(); + logger.info("header: " + Arrays.toString(csvFormat.getHeader())); + CSVParser csvParser = new CSVParser(reader, csvFormat); + for (CSVRecord csvRecord : csvParser) { + logger.info("csvRecord: " + csvRecord); + + // Convert from CSV to Java + + VideoLearningEvent videoLearningEvent = new VideoLearningEvent(); + + long timestampInMillis = Long.valueOf(csvRecord.get("timestamp")); + Calendar timestamp = Calendar.getInstance(); + timestamp.setTimeInMillis(timestampInMillis); + videoLearningEvent.setTimestamp(timestamp); + + String androidId = csvRecord.get("android_id"); + videoLearningEvent.setAndroidId(androidId); + + String packageName = csvRecord.get("package_name"); + videoLearningEvent.setPackageName(packageName); + + Long videoId = Long.valueOf(csvRecord.get("video_id")); + videoLearningEvent.setVideoId(videoId); + + String videoTitle = csvRecord.get("video_title"); + videoLearningEvent.setVideoTitle(videoTitle); + + LearningEventType learningEventType = LearningEventType.valueOf(csvRecord.get("learning_event_type")); + videoLearningEvent.setLearningEventType(learningEventType); + + String additionalData = csvRecord.get("additional_data"); + videoLearningEvent.setAdditionalData(additionalData); + + videoLearningEvents.add(videoLearningEvent); + } + csvParser.close(); + } catch (IOException ex) { + logger.error(ex); + } + + return videoLearningEvents; + } } diff --git a/src/main/webapp/WEB-INF/spring/applicationContext-jpa.xml b/src/main/webapp/WEB-INF/spring/applicationContext-jpa.xml index ca09b9f50..7380d2ed6 100644 --- a/src/main/webapp/WEB-INF/spring/applicationContext-jpa.xml +++ b/src/main/webapp/WEB-INF/spring/applicationContext-jpa.xml @@ -74,6 +74,7 @@ + diff --git a/src/test/java/ai/elimu/dao/VideoLearningEventDaoJpaTest.java b/src/test/java/ai/elimu/dao/VideoLearningEventDaoJpaTest.java new file mode 100644 index 000000000..121350769 --- /dev/null +++ b/src/test/java/ai/elimu/dao/VideoLearningEventDaoJpaTest.java @@ -0,0 +1,53 @@ +package ai.elimu.dao; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.util.Calendar; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; + +import ai.elimu.model.analytics.VideoLearningEvent; + +@SpringJUnitConfig(locations = { + "file:src/main/webapp/WEB-INF/spring/applicationContext.xml", + "file:src/main/webapp/WEB-INF/spring/applicationContext-jpa.xml" +}) +public class VideoLearningEventDaoJpaTest { + + @Autowired + private VideoLearningEventDao videoLearningEventDao; + + @Test + public void testRead() { + Calendar timestamp = Calendar.getInstance(); + String androidId = "e387e38700000001"; + String packageName = "ai.elimu.filamu"; + String videoTitle = "akili and me - the rectangle song"; + + VideoLearningEvent existingEvent = videoLearningEventDao.read( + timestamp, + androidId, + packageName, + videoTitle + ); + assertNull(existingEvent); + + VideoLearningEvent event = new VideoLearningEvent(); + event.setTimestamp(timestamp); + event.setAndroidId(androidId); + event.setPackageName(packageName); + event.setVideoTitle(videoTitle); + videoLearningEventDao.create(event); + + existingEvent = videoLearningEventDao.read( + timestamp, + androidId, + packageName, + videoTitle + ); + assertNotNull(existingEvent); + } +} diff --git a/src/test/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelperTest.java b/src/test/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelperTest.java new file mode 100644 index 000000000..5dbe17c62 --- /dev/null +++ b/src/test/java/ai/elimu/util/csv/CsvAnalyticsExtractionHelperTest.java @@ -0,0 +1,55 @@ +package ai.elimu.util.csv; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.jupiter.api.Test; +import org.springframework.core.io.ClassRelativeResourceLoader; +import org.springframework.core.io.Resource; +import org.springframework.core.io.ResourceLoader; + +import ai.elimu.model.analytics.VideoLearningEvent; +import ai.elimu.model.v2.enums.analytics.LearningEventType; + +public class CsvAnalyticsExtractionHelperTest { + + private Logger logger = LogManager.getLogger(); + + @Test + public void testExtractVideoLearningEvents() throws IOException { + ResourceLoader resourceLoader = new ClassRelativeResourceLoader(CsvAnalyticsExtractionHelper.class); + Resource resource = resourceLoader.getResource("e387e38700000001_3001018_video-learning-events_2024-10-09.csv"); + File csvFile = resource.getFile(); + logger.debug("csvFile: " + csvFile); + + List videoLearningEvents = CsvAnalyticsExtractionHelper.extractVideoLearningEvents(csvFile); + assertEquals(6, videoLearningEvents.size()); + + // Test the 1st row of data + VideoLearningEvent event1st = videoLearningEvents.get(0); + assertEquals(1728486312687L, event1st.getTimestamp().getTimeInMillis()); + assertEquals("e387e38700000001", event1st.getAndroidId()); + assertEquals("ai.elimu.analytics", event1st.getPackageName()); + assertEquals(13, event1st.getVideoId()); + assertEquals("akili and me - the rectangle song", event1st.getVideoTitle()); + assertEquals(LearningEventType.VIDEO_OPENED, event1st.getLearningEventType()); + assertEquals("", event1st.getAdditionalData()); + + // Test the 2nd row of data + VideoLearningEvent event2nd = videoLearningEvents.get(1); + assertEquals(1728486319885L, event2nd.getTimestamp().getTimeInMillis()); + assertEquals("e387e38700000001", event2nd.getAndroidId()); + assertEquals("ai.elimu.analytics", event2nd.getPackageName()); + assertEquals(13, event2nd.getVideoId()); + assertEquals("akili and me - the rectangle song", event2nd.getVideoTitle()); + assertEquals(LearningEventType.VIDEO_CLOSED_BEFORE_COMPLETION, event2nd.getLearningEventType()); + assertEquals("{'video_playback_position_ms': 58007}", event2nd.getAdditionalData()); + } +} diff --git a/src/test/resources/ai/elimu/util/csv/e387e38700000001_3001018_video-learning-events_2024-10-09.csv b/src/test/resources/ai/elimu/util/csv/e387e38700000001_3001018_video-learning-events_2024-10-09.csv new file mode 100644 index 000000000..0cf083e8e --- /dev/null +++ b/src/test/resources/ai/elimu/util/csv/e387e38700000001_3001018_video-learning-events_2024-10-09.csv @@ -0,0 +1,7 @@ +id,timestamp,android_id,package_name,video_id,video_title,learning_event_type,additional_data +0,1728486312687,e387e38700000001,ai.elimu.analytics,13,akili and me - the rectangle song,VIDEO_OPENED, +0,1728486319885,e387e38700000001,ai.elimu.analytics,13,akili and me - the rectangle song,VIDEO_CLOSED_BEFORE_COMPLETION,{'video_playback_position_ms': 58007} +0,1728486312687,e387e38700000001,ai.elimu.analytics,6,akili and me - letter f,VIDEO_OPENED, +0,1728486316065,e387e38700000001,ai.elimu.analytics,6,akili and me - letter f,VIDEO_CLOSED_BEFORE_COMPLETION,{'video_playback_position_ms': 6831} +0,1728486312687,e387e38700000001,ai.elimu.analytics,34,akili and me - letter j,VIDEO_OPENED, +0,1728486363791,e387e38700000001,ai.elimu.analytics,34,akili and me - letter j,VIDEO_CLOSED_BEFORE_COMPLETION,{'video_playback_position_ms': 32243}