feat(server): Restructure videos directory by label (#92)

BREAKING CHANGE: The structure of the videos directory has changed to include video labels as subdirectories, and you will need to adjust your workflow accordingly. Details of the new structure are available in the README at https://github.com/gafirst/match-uploader/blob/main/README.md#video-directory-structure.

.github/workflows/task-list-checker.yml

@@ -0,0 +1,14 @@
+# https://github.com/Shopify/task-list-checker
+name: pr
+on:
+  pull_request:
+    types: [opened, edited, synchronize, reopened]
+jobs:
+  task-list-checker:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.pull_request.user.login != 'dependabot[bot]' }}
+    steps:
+      - name: Check for incomplete task list items
+        uses: Shopify/task-list-checker@main
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

README.md

@@ -39,6 +39,50 @@ To get started:
    4. YouTube playlist mappings: If you have playlists that you'd like match videos added to, follow the instructions in this section to set this up.
    5. Video description template: While no warning appears for this, you should double-check that the default video
       description template fits your needs and adjust it as needed.
+   6. Read below on how to structure your `videos` directory, which is where you'll place the video files for matches.
+
+### Video directory structure
+
+Match Uploader expects a specific directory structure for your videos. When running Match Uploader in Docker, you can
+mount any directory (such as one that your video production software writes recordings to) as the videos volume (for
+specifics, see [Docker volumes](#docker-volumes), below).
+
+The expected directory structure is as follows:
+```
+videos/
+├─ unlabeled/
+│  ├─ Qualification 1.mp4
+├─ $LABEL/
+│  ├─ Qualification 1.mp4
+```
+
+> [!TIP]
+> The `.mp4` video extension is just an example. You can use any file type that YouTube supports.
+
+A video label is an extra description for when you have multiple videos to upload for one match. Match Uploader will
+include the video label in the middle of the video title, e.g., `Qualification Match 1 - $LABEL - Event Name`.
+
+**What if I don't want a label in the video title?** A video with no label is labeled `unlabeled` (so you would put
+videos that should be unlabeled in a directory called `unlabeled`). This will title the video like 
+`Qualification Match 1 - Event Name`.
+
+After being uploaded, videos are moved to a directory called `uploaded` within each label directory. (You don't need to
+create the `uploaded` directories; they'll get created automatically when needed.) For instance:
+```
+videos/
+├─ unlabeled/
+│  ├─ uploaded/
+│  │  ├─ Qualification 1.mp4
+|  ├─ Qualification 2.mp4
+├─ $LABEL/
+│  ├─ uploaded/
+│  │  ├─ Qualification 1.mp4
+|  ├─ Qualification 2.mp4
+```
+
+> [!CAUTION]
+> Don't mount your videos directory as a read-only Docker volume. Otherwise, the server won't be able to move videos to 
+> the `uploaded` directories.
 
 ### Docker Compose setup in-depth
 
@@ -76,6 +120,7 @@ defined in the file that are not specified below; please leave those intact.
 
 There are 3 required Docker volumes for the `web` and `worker` containers:
 - **Videos:** Mount your local videos directory as a volume to `/home/node/app/server/videos`
+    - The directory structure is described in [Video directory structure](#video-directory-structure)
 - **Environment variables:** Server environment variables located in `/home/node/app/server/env`
   - Make a copy of [`server/env/production.env.example`](server/env/production.env.example) and fill in the values.
     Descriptions of what you need to fill in are described [above](#environment-variables).

client/src/components/help/MissingMatchVideosHelp.vue

@@ -9,10 +9,6 @@
         <li>
           Ensure your video files are named correctly (review <strong>How to name match video files</strong>).
         </li>
-        <li>
-          Go to <strong>Settings</strong> and confirm that the value of the <strong>Video search directory</strong>
-          setting is correct.
-        </li>
       </ol>
     </VExpansionPanelText>
   </VExpansionPanel>

client/src/components/help/NameMatchVideoFilesHelp.vue

@@ -5,26 +5,42 @@
       How to name match video files
     </VExpansionPanelTitle>
     <VExpansionPanelText>
+      <p>
+        For a more detailed explanation, see the
+        <a href="https://github.com/gafirst/match-uploader/blob/main/README.md#video-directory-structure"
+           target="_blank"
+        >
+          Video directory structure</a> section in the Match Uploader README.
+      </p>
+      <br />
       <p>File name matching is <strong>not</strong> case-sensitive.</p>
       <br />
       <p>
         The .mp4 extension is used in examples as a placeholder, but you may use any file extension/type that is
         accepted for uploads on YouTube.
       </p>
       <br />
+      <p>
+        Within the videos directory, place match videos within subdirectories for each video label (or
+        use <code>unlabeled</code> for no label) that you have:
+      </p>
+      <pre>
+videos/
+├─ unlabeled/
+│  ├─ Qualification 1.mp4
+│  ├─ Qualification 2.mp4
+├─ $LABEL/
+│  ├─ Qualification 1.mp4
+│  ├─ Qualification 2.mp4
+        </pre>
       <strong>Qualification matches:</strong>
-      <pre>Qualification #[ Label].mp4</pre>
-      <pre>Qualification #[ Label].mp4</pre>
-      <p><strong>Examples</strong>: Qualification 1.mp4, Qualification 1 Overhead.mp4, Qualification 1.wav</p>
+      <pre>$Label/Qualification #.mp4</pre>
+      <p><strong>Examples</strong>: unlabeled/Qualification 1.mp4, Overhead/Qualification 1.mp4</p>
       <br />
       <strong>Double elimination playoff matches:</strong>
-      <pre>Playoff #[ Label].mp4</pre>
-      <br />
-      <strong>Best of 3 playoff matches:</strong>
-      <pre>Quarterfinal #[ Label].mp4</pre>
-      <pre>Semifinal #[ Label].mp4</pre>
-      <pre>Final #[ Label].mp4</pre>
-
+      <pre>$Label/Playoff #.mp4</pre>
+      <pre>$Label/Final #.mp4</pre>
+      <p><strong>Examples</strong>: Overhead/Playoff 10.mp4, Feed B/Final 1.mp4</p>
       <br />
       Be sure to set your playoff type in Settings so we know how to parse your playoff matches!
     </VExpansionPanelText>

client/src/components/matches/MatchVideoListItem.vue

@@ -8,7 +8,7 @@
              size="large"
       />
     </template>
-    <template v-slot:append>
+    <template v-if="!video.isUploaded" v-slot:append>
       <VBtn v-if="videoJob?.youTubeVideoId"
             variant="text"
             icon="mdi-open-in-new"
@@ -53,6 +53,10 @@ interface IProps {
 const props = defineProps<IProps>();
 
 const uploadStatus = computed(() => {
+  if (props.video.isUploaded) {
+    return "Uploaded";
+  }
+
   if (props.video.isRequestingJob) {
     return "Creating job";
   }
@@ -122,6 +126,13 @@ const icon = computed((): {
   icon: string;
   color: string;
 } => {
+  if (props.video.isUploaded) {
+    return {
+      icon: "mdi-cloud-check-variant",
+      color: "success",
+    };
+  }
+
   if (props.video.isRequestingJob) {
     return {
       icon: "mdi-loading mdi-spin",

client/src/components/matches/MatchVideosUploader.vue

@@ -44,7 +44,6 @@
               color="success"
               variant="tonal"
               icon="mdi-check-circle"
-              class="mt-2 mb-4"
       >
         All videos uploaded!
       </VAlert>
@@ -57,8 +56,8 @@
       >
         {{ queueAllBtnText }}
       </VBtn>
-      <SandboxModeAlert class="mt-4" :rounded="4" />
-      <PrivateUploads class="mt-4" :rounded="4" />
+      <SandboxModeAlert class="mt-2" :rounded="4" />
+      <PrivateUploads class="mt-2" :rounded="4" />
     </VCardText>
   </VCard>
 </template>
@@ -97,6 +96,9 @@ const queueAllBtnText = computed(() => {
   if (matchStore.uploadInProgress) {
     return "Uploading...";
   }
+  if (matchStore.someMatchVideosUploaded) {
+    return "Queue all remaining";
+  }
   return "Queue all";
 });
 </script>

client/src/stores/match.ts

@@ -82,14 +82,21 @@ export const useMatchStore = defineStore("match", () => {
 
   const allMatchVideosQueued = computed(() => {
     return matchVideos.value.length > 0 &&
-      matchVideos.value.every(video => !!video.workerJobId);
+      matchVideos.value.every(video => video.isUploaded || !!video.workerJobId);
   });
 
+  function videoIsUploaded(video: MatchVideoInfo): boolean {
+    return video.isUploaded || workerStore.jobHasStatus(video.workerJobId,[WorkerJobStatus.COMPLETED]);
+  }
+
   const allMatchVideosUploaded = computed(() => {
     return matchVideos.value.length > 0 &&
-      matchVideos.value.every(
-        video => workerStore.jobHasStatus(video.workerJobId,[WorkerJobStatus.COMPLETED]),
-      );
+      matchVideos.value.every(video => videoIsUploaded(video));
+  });
+
+  const someMatchVideosUploaded = computed(() => {
+    return matchVideos.value.length > 0 &&
+      matchVideos.value.some(video => videoIsUploaded(video));
   });
 
   async function uploadVideo(video: MatchVideoInfo): Promise<any> {
@@ -153,7 +160,9 @@ export const useMatchStore = defineStore("match", () => {
   async function uploadVideos(): Promise<void> {
     uploadInProgress.value = true;
     for (const video of matchVideos.value) {
-      await uploadVideo(video);
+      if (!video.isUploaded) {
+        await uploadVideo(video);
+      }
     }
     uploadInProgress.value = false;
   }
@@ -232,6 +241,7 @@ export const useMatchStore = defineStore("match", () => {
     postUploadStepsSucceeded,
     selectMatch,
     selectedMatchKey,
+    someMatchVideosUploaded,
     uploadInProgress,
     uploadSingleVideo,
     uploadVideos,

client/src/types/MatchVideoInfo.ts

@@ -8,4 +8,5 @@ export interface MatchVideoInfo {
     workerJobId: string | null;
     jobCreationError: string | null;
     isRequestingJob: boolean;
+    isUploaded: boolean;
 }

server/.eslintrc.json

@@ -50,7 +50,8 @@
     "@typescript-eslint/no-misused-promises": "off",
     "no-extra-boolean-cast": "off",
     "node/no-missing-import": "off",
-    "node/no-unpublished-import": "off"
+    "node/no-unpublished-import": "off",
+    "no-unused-vars": "off"
   },
   "ignorePatterns": ["**/.eslintrc.*", "src/public/"],
   "settings": {

server/README.md

@@ -34,7 +34,7 @@ Run the production build (Must be built first).
 Run production build with a different env file.
 
 ## Linting
-## Linting
+
 ```
 # lint
 yarn lint

server/package.json

@@ -48,6 +48,7 @@
     "module-alias": "^2.2.2",
     "morgan": "^1.10.0",
     "mustache": "^4.2.0",
+    "mv": "^2.1.1",
     "node-fetch": "2",
     "prisma": "^5.7.1",
     "sanitize-filename": "^1.6.3",

server/src/models/MatchVideoInfo.ts

@@ -2,18 +2,21 @@ export class MatchVideoInfo {
     path: string;
     videoLabel: string | null;
     videoTitle: string;
+    isUploaded: boolean;
 
-    constructor(path: string, videoLabel: string | null, videoTitle: string) {
+    constructor(path: string, videoLabel: string | null, videoTitle: string, isUploaded: boolean = false) {
         this.path = path;
         this.videoLabel = videoLabel;
         this.videoTitle = videoTitle;
+        this.isUploaded = isUploaded;
     }
 
     toJson(): object {
         return {
             path: this.path,
             videoLabel: this.videoLabel,
             videoTitle: this.videoTitle,
+            isUploaded: this.isUploaded,
         };
     }
 }

server/src/repos/FileStorageRepo.ts

@@ -1,16 +1,28 @@
 import { type PathLike } from "fs";
 import fastGlob from "fast-glob";
 import logger from "jet-logger";
-import { isFileDoesNotExistError } from "@src/util/file.ts";
+import { isFileDoesNotExistError } from "@src/util/file";
 import { copyFile, readFile, writeFile } from "fs/promises";
 
 const DEFAULT_ENCODING = "utf-8";
 
-export async function getFilesMatchingPattern(dir: PathLike, pattern: string): Promise<string[]> {
-    logger.info(`getFilesMatchingPattern: dir: ${dir}, pattern: ${pattern}`);
+/**
+ * Get all files in a directory that match a given pattern (glob syntax from the fast-glob library is available)
+ *
+ * @param dir What to set as the current working directory for glob lookup
+ * @param pattern Glob pattern to match files against
+ * @param depth Corresponds to fast-glob's `deep` option, see fast-glob docs for more information
+ */
+export async function getFilesMatchingPattern(
+  dir: PathLike,
+  pattern: string,
+  depth: number = Infinity,
+): Promise<string[]> {
+    logger.info(`getFilesMatchingPattern: dir: ${dir}, pattern: ${pattern}, deep: ${depth}`);
     return await fastGlob.async(pattern, {
         cwd: dir.toString(), // cwd is relative to the directory the server is running out of
         onlyFiles: true,
+        deep: depth,
     });
 }
 

server/src/repos/YouTubeRepo.ts

@@ -96,6 +96,22 @@ export async function uploadYouTubeVideo(title: string,
         };
     }
 
+    const videoPathForUpload = path.join(videoSearchDirectory, videoPath);
+
+    if (!(await fs.exists(videoPathForUpload))) {
+        return {
+            error: `File ${videoPathForUpload} does not exist`,
+        };
+    }
+
+    let uploadBody: fs.ReadStream;
+    try {
+        uploadBody = fs.createReadStream(videoPathForUpload);
+    } catch (e: unknown) {
+        return {
+            error: `Error reading file ${videoPathForUpload} for upload: ${JSON.stringify(e)}`,
+        };
+    }
     const uploadParams = {
         part: ["snippet", "status"],
         requestBody: {
@@ -109,8 +125,7 @@ export async function uploadYouTubeVideo(title: string,
             },
         },
         media: {
-            // TODO(#78): Confirm that the file actually exists first to avoid crashing the Node process otherwise
-            body: fs.createReadStream(path.join(videoSearchDirectory, videoPath)),
+            body: uploadBody,
         },
     };