Merge pull request #16 from chilipot/integration/v0.1.0

Release Version 0.1.0

.gitignore

@@ -2,3 +2,28 @@
 *.pyc
 __pycache__/
 venv3/
+.idea/
+package-lock.json
+.vscode/
+
+# dependencies
+static/node_modules
+static/.pnp
+.pnp.js
+
+# testing
+static/coverage
+
+# production
+static/build
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*

Dockerfile

@@ -0,0 +1,33 @@
+FROM python:3.7-alpine AS base-image
+
+LABEL Author="Daniel Guddemi [email protected]"
+
+ENV PYTHONUNBUFFERED 1
+
+COPY ./requirements.txt /songflong/requirements.txt
+
+RUN apk update && \
+    apk add --no-cache --virtual .build-deps build-base linux-headers git g++ gcc libc-dev libxslt-dev && \
+    apk add --no-cache libxslt jpeg-dev zlib-dev && \
+    pip wheel --wheel-dir=/root/wheels -r /songflong/requirements.txt &&\
+    apk del .build-deps
+
+FROM python:3.7-alpine
+
+COPY --from=base-image /root/wheels /root/wheels
+
+COPY ./requirements.txt /songflong/requirements.txt
+COPY ./songflong.ini /songflong/songflong.ini
+
+WORKDIR /songflong
+
+RUN apk add --no-cache libxslt ffmpeg git && \
+    pip install \
+      --no-index \
+      --find-links=/root/wheels \
+      -r requirements.txt &&\
+    apk del git
+
+COPY ./run.py /songflong/run.py
+COPY ./worker.py /songflong/worker.py
+COPY ./app /songflong/app

Dockerfile.dashboard

@@ -0,0 +1,7 @@
+FROM python:3.7.0-alpine
+
+RUN pip install rq-dashboard
+
+EXPOSE 9181
+
+CMD ["rq-dashboard"]

Dockerfile.react

@@ -0,0 +1,24 @@
+FROM node:9.5 as builder
+
+RUN mkdir /usr/src/app
+WORKDIR /usr/src/app
+
+COPY static/ /usr/src/app
+
+ENV PATH /usr/src/app/node_modules/.bin:$PATH
+
+ADD ./static/package.json /usr/src/app/package.json
+RUN npm install
+RUN npm install -g [email protected]
+RUN npm run build
+
+FROM nginx:1.13.3-alpine
+
+RUN rm -rf /usr/share/nginx/html/*
+
+COPY --from=builder /usr/src/app/build /usr/share/nginx/html
+COPY config/nginx.conf /etc/nginx/conf.default
+
+EXPOSE 80
+
+CMD ["nginx", "-g", "daemon off;"]

README.md

@@ -1,5 +1,14 @@
-## Bug Fixes
-*Sorry David*
+# SongFlong
 
-There are bugs in the 9.5.2 version of "pytube". You need to install the requirement and then manually implement the fixes in the files. They're are located in the virtualenv/lib/python3.7/site-packages/pytube
-https://github.com/nficano/pytube/pull/453/files/fac934b149bc2d49ea54d566e0639d9b4c2862d2
+## Setup
+*SongFlong is built to run inside a Docker swarm and routing issues can occur if ran locally.*
+
+To run it locally, you need to manually replace the hostname of the Redis connection. This has to be done for the main Flask app and for the Worker. Changing it to `localhost` should suffice.
+
+To run it using Docker, all you need to do is install Docker and Docker-Compose.
+```
+docker-compose up --build --scale worker=3
+```
+
+The `--build` flag is not necessary; it does stuff that I don't feel like explaining to David (just look it up).
+The `--scale` can be modified to replicate as many workers as you want. Removing it completely will default to just 1.

app/__init__.py

@@ -1,14 +1,17 @@
 import os
 
 from flask import Flask
+from flask_cors import CORS
 from redis import StrictRedis
 from rq import Queue
 
 
 def create_app():
-    app = Flask(__name__)
-    app.config["FFMPEG_PATH"] = os.environ.get("FFMPEG_PATH", default="/usr/bin/ffmpeg")
-    app.q = Queue(connection=StrictRedis())
+    app = Flask(__name__, static_folder="../temp")
+    CORS(app)
+    app.config["FFMPEG_PATH"] = os.environ.get(
+        "FFMPEG_PATH", default="/usr/bin/ffmpeg")
+    app.q = Queue(connection=StrictRedis(host='redis', port='6379'))
     from app.routes import JOBS
     app.register_blueprint(JOBS)
     return app

app/job_service.py

@@ -1,3 +1,4 @@
+import logging
 from typing import List
 
 from flask import current_app
@@ -6,14 +7,26 @@
 from app.songflong.links import video_links
 from app.songflong.transcribe import setup_download_dir, generate_videos
 
+logger = logging.getLogger('songflong_builder')
+logger.setLevel(logging.INFO)
+
 
 def setup_jobs(song_name: str) -> List[int]:
     download_dir = setup_download_dir()
     video_link, similar_links = video_links(song_name)
     video_file = download_video_stream(video_link, download_dir)
+    if video_file is None:
+        logger.error(f"Cancelling request since no video stream could be found for {video_link}")
+        return None
+    logger.info(f"Video Submission using {video_link}:{video_file}")
     jobs = []
     for song, link in similar_links:
-        job = current_app.q.enqueue(generate_videos, str(video_file), song["title"], link, str(download_dir))
+        job = current_app.q.enqueue(generate_videos,
+                                    str(video_file),
+                                    link,
+                                    str(download_dir),
+                                    current_app.config.get("FFMPEG_PATH"),
+                                    **song)
         jobs.append(job.get_id())
     return jobs
 

app/routes.py

@@ -1,29 +1,38 @@
-from flask import jsonify, send_file, Blueprint
+from flask import jsonify, Blueprint, send_from_directory, current_app
 
 from app import job_service
 
 JOBS = Blueprint('jobs', __name__)
 
 
-@JOBS.route('/')
-def index():
-    return "Hello, World!", 200
+@JOBS.route('/ping')
+def ping():
+    return "pong", 200
 
 
 @JOBS.route('/submit/<string:song_name>')
 def submit(song_name):
     jobs = job_service.setup_jobs(song_name)
-    return jsonify(job_ids=jobs)
+    if jobs:
+        return jsonify(job_ids=jobs)
+    else:
+        return {"status", 'Unable to find video stream!'}, 424
 
 
 @JOBS.route('/results/<string:job_id>')
 def results(job_id):
     job = job_service.get_job(job_id)
 
     if job.is_failed:
-        return 'Job has failed!', 400
+        return {"status": 'Job has failed!'}, 400
 
     if job.is_finished:
-        return send_file(filename_or_fp=str(job.result.absolute()))
+        return jsonify(job.result)
 
-    return 'Job has not finished!', 202
+    return {"status": 'Job has not finished!'}, 202
+
+
+@JOBS.route('/video/<string:video_path>')
+def get_video(video_path):
+    # Serve directly from a better web server
+    return send_from_directory(directory=current_app.static_folder, filename=video_path)

app/songflong/__init__.py

@@ -1,6 +1,8 @@
 import logging
 
-logging.basicConfig(level=logging.INFO,
-                    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+log_format = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+handler = logging.StreamHandler()
+handler.setLevel(logging.INFO)
 
-logger = logging.getLogger(__name__)
+logger = logging.getLogger('songflong_builder')
+logger.addHandler(handler)

app/songflong/bpm_search.py

@@ -1,8 +1,11 @@
 import requests
 import urllib
 import re
+import logging
 from bs4 import BeautifulSoup
 
+logger = logging.getLogger('songflong_builder')
+logger.setLevel(logging.INFO)
 
 def get_track_bpm(in_query):
     query_tokens = re.split(r'[\(\[]', in_query)
@@ -13,6 +16,8 @@ def get_track_bpm(in_query):
     r = requests.get(url).content
     soup = BeautifulSoup(r, 'lxml')
     result_elem = soup.find(class_="search-info-container")
+    if result_elem is None:
+        logger.error("BPM Search failed to load the necessary search information")
 
     bpm_string = result_elem.find_all(
         class_="row search-attribute-value")[2].string

app/songflong/download.py

@@ -5,10 +5,7 @@
 import requests
 import logging
 
-logging.basicConfig(level=logging.INFO,
-                    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-
-logger = logging.getLogger(__name__)
+logger = logging.getLogger('songflong_builder')
 
 
 def download_stream(video_url, itag, download_dir):
@@ -77,8 +74,9 @@ def download_audio_stream(url: str, download_dir: Path) -> Path:
     try:
         logger.info(f"Downloading audio stream -> {url}")
         return download_stream(url, 140, download_dir)
-    except:
+    except Exception as err:
         logger.error(f"Unable to find an audio stream for {url}")
+        logger.exception(err)
 
 
 def download_video_stream(url: str, download_dir: Path) -> Path:
@@ -96,5 +94,6 @@ def download_video_stream(url: str, download_dir: Path) -> Path:
     try:
         logger.info(f"Downloading video stream -> {url}")
         return download_stream(url, 135, download_dir)
-    except Exception as e:
+    except Exception as err:
         logger.error(f"Unable to find an video stream for {url}")
+        logger.exception(err)

app/songflong/ffmpeg.py

@@ -4,7 +4,9 @@
 import subprocess as sp
 import proglog
 from flask import current_app
+import logging
 
+logger = logging.getLogger('songflong_builder')
 
 def subprocess_call(cmd):
     """
@@ -37,7 +39,7 @@ def subprocess_call(cmd):
     del proc
 
 
-def ffmpeg_merge_video_audio(video: Path, audio: Path, output: Path):
+def ffmpeg_merge_video_audio(video: Path, audio: Path, output: Path, ffmpeg_path: str):
     """
     Merges video and audio files into a single movie file.
 
@@ -48,7 +50,12 @@ def ffmpeg_merge_video_audio(video: Path, audio: Path, output: Path):
     :param output: The destination Path for the merged movie file
     :param output: Path
     """
-    cmd = [current_app.config.get("FFMPEG_PATH"), "-y", "-i", str(audio), "-i", str(video),
+
+    cmd = [ffmpeg_path, "-y", "-i", str(audio), "-i", str(video),
            "-vcodec", 'copy', "-acodec", 'copy', str(output)]
 
-    subprocess_call(cmd)
+    try:
+        subprocess_call(cmd)
+    except Exception as err:
+        logger.warning(f"Merging audio:{str(audio)} and video:{str(video)} to output{str(output)}")
+        logger.error(err)

app/songflong/transcribe.py

@@ -12,10 +12,12 @@ def setup_download_dir():
     return download_dir
 
 
-def transcribe_video(video_file: Path, audio_file: Path, download_dir: Path) -> Path:
+def transcribe_video(video_file: Path, audio_file: Path, download_dir: Path, ffmpeg_path: str) -> Path:
     """
     Transcribes the audio to the video and outputs a file.
 
+    :param ffmpeg_path: FFMPEG PATH
+    :type ffmpeg_path: str
     :param video_file: The base video stream
     :type video_file: Path
     :param audio_file: The downloaded audio stream
@@ -24,14 +26,14 @@ def transcribe_video(video_file: Path, audio_file: Path, download_dir: Path) ->
     :type download_dir: Path
     """
     output = download_dir / f"output-{uuid.uuid4()}.mp4"
-    ffmpeg_merge_video_audio(video_file, audio_file, output)
+    ffmpeg_merge_video_audio(video_file, audio_file, output, ffmpeg_path)
     return output
 
 
-def generate_videos(video_file, title, link, download_dir):
+def generate_videos(video_file, link, download_dir, ffmpeg_path, **kwargs):
     download_dir = Path(download_dir)
     video_file = Path(video_file)
     audio_file = download_audio_stream(link, download_dir)
-    finished_video = transcribe_video(video_file, audio_file, download_dir)
+    finished_video = transcribe_video(video_file, audio_file, download_dir, ffmpeg_path)
     print(f"******FINISHED TRANSCRIBING*****")
-    return finished_video
+    return {"filepath": str(finished_video.name), **kwargs}

config/nginx.conf

@@ -0,0 +1,17 @@
+events {
+    worker_connections 1024;
+}
+
+server {
+    listen       80;
+    server_name  localhost;
+    sendfiles on;
+
+
+    location / {
+        root   /usr/share/nginx/html/;
+        index  index.html;
+
+        try_files $uri;
+    }
+}