Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add windows support #180

Open
wants to merge 5 commits into
base: dev
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions build.bat
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
@echo off
call setup.bat || exit /b 1
call .venv/Scripts/activate || exit /b 1
pip install pyinstaller || exit /b 1
python win-package.py || exit /b 1
2 changes: 1 addition & 1 deletion build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ package_scripts() {
continue
fi
tmp_name=$(mktemp).spec
cat "precise.template.spec" | replace "%%SCRIPT%%" "$script" | replace "%%TRAIN_LIBS%%" "$train_libs" > "$tmp_name"
cat "precise.template.spec" | replace "%%SCRIPT%%" "$script" | replace "%%TRAIN_LIBS%%" "$train_libs" | replace "%%STRIP%%" "True" > "$tmp_name"
pyinstaller -y "$tmp_name"
if [ "$exe" != "$combined_folder" ]; then
cp -R dist/$exe/* "dist/$combined_folder"
Expand Down
8 changes: 6 additions & 2 deletions precise.template.spec
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,11 @@ from glob import iglob
from os.path import basename, dirname, abspath
import os
import fnmatch
from PyInstaller.utils.hooks import collect_data_files

script_name = '%%SCRIPT%%'
train_libs = %%TRAIN_LIBS%%
strip = True
strip = %%STRIP%%
site_packages = '.venv/lib/python3.6/site-packages/'
hidden_imports = ['prettyparse', 'speechpy']
binaries = []
Expand All @@ -29,11 +30,13 @@ if train_libs:
]
hidden_imports += ['h5py']

datas = collect_data_files('astor', subdir=None, include_py_files=False)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What does this return? A list of data files?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The astor lib uses an internal file called VERSION.l, which pyinstaller does not include by default.

Pyinstaller does allow you to specify data files which should be bundled along.

See
https://pyinstaller.readthedocs.io/en/stable/spec-files.html#adding-data-files

The function returns a list of tuples where the first is a file name and the second is the folder to insert it to.

Here it will collect all yhe non python files in the root directory of astor.

If that's preferable I think I can specify the exact file instead as shown here:
https://pyinstaller.readthedocs.io/en/stable/spec-files.html#using-data-files-from-a-module


a = Analysis(
[abspath('precise/scripts/{}.py'.format(script_name))],
pathex=['.'],
binaries=binaries,
datas=[],
datas=datas,
hiddenimports=hidden_imports,
hookspath=[],
runtime_hooks=[],
Expand All @@ -43,6 +46,7 @@ a = Analysis(
cipher=block_cipher
)


for i in range(len(a.binaries)):
dest, origin, kind = a.binaries[i]
if '_pywrap_tensorflow_internal' in dest:
Expand Down
9 changes: 9 additions & 0 deletions setup.bat
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
@echo off
if not exist ".venv" (
py -3.6 -m venv .venv || exit /b 1
)
call .venv/Scripts/activate || exit /b 1
pip install -e runner/ || exit /b 1
pip install -e . || exit /b
rem Optional, for comparison
pip install pocketsphinx || exit /b 1
4 changes: 2 additions & 2 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@
},
include_package_data=True,
install_requires=[
'numpy==1.16',
'tensorflow>=1.13,<1.14', # Must be on piwheels
'numpy==1.16.2',
'tensorflow==1.13.1', # Must be on piwheels
'sonopy',
'pyaudio',
'keras<=2.1.5',
Expand Down
159 changes: 159 additions & 0 deletions win-package.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
# This file should not be used directly but using build.bat

import os
import os.path
from os.path import join, normpath, basename
import platform
import re
import tempfile
import shutil
import glob
import tarfile
from distutils import dir_util

from precise import __version__

import hashlib

def main():
all_scripts=re.findall('(?<=precise.scripts.)[a-z_]+', open('setup.py').read())
package_scripts("precise-all", "precise", all_scripts, True)
package_scripts("precise-engine", "precise-engine", ["engine"], False)

tar_1 = join("dist",tar_name("precise-all"))
tar_2 = join("dist",tar_name("precise-engine"))
print("Wrote to {} and {}".format(tar_1, tar_2))

def package_scripts(tar_prefix, combined_folder, scripts, train_libs):
"""Use pyinstaller to create EXEs for the scripts
and bundle them to a tar.gz file in dist/.

Args:
tar_prefix (str): The prefix of the output tar.gz file.
combined_folder (str): The name of the directory in dist on which put all the files produced by pyinstaller.

"""
completed_file=join("dist", "completed_{}.txt".format(combined_folder))
if not os.path.isfile(completed_file):
delete_dir_if_exists(join("dist", combined_folder))

for script in scripts:
exe = "precise-{}".format(script.replace('_', '-'))
if filecontains(completed_file, exe):
continue

spec_path = createSpecFile(script, train_libs)
if os.system("pyinstaller -y {} --workpath=build/{}".format(spec_path, exe)) != 0:
raise Exception("pyinstaller error")

if exe != combined_folder:
dir_util.copy_tree(join("dist",exe), join("dist",combined_folder), update=1)
shutil.rmtree(join("dist",exe))
shutil.rmtree(join("build",exe))
with open(completed_file, 'a') as f:
f.write(exe + '\n')

out_name = tar_name(tar_prefix)
make_tarfile(join('dist', combined_folder), join('dist', out_name))
with open(join('dist', "{}.md5".format(out_name)), 'w') as md5file:
md5file.write(filemd5(join('dist', out_name)))

def filemd5(fname):
"""Calculate md5 hash of a file.

Args:
fname (str): (The path of) the file to be hashed.

Returns:
The md5 hash of the file in a hexadecimal string.

"""
hash_md5 = hashlib.md5()
with open(fname, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_md5.update(chunk)
return hash_md5.hexdigest()

def make_tarfile(source_dir, output_filename):
"""Compress a directory into a gzipped tar file.

Args:
source_dir (str): (The path of) the directory to be compressed.
output_filename (str): The tar.gz file name/path the directory should be compressed to.

"""
with tarfile.open(output_filename, "w:gz") as tar:
tar.add(source_dir, arcname=basename(source_dir))

def tar_name(tar_prefix):
"""Generate a name for a tar.gz file which includes the version, the os name (windows)
and the architacture.

Args:
tar_prefix (str): The tar.gz filename will start with this value.

Returns:
A string in the following format: "{tar_prefix}_{version}_win_{archname}.tar.gz".

"""
arch = platform.machine()
return "{tar_prefix}_{version}_win_{archname}.tar.gz".format(
tar_prefix=tar_prefix,
version=__version__,
archname = arch)

def filecontains(path, string):
"""Check if a file contains a given phrase.

Args:
path (str): (The path of) the file to search in.
string (str): The phrase to look for.

Returns:
True if the given file contains the given phrase, False otherwise.

"""
try:
f = open(path)
content = f.read()
f.close()
return string in content
except FileNotFoundError:
return False

def delete_dir_if_exists(folder):
"""Delete a folder, ignore if it does not exist.

Args:
folder (str): The folder to be deleted.

"""

try:
shutil.rmtree(folder)
except FileNotFoundError:
pass

def createSpecFile(script, train_libs):
"""Create a pyinstaller spec file based on precise.template.spec.

Args:
script (str): the python script for which this spec file is intended.
train_libs (bool): whether the spec should include the training libs.

Returns:
The path of the created spec file.

"""
with tempfile.NamedTemporaryFile(mode = 'w',suffix='.spec', delete=False) as temp_file:
spec_path = temp_file.name
with open("precise.template.spec") as template_spec:
spec = template_spec.read() \
.replace("%%SCRIPT%%",script) \
.replace("%%TRAIN_LIBS%%", str(train_libs)) \
.replace("%%STRIP%%", "False")
temp_file.write(spec + '\n')
return spec_path

if __name__ == '__main__':
main()