Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Galv/ia download unsupervised #64

Open
wants to merge 2 commits into
base: galv/setuptools
Choose a base branch
from
Open

Galv/ia download unsupervised #64

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a30eef6
Update IA download script to download all possible supervised data.
galv May 27, 2022
959f9ff
fixup: Format Python code with Black
May 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions galvasr2/align/export.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,13 +93,13 @@ def get_sample_size(population_size):
margin_of_error = 0.01
fraction_picking = 0.50
z_score = 2.58 # Corresponds to confidence level 99%
numerator = (z_score ** 2 * fraction_picking * (1 - fraction_picking)) / (
margin_of_error ** 2
numerator = (z_score**2 * fraction_picking * (1 - fraction_picking)) / (
margin_of_error**2
)
sample_size = 0
for train_size in range(population_size, 0, -1):
denominator = 1 + (z_score ** 2 * fraction_picking * (1 - fraction_picking)) / (
margin_of_error ** 2 * train_size
denominator = 1 + (z_score**2 * fraction_picking * (1 - fraction_picking)) / (
margin_of_error**2 * train_size
)
sample_size = int(numerator / denominator)
if 2 * sample_size + train_size <= population_size:
Expand Down
2 changes: 1 addition & 1 deletion galvasr2/align/spark/align_cuda_decoder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ def main(argv):
mem_bytes = os.sysconf("SC_PAGE_SIZE") * os.sysconf(
"SC_PHYS_PAGES"
) # e.g. 4015976448
mem_gib = int((mem_bytes / (1024.0 ** 3)) * 0.9)
mem_gib = int((mem_bytes / (1024.0**3)) * 0.9)
(jar_path,) = galvasr2.__path__
jars = ",".join(glob.glob(os.path.join(jar_path, "*.jar")))
print("GALVEZ:jars=", jars)
Expand Down
2 changes: 1 addition & 1 deletion galvasr2/align/spark/align_lib_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ def setUpClass(cls):
mem_bytes = os.sysconf("SC_PAGE_SIZE") * os.sysconf(
"SC_PHYS_PAGES"
) # e.g. 4015976448
mem_gib = int((mem_bytes / (1024.0 ** 3)) * 0.9)
mem_gib = int((mem_bytes / (1024.0**3)) * 0.9)
tar_jar = os.path.join(
find_runfiles(), "__main__/galvasr2/spark/tar_spark_datasource.jar"
)
Expand Down
8 changes: 5 additions & 3 deletions scripts/archive.org/download_items.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,8 @@
# TODO: Consider adding AND NOT format:ASR to disclude speech recognition-based labels.
# ALL_CAPTIONED_DATA=f"{LICENSE_WHITELIST} AND (mediatype:audio OR mediatype:movies) AND (closed_captioning:yes OR format:SubRip OR format:\"Web Video Text Tracks\") AND (NOT access-restricted-item:TRUE)",
# NON_CAPTIONED_DATA_WITH_TEXT=f"{LICENSE_WHITELIST} AND (format:DjVuTXT AND format:MP3 AND NOT format:SubRip) AND NOT (subject:'librivox')",
CC_BY_SA_EXPANDED_LICENSES_FILTERED_ACCESS=f'{LICENSE_WHITELIST} AND (mediatype:audio OR mediatype:movies) AND (closed_captioning:yes OR format:SubRip OR format:"Web Video Text Tracks") AND (NOT access-restricted-item:TRUE)',
# CC_BY_SA_EXPANDED_LICENSES_FILTERED_ACCESS=f'{LICENSE_WHITELIST} AND (mediatype:audio OR mediatype:movies) AND (closed_captioning:yes OR format:SubRip OR format:"Web Video Text Tracks") AND (NOT access-restricted-item:TRUE)',
CC_BY_SA_ALL_AUDIO_LABELED_OR_UNLABELED=f"{LICENSE_WHITELIST} AND (mediatype:audio OR mediatype:movies) AND (NOT access-restricted-item:TRUE)",
)


Expand Down Expand Up @@ -89,10 +90,11 @@ def get_metadata(result: dict):
print(query)
print(f"Dumping metadata for {key}")
save_file = key + ".jsonl.gz"
# download_metadata(query, save_file)
download_metadata(query, save_file)
download_data(
save_file,
f"gs://the-peoples-speech-west-europe/archive_org/Mar_7_2021/{key}",
# Ryan: Change this output path
f"download_output/{key}",
)

# download_data("CAPTIONED_DATA.jsonl.gz", "gs://the-peoples-speech-west-europe/archive_org/Aug_18_2020/CAPTIONED_DATA")
Expand Down