-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathcheck_scaler_params.py
130 lines (104 loc) · 4.26 KB
/
check_scaler_params.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
# def check_scaler_params(remaining_list_of_filepaths,
# csp_max_abs, csp_scaler, csp_scaler_type, csp_data_path):
# if csp_max_abs is not None and remaining_list_of_filepaths is not None:
# csp_scaler.csp_max_abs_ = csp_max_abs
# print(
# f" --- compute_csp_scaler(): For channel {os.path.split(csp_data_path)[-1]}, for scaler parameters in ,{f'{csp_scaler_type}_scaler_values.npy'} there are {len(remaining_list_of_filepaths)} files left to parse--- ")
# if len(remaining_list_of_filepaths) == 0:
# return None # TODO this will not work for multidim csp_scalers and will need an update
# elif remaining_list_of_filepaths is None:
# print(
# f" --- compute_csp_scaler(): For channel {os.path.split(csp_data_path)[-1]}, remaining filepaths are not yet saved --- ")
# test_train_paths = [csp_data_path.replace("Test", "Train"), csp_data_path]
# if "Train" in csp_data_path:
# test_train_paths = [csp_data_path, csp_data_path.replace("Train", "Test")]
# list_of_filepaths = []
# for path in test_train_paths: # generate files to load
# crt_filepaths = [os.path.join(path, file) for file in sorted(os.listdir(path))]
# list_of_filepaths.extend(crt_filepaths)
# remaining_list_of_filepaths = list_of_filepaths # todo check how to treat this - should initially load ALL files for both test and train for csp_scaler computation
# return remaining_list_of_filepaths, csp_scaler, csp_max_abs
import threading as th
import time
import datetime
from cffi.backend_ctypes import xrange
# init_time = datetime.datetime.now()
# time.sleep(5)
# print("non parallel for loop took: ", datetime.datetime.now() - init_time)
# for i in a:
# print(i)
# sleep(1)
# print("non parallel for loop took: ", datetime.datetime.now() - init_time)
# todo call moro
# todo prod music synth etc
# todo multithread
# pr_lock = th.Lock()
# n_t = 4
threads = []
# a = list(range(2*2*2*3*5*7*11))
# a = list(range(240)) # 120*2 = 240. divide by 40 = 6. divide by 20 = 12
a = list(range(16)) # 120*2 = 240. divide by 40 = 6. divide by 20 = 12
# a = list(range(20)) # 120*2 = 240. divide by 40 = 6. divide by 20 = 12
def parallel_print(list_print):
for i in list_print:
print(th.current_thread(), "parprint i=",i)
time.sleep(1) # simulate operation that takes time
print("return sleep")
# return max(list_print)
lvec = len(a)
# 120 sec divided by... 8 threads = 15 sec
# 120 / 20 threads = 6 sec
# 120 / 24 = 5 sec
n_t = 4 # 40 threads = 3s
len_subvec = lvec//n_t
for i in range(n_t):
crt_rng = ((i)*len_subvec,(i)*n_t+len_subvec)
crt_vec = a[slice(*crt_rng)]
# print(crt_rng)
# print("////i", crt_vec)
t = th.Thread(target=parallel_print, args=(crt_vec,))
# print("get thread b4 start",t.getName())
threads.append(t)
print("-------start code-------")
init_time = datetime.datetime.now()
# init_time = time.thread_time()
for t in threads:
t.start()
# t.join() # why does join have to be in a separate for?
for t in threads: # TODO why does join make such a big difference? why does it make the running time correct but no join makes it appear as if it's 0??
t.join()
# if you don't join, the main thread will continue and print the time before the threads finish
# join indicates that the main thread should wait for the threads to finish before continuing
# or also the end of a parallel section
# if join is in the same for loop as start, the main thread will wait for each thread to finish before starting the next one
time_duration = datetime.datetime.now() - init_time
print(f"parallel for loop took: , {time_duration}")
import threading
def something():
ao = [5,1,3,9,7]
mx = ao[0]
for i in ao:
if i > mx:
mx = i
yield mx
print ("Hello")
def my_thing():
bo = [4, 2,8,6,10]
mx = bo[0]
for i in bo:
if i > mx:
mx = i
yield mx
print ("world")
def get_max_from_inlist(inlist):
mx = inlist[0]
for i in inlist:
if i > mx:
mx = i
yield mx
aq = threading.Thread(target=something)
aq.start()
aqb = threading.Thread(target=my_thing)
aqb.start()
aq.join()
aqb.join()