-
Notifications
You must be signed in to change notification settings - Fork 0
/
arc_multi_sources.py
204 lines (171 loc) · 7.13 KB
/
arc_multi_sources.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
import calendar
import os
from ftplib import FTP
from datetime import datetime as dt
class ARC_MULTI_SOURCES:
def __init__(self, input_dir, input_dir_org,moi_user,moi_pass,use_myint_sources, verbose):
self.verbose = verbose
self.input_dir = input_dir
if input_dir_org is None:
self.input_dir_org = 'YYYY/jjj'
self.input_dir_org = input_dir_org
self.name_product = 'OCEANCOLOUR_GLO_BGC_L3_MY_009_107'
self.name_dataset = 'c3s_obs-oc_glo_bgc-reflectance_my_l3-multi-4km_P1D'
self.use_myint_sources = use_myint_sources
self.server = 'my.cmems-du.eu'
self.variable_check = ['latitude','longitude','time','RRS412','RRS443','RRS490','RRS510','RRS560','RRS665']
if moi_user is not None and moi_pass is not None:
self.moi_user = moi_user
self.moi_pass = moi_pass
def get_output_file(self,date_here,datasetname,output_path,output_path_organization):
print('fdf')
def get_file_date(self, date_here, make_download):
dirdate = self.get_input_folder_date(date_here, True)
if dirdate is None:
return None
date_here_str = date_here.strftime('%Y%m%d')
name = f'{date_here_str}_{self.name_dataset}.nc'
if self.use_myint_sources:
name = name.replace('my','myint')
fdate = os.path.join(dirdate, name)
if not os.path.exists(fdate) and make_download:
self.download_file(date_here, dirdate, name)
if os.path.exists(fdate):
from netCDF4 import Dataset
try:
dataset = Dataset(fdate)
for var in self.variable_check:
if not var in dataset.variables:
print(f'[ERROR] Variable {var} is not available in file: {fdate}')
dataset.close()
return 'INVALIDFILE'
dataset.close()
return fdate
except:
print(f'[ERROR] File {fdate} is not a valid NetCDF file')
return 'INVALIDFILE'
else:
return 'NOFILE'
def check_file_ftp(self,date_here):
date_here_str = date_here.strftime('%Y%m%d')
name = f'{date_here_str}_{self.name_dataset}.nc'
fwork = FTPWork(self.server, self.moi_user, self.moi_pass)
fwork.path_dataset = f'/Core/{self.name_product}/{self.name_dataset}'
return fwork.check_daily_file(date_here,name)
def check_month_files_ftp(self,year,month):
fwork = FTPWork(self.server, self.moi_user, self.moi_pass)
fwork.path_dataset = f'/Core/{self.name_product}/{self.name_dataset}'
missing_dates = fwork.check_daily_files_month(year,month,self.name_dataset)
return missing_dates
def download_file(self, date_here, dirdate, name):
if self.verbose:
print(f'[INFO] Downloading file {name} in folder: {dirdate}')
fwork = FTPWork(self.server,self.moi_user,self.moi_pass)
fwork.path_dataset = f'/Core/{self.name_product}/{self.name_dataset}'
fwork.donwload_daily_file(date_here,dirdate,name)
fwork.close()
def get_input_folder_date(self, date_here, create):
return self.get_folder_date(date_here,create,self.input_dir,self.input_dir_org)
def get_folder_date(self,date_here,create, input_dir,input_dir_org):
replaces = {
'YYYY': date_here.strftime('%Y'),
'jjj': date_here.strftime('%j'),
'mm': date_here.strftime('%m'),
'dd': date_here.strftime('%d')
}
dirs = input_dir_org.split('/')
dirdate = input_dir
for d in dirs:
dhere = d
for r in replaces:
if d.find(r) >= 0:
dhere = dhere.replace(r, replaces[r])
dirdate = os.path.join(dirdate, dhere)
if not os.path.exists(dirdate) and create:
try:
os.mkdir(dirdate)
except:
pass
if not os.path.exists(dirdate):
if create:
print(f'[ERROR] Folder {dirdate} could not be created')
else:
print(f'[WARNING] Folder {dirdate} does not exist')
return None
return dirdate
class FTPWork():
def __init__(self, server,uname,passwd):
self.server = server
self.ftpdu = FTP(server, uname, passwd)
self.path_dataset = None
def go_subdir(self, rpath):
# print('Changing directory to: ', rpath)
self.ftpdu.cwd(rpath)
def go_year_subdir(self, rpath, year):
if rpath is None:
rpath = self.path_dataset
if rpath is None:
return False
dateref = dt(year, 1, 1)
yearstr = dateref.strftime('%Y')
self.ftpdu.cwd(rpath)
if yearstr in self.ftpdu.nlst():
self.ftpdu.cwd(yearstr)
return True
else:
return False
def go_month_subdir(self, rpath, year, month):
if rpath is None:
rpath = self.path_dataset
if rpath is None:
return False
dateref = dt(year, month, 1)
yearstr = dateref.strftime('%Y') # dt.strptime(str(year), '%Y')
monthstr = dateref.strftime('%m') # dt.strptime(month, '%m')
self.ftpdu.cwd(rpath)
if yearstr in self.ftpdu.nlst():
self.ftpdu.cwd(yearstr)
if monthstr in self.ftpdu.nlst():
self.ftpdu.cwd(monthstr)
return True
return False
def check_daily_file(self,date_here,name_file):
b = self.go_month_subdir(None,date_here.year,date_here.month)
if not b:
return False
if name_file in self.ftpdu.nlst():
return True
else:
return False
def check_daily_files_month(self,year,month,name_ref):
res = calendar.monthrange(year,month)
last_day_month = res[1]
b = self.go_month_subdir(None, year, month)
if not b:
missing_dates = []
for day in range(1, last_day_month + 1):
date_here = dt(year, month, day)
missing_dates.append(date_here)
return missing_dates
list = self.ftpdu.nlst()
missing_dates = []
for day in range(1,last_day_month+1):
date_here = dt(year,month,day)
date_here_str = date_here.strftime('%Y%m%d')
name_file = f'{date_here_str}_{name_ref}.nc'
if name_file not in list:
missing_dates.append(date_here)
return missing_dates
def donwload_daily_file(self,date_here,dir_out,name_file):
b = self.check_daily_file(date_here,name_file)
if not b:
print(f'[ERROR] {self.path_dataset}/{name_file} is not available for download from {self.server}')
return False
# Write file in binary mode
file_out = os.path.join(dir_out,name_file)
with open(file_out, "wb") as file:
# Command for Downloading the file "RETR filename"
self.ftpdu.retrbinary(f"RETR {name_file}", file.write)
return True
def close(self):
self.ftpdu.close()