From 943114ff037b8fb5f0daa7eb7b15bfb7fd2b9f6b Mon Sep 17 00:00:00 2001 From: stas Date: Sat, 9 Jul 2022 16:01:42 +0300 Subject: [PATCH] #2 --- terrarium_assembler/ta.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/terrarium_assembler/ta.py b/terrarium_assembler/ta.py index 9ef3041..bc489d5 100644 --- a/terrarium_assembler/ta.py +++ b/terrarium_assembler/ta.py @@ -141,6 +141,8 @@ def is_need_patch(self, f): return False def is_need_exclude(self, f): + if 'libtorch_cuda.so' in f: + wtf=34342 for re_ in self.need_exclude_re: if re_.match(f): return True @@ -291,7 +293,7 @@ class TerrariumAssembler: ''' def __init__(self): - self.curdir = os.getcwd() + # self.curdir = os.getcwd() self.root_dir = None # Потом сделать параметром функций. @@ -368,13 +370,15 @@ def __init__(self): self.args.stage_download_wheels = True specfile_ = expandpath(args.specfile) - os.environ['TERRA_SPECDIR'] = os.path.split(specfile_)[0] + self.start_dir = self.curdir = os.path.split(specfile_)[0] + os.environ['TERRA_SPECDIR'] = self.start_dir + os.chdir(self.curdir) - self.pipenv_dir = '' - from pipenv.project import Project - p = Project() - self.pipenv_dir = p.virtualenv_location + # self.pipenv_dir = '' + # from pipenv.project import Project + # p = Project() + # self.pipenv_dir = p.virtualenv_location self.tvars = edict() @@ -384,7 +388,7 @@ def __init__(self): self.tvars.py_ext = ".py" self.tvars.release = not self.args.debug self.tvars.fc_version = '' - self.tvars.pipenv_dir = self.pipenv_dir + # self.tvars.pipenv_dir = self.pipenv_dir self.tvars.python_major_version = sys.version_info.major self.tvars.python_minor_version = sys.version_info.minor @@ -399,9 +403,7 @@ def __init__(self): self.tvars = edict(vars_) spec = self.spec - self.start_dir = os.getcwd() - - + # self.start_dir = os.getcwd() need_patch = just_copy = need_exclude = None if 'bin_regexps' in spec: @@ -513,6 +515,7 @@ def lines2sh(self, name, lines, stage=None): lf.write(''' # Stage "%s" # Automatically called when terrarium_assembler --stage-%s "%s" +export TA_PIPENV_DIR=`python -m pipenv --venv` ''' % (desc, stage_, self.args.specfile)) for k, v in self.tvars.items(): if isinstance(v, str) or isinstance(v, int): @@ -781,7 +784,7 @@ def dependencies(self, package_list, local=True): pl_ = self.packages2list(package_list) package_list_md5 = hashlib.md5((self.rpm_update_time() + '\n' + '\n'.join(pl_)).encode('utf-8')).hexdigest() - cache_filename = 'cache_' + package_list_md5 + '.list' + cache_filename = 'tmp/cache_' + package_list_md5 + '.list' if os.path.exists(cache_filename): with open(cache_filename, 'r', encoding='utf-8') as lf: ls_ = lf.read() @@ -886,7 +889,7 @@ def generate_file_list_from_packages(self, packages): ''' package_list_md5 = hashlib.md5((self.rpm_update_time() + '\n' + '\n'.join(packages)).encode('utf-8')).hexdigest() - cache_filename = 'cachefilelist_' + package_list_md5 + '.list' + cache_filename = 'tmp/cachefilelist_' + package_list_md5 + '.list' if os.path.exists(cache_filename): with open(cache_filename, 'r', encoding='utf-8') as lf: ls_ = lf.read() @@ -1124,7 +1127,7 @@ def checkout_sources(self): fi """ % vars()) - lines.append(f""" + lines.append(f""" # We need to update all shell files after checkout. terrarium_assembler "{self.args.specfile}" """) @@ -1588,14 +1591,17 @@ def analyse(self): used_files = set() for trace_file in glob.glob(trace_file_glob): re_file = re.compile(r'''.*\(.*"(?P[^"]+)".*''') - for line in open(trace_file, 'r', encoding='utf-8').readlines(): + for linenum, line in enumerate(open(trace_file, 'r', encoding='utf-8').readlines()): m_ = re_file.match(line) if m_: fname = m_.group('filename') + if 'pbin/ld.so' in fname or linenum>140: + wtf=1 # Heuristic to process strace files from Vagrant virtualboxes fname = fname.replace('/vagrant', self.curdir) # Heuristic to process strace files from remote VM, mounted by sshmnt fname = re.sub(fr'''/mnt/.*{lastdirs}''', abs_path_to_out_dir, fname) + fname = re.sub(fr'''/opt/dm/''', abs_path_to_out_dir, fname) if os.path.isabs(fname): fname = os.path.abspath(fname) if fname.startswith(abs_path_to_out_dir): @@ -1616,7 +1622,7 @@ def analyse(self): size_ = os.stat(fname_).st_size existing_files[fname_] = size_ - top10 = sorted(existing_files.items(), key=lambda x: -x[1])[:1000] + top10 = sorted(existing_files.items(), key=lambda x: -x[1])[:4000] print("Analyse first:") for f, s in top10: rel_f = f.replace(abs_path_to_out_dir, '') @@ -1659,7 +1665,7 @@ def pack_me(self): banned_ext = ['.old', '.iso', '.lock', disabled_suffix, '.dblite', '.tmp', '.log'] banned_start = ['tmp'] - banned_mid = ['/out', '/wtf', '/ourwheel/', '/.vagrant', '/.git', '/.vscode', '/key/', '/tmp/', '/src.', '/bin.', '/cache_', 'cachefilelist_', '/.image', '/!'] + banned_mid = ['/out', '/wtf', '/ourwheel/', '/.vagrant', '/.git', '/.vscode', '/key/', '/tmp/', '/src.', '/bin.', '/cache_', 'cachefilelist_', '/tmp', '/.image', '/!'] # there are regularly some files unaccessable for reading. self.cmd('sudo chmod a+r /usr/lib/cups -R')