diff --git a/python/spacewalk/satellite_tools/reposync.py b/python/spacewalk/satellite_tools/reposync.py index 01de32847503..4a2afd7134ce 100644 --- a/python/spacewalk/satellite_tools/reposync.py +++ b/python/spacewalk/satellite_tools/reposync.py @@ -1366,10 +1366,6 @@ def import_package_batch(self, to_process, to_disassociate, is_non_local_repo, b raise except Exception as e: e_message = f'Exception: {e}' - if importer: - e_message += f'\nPackage: {repr(importer)}' - if src_importer: - e_message += f'\nSource package: {repr(src_importer)}' log2(0, 1, e_message, stream=sys.stderr) if self.fail: raise diff --git a/python/spacewalk/server/importlib/backend.py b/python/spacewalk/server/importlib/backend.py index cd0fdb099588..1606c1d87d5d 100644 --- a/python/spacewalk/server/importlib/backend.py +++ b/python/spacewalk/server/importlib/backend.py @@ -26,6 +26,7 @@ from spacewalk.common.rhnConfig import CFG from spacewalk.common.rhnException import rhnFault from spacewalk.common.rhnLog import log_debug +from spacewalk.satellite_tools import syncLib from spacewalk.server import rhnSQL, rhnChannel, taskomatic from .importLib import Diff, Package, IncompletePackage, Erratum, \ AlreadyUploadedError, InvalidPackageError, TransactionError, \ @@ -33,6 +34,7 @@ from .backendLib import TableCollection, sanitizeValue, TableDelete, \ TableUpdate, TableLookup, addHash, TableInsert + sequences = { 'rhnPackageCapability': 'rhn_pkg_capability_id_seq', 'rhnPackage': 'rhn_package_id_seq', @@ -984,10 +986,16 @@ def processPackages(self, packages, uploadForce=0, ignoreUploaded=0, package['header_start'] = -1 package['header_end'] = -1 - self.__processObjectCollection__([package, ], 'rhnPackage', tableList, + try: + self.__processObjectCollection__([package, ], 'rhnPackage', tableList, uploadForce=uploadForce, forceVerify=forceVerify, ignoreUploaded=ignoreUploaded, severityLimit=1, transactional=transactional) + except Exception as e: + syncLib.log(0, "Error during processing package %s-%s-%s:%s.%s.\n%s" % + (package['name'], package['version'], package['release'], package['epoch'], package['arch'], + str(e))) + raise def processErrata(self, errata): # Insert/update the packages diff --git a/python/spacewalk/server/importlib/packageImport.py b/python/spacewalk/server/importlib/packageImport.py index 41aca288f0e1..ea21d327d3df 100644 --- a/python/spacewalk/server/importlib/packageImport.py +++ b/python/spacewalk/server/importlib/packageImport.py @@ -285,16 +285,20 @@ def _processPackage(self, package): self.checksums[fchecksumTuple] = None # Uniquify changelog entries - unique_package_changelog_hash = {} + unique_package_changelog_hash = set() unique_package_changelog = [] for changelog in package['changelog']: - key = (self._fix_encoding(changelog['name'][:128]), self._fix_encoding(changelog['time']), self._fix_encoding(changelog['text'])[:3000]) + changelog_name = self._fix_encoding(changelog['name'][:128]) + changelog_time = self._fix_encoding(changelog['time']) + changelog_text = self._fix_encoding(changelog['text'])[:3000] + key = (changelog_name, changelog_time, changelog_text) if key not in unique_package_changelog_hash: self.changelog_data[key] = None - changelog['name'] = changelog['name'][:128] - changelog['text'] = changelog['text'][:3000] + changelog['name'] = changelog_name + changelog['text'] = changelog_text + changelog['time'] = changelog_time unique_package_changelog.append(changelog) - unique_package_changelog_hash[key] = 1 + unique_package_changelog_hash.add(key) package['changelog'] = unique_package_changelog # fix encoding issues in package summary and description diff --git a/python/spacewalk/spacewalk-backend.changes.witek.add-unique-index b/python/spacewalk/spacewalk-backend.changes.witek.add-unique-index new file mode 100644 index 000000000000..c635f8472974 --- /dev/null +++ b/python/spacewalk/spacewalk-backend.changes.witek.add-unique-index @@ -0,0 +1 @@ +- Add unique index for rhnpackagechangelogdata table diff --git a/schema/spacewalk/common/tables/rhnPackageChangeLogData.sql b/schema/spacewalk/common/tables/rhnPackageChangeLogData.sql index 215255e0e3ff..67950a49fb25 100644 --- a/schema/spacewalk/common/tables/rhnPackageChangeLogData.sql +++ b/schema/spacewalk/common/tables/rhnPackageChangeLogData.sql @@ -13,7 +13,6 @@ -- in this software or its documentation. -- - CREATE TABLE rhnPackageChangeLogData ( id NUMERIC NOT NULL @@ -28,8 +27,9 @@ CREATE TABLE rhnPackageChangeLogData ; -CREATE INDEX rhn_pkg_cld_nt_idx - ON rhnPackageChangeLogData (name, time) +CREATE UNIQUE INDEX rhn_pkg_cld_ntt_idx + ON rhnPackageChangeLogData + USING btree(name, digest("text", 'sha512'::text), time) ; diff --git a/schema/spacewalk/postgres/start.sql b/schema/spacewalk/postgres/start.sql index ea80ba2a3683..c92263d809ef 100644 --- a/schema/spacewalk/postgres/start.sql +++ b/schema/spacewalk/postgres/start.sql @@ -21,3 +21,5 @@ create temporary table store_searchpath as select setting from pg_settings where update pg_settings set setting = (select setting from store_searchpath) where name = 'search_path'; drop table store_searchpath; + +CREATE EXTENSION pgcrypto; diff --git a/schema/spacewalk/upgrade/susemanager-schema-4.4.6-to-susemanager-schema-4.4.7/203-add-changelogdata-index.sql b/schema/spacewalk/upgrade/susemanager-schema-4.4.6-to-susemanager-schema-4.4.7/203-add-changelogdata-index.sql new file mode 100644 index 000000000000..5052829aaa31 --- /dev/null +++ b/schema/spacewalk/upgrade/susemanager-schema-4.4.6-to-susemanager-schema-4.4.7/203-add-changelogdata-index.sql @@ -0,0 +1,38 @@ +create or replace function remove_duplicate_changelogdata() +returns void as +$$ +declare original record; +declare duplicate record; +begin + for original in select min(id) as id + from rhnpackagechangelogdata + group by name, text, time + having count(*) > 1 loop + for duplicate in select data2.id + from rhnpackagechangelogdata data1, rhnpackagechangelogdata data2 + where data1.name = data2.name + and data1.text = data2.text + and data1.time = data2.time + and data1.id != data2.id + and data1.id = original.id loop + update rhnpackagechangelogrec + set changelog_data_id = original.id + where changelog_data_id = duplicate.id; + delete from rhnpackagechangelogdata + where id = duplicate.id; + end loop; + end loop; +end; +$$ language plpgsql; + +select remove_duplicate_changelogdata(); +drop function remove_duplicate_changelogdata(); + +drop index if exists rhn_pkg_cld_nt_idx; +drop index if exists rhn_pkg_cld_ntt_idx; + +create extension if not exists pgcrypto; +create unique index concurrently rhn_pkg_cld_ntt_idx + on "rhnpackagechangelogdata" + using btree(name, digest("text", 'sha512'::text), "time"); +