Extend unique index for rhnpackagechangelogdata table

python/spacewalk/satellite_tools/reposync.py

@@ -1366,10 +1366,6 @@ def import_package_batch(self, to_process, to_disassociate, is_non_local_repo, b
                     raise
                 except Exception as e:
                     e_message = f'Exception: {e}'
-                    if importer:
-                        e_message += f'\nPackage: {repr(importer)}'
-                    if src_importer:
-                        e_message += f'\nSource package: {repr(src_importer)}'
                     log2(0, 1, e_message, stream=sys.stderr)
                     if self.fail:
                         raise

python/spacewalk/server/importlib/backend.py

@@ -26,13 +26,15 @@
 from spacewalk.common.rhnConfig import CFG
 from spacewalk.common.rhnException import rhnFault
 from spacewalk.common.rhnLog import log_debug
+from spacewalk.satellite_tools import syncLib
 from spacewalk.server import rhnSQL, rhnChannel, taskomatic
 from .importLib import Diff, Package, IncompletePackage, Erratum, \
     AlreadyUploadedError, InvalidPackageError, TransactionError, \
     SourcePackage
 from .backendLib import TableCollection, sanitizeValue, TableDelete, \
     TableUpdate, TableLookup, addHash, TableInsert
 
+
 sequences = {
     'rhnPackageCapability': 'rhn_pkg_capability_id_seq',
     'rhnPackage': 'rhn_package_id_seq',
@@ -984,10 +986,16 @@ def processPackages(self, packages, uploadForce=0, ignoreUploaded=0,
                 package['header_start'] = -1
                 package['header_end'] = -1
 
-            self.__processObjectCollection__([package, ], 'rhnPackage', tableList,
+            try:
+                self.__processObjectCollection__([package, ], 'rhnPackage', tableList,
                                              uploadForce=uploadForce, forceVerify=forceVerify,
                                              ignoreUploaded=ignoreUploaded, severityLimit=1,
                                              transactional=transactional)
+            except Exception as e:
+                syncLib.log(0, "Error during processing package %s-%s-%s:%s.%s.\n%s" %
+                            (package['name'], package['version'], package['release'], package['epoch'], package['arch'],
+                             str(e)))
+                raise
 
     def processErrata(self, errata):
         # Insert/update the packages

python/spacewalk/server/importlib/packageImport.py

@@ -285,16 +285,20 @@ def _processPackage(self, package):
                 self.checksums[fchecksumTuple] = None
 
         # Uniquify changelog entries
-        unique_package_changelog_hash = {}
+        unique_package_changelog_hash = set()
         unique_package_changelog = []
         for changelog in package['changelog']:
-            key = (self._fix_encoding(changelog['name'][:128]), self._fix_encoding(changelog['time']), self._fix_encoding(changelog['text'])[:3000])
+            changelog_name = self._fix_encoding(changelog['name'][:128])
+            changelog_time = self._fix_encoding(changelog['time'])
+            changelog_text = self._fix_encoding(changelog['text'])[:3000]
+            key = (changelog_name, changelog_time, changelog_text)
             if key not in unique_package_changelog_hash:
                 self.changelog_data[key] = None
-                changelog['name'] = changelog['name'][:128]
-                changelog['text'] = changelog['text'][:3000]
+                changelog['name'] = changelog_name
+                changelog['text'] = changelog_text
+                changelog['time'] = changelog_time
                 unique_package_changelog.append(changelog)
-                unique_package_changelog_hash[key] = 1
+                unique_package_changelog_hash.add(key)
         package['changelog'] = unique_package_changelog
 
         # fix encoding issues in package summary and description

python/spacewalk/spacewalk-backend.changes.witek.add-unique-index

@@ -0,0 +1 @@
+- Add unique index for rhnpackagechangelogdata table

schema/spacewalk/common/tables/rhnPackageChangeLogData.sql

@@ -13,7 +13,6 @@
 -- in this software or its documentation.
 --
 
-
 CREATE TABLE rhnPackageChangeLogData
 (
     id          NUMERIC NOT NULL
@@ -28,8 +27,9 @@ CREATE TABLE rhnPackageChangeLogData
 
 ;
 
-CREATE INDEX rhn_pkg_cld_nt_idx
-    ON rhnPackageChangeLogData (name, time)
+CREATE UNIQUE INDEX rhn_pkg_cld_ntt_idx
+    ON rhnPackageChangeLogData
+    USING btree(name, digest("text", 'sha512'::text), time)
 
     ;
 

schema/spacewalk/postgres/start.sql

@@ -21,3 +21,5 @@ create temporary table store_searchpath as select setting from pg_settings where
 
 update pg_settings set setting = (select setting from store_searchpath) where name = 'search_path';
 drop table store_searchpath;
+
+CREATE EXTENSION pgcrypto;

schema/spacewalk/upgrade/susemanager-schema-4.4.6-to-susemanager-schema-4.4.7/203-add-changelogdata-index.sql

@@ -0,0 +1,38 @@
+create or replace function remove_duplicate_changelogdata()
+returns void as
+$$
+declare original record;
+declare duplicate record;
+begin
+    for original in select min(id) as id
+            from rhnpackagechangelogdata
+            group by name, text, time
+            having count(*) > 1 loop
+        for duplicate in select data2.id
+                from rhnpackagechangelogdata data1, rhnpackagechangelogdata data2
+                where data1.name = data2.name
+                and data1.text = data2.text
+                and data1.time = data2.time
+                and data1.id != data2.id
+                and data1.id = original.id loop
+            update rhnpackagechangelogrec
+                set changelog_data_id = original.id
+                where changelog_data_id = duplicate.id;
+            delete from rhnpackagechangelogdata
+                where id = duplicate.id;
+        end loop;
+    end loop;
+end;
+$$ language plpgsql;
+
+select remove_duplicate_changelogdata();
+drop function remove_duplicate_changelogdata();
+
+drop index if exists rhn_pkg_cld_nt_idx;
+drop index if exists rhn_pkg_cld_ntt_idx;
+
+create extension if not exists pgcrypto;
+create unique index concurrently rhn_pkg_cld_ntt_idx
+    on "rhnpackagechangelogdata"
+    using btree(name, digest("text", 'sha512'::text), "time");
+