From b2ef4c2d0f0fd5717a231d3f4ffcde1f3684ab80 Mon Sep 17 00:00:00 2001 From: angelic_knight Date: Wed, 27 Oct 2021 01:07:29 -0500 Subject: [PATCH 01/13] First few fixes for exporting mods. This is mostly .dat spec fixes. --- PyPoE/poe/constants.py | 4 ++ PyPoE/poe/file/specification/data/stable.py | 58 ++++++++------------- PyPoE/poe/file/specification/fields.py | 3 +- 3 files changed, 27 insertions(+), 38 deletions(-) diff --git a/PyPoE/poe/constants.py b/PyPoE/poe/constants.py index 63e97963..019e7953 100644 --- a/PyPoE/poe/constants.py +++ b/PyPoE/poe/constants.py @@ -791,6 +791,10 @@ class MOD_GENERATION_TYPE(IntEnumOverride): FLASK_ENCHANTMENT_ENKINDLING = 21 FLASK_ENCHANTMENT_INSTILLING = 22 EXPEDITION_LOGBOOK = 23 + SCOURGE_UPSIDE = 24 + SCOURGE_DOWNSIDE = 25 + SCOURGE_SHIFT_GIMMICK = 26 + class WORDLISTS(IntEnumOverride): diff --git a/PyPoE/poe/file/specification/data/stable.py b/PyPoE/poe/file/specification/data/stable.py index 0b74c754..7ffde5f9 100644 --- a/PyPoE/poe/file/specification/data/stable.py +++ b/PyPoE/poe/file/specification/data/stable.py @@ -205,6 +205,10 @@ name='Flag2', type='bool', ), + Field( + name='Flag3', + type='bool', + ), ), ), 'AchievementSetRewards.dat': File( @@ -2417,24 +2421,6 @@ type='ulong', key='SoundEffects.dat', ), - Field( - name='NormalPurchase_BaseItemTypesKeys', - type='ref|list|ref|generic', - key='BaseItemTypes.dat', - ), - Field( - name='NormalPurchase_Costs', - type='ref|list|int', - ), - Field( - name='MagicPurchase_BaseItemTypesKeys', - type='ref|list|ref|generic', - key='BaseItemTypes.dat', - ), - Field( - name='MagicPurchase_Costs', - type='ref|list|int', - ), Field( name='TagsKeys', type='ref|list|ulong', @@ -2455,7 +2441,7 @@ key='ItemVisualIdentity.dat', ), Field( - name='Unknown0', + name='Hash', type='int', unique=True, ), @@ -2464,24 +2450,6 @@ type='ref|list|ulong', key='AchievementItems.dat', ), - Field( - name='RarePurchase_BaseItemTypesKeys', - type='ref|list|ref|generic', - key='BaseItemTypes.dat', - ), - Field( - name='RarePurchase_Costs', - type='ref|list|int', - ), - Field( - name='UniquePurchase_BaseItemTypesKeys', - type='ref|list|ref|generic', - key='BaseItemTypes.dat', - ), - Field( - name='UniquePurchase_Costs', - type='ref|list|int', - ), Field( name='Inflection', type='ref|string', @@ -5220,6 +5188,10 @@ name='XBoxText', type='ref|string', ), + Field( + name='XBoxText2', + type='ref|string', + ), Field( name='Unknown0', type='int', @@ -5345,6 +5317,10 @@ name='FormatText', type='ref|string', ), + Field( + name='Unknown0', #Might be a rate divisor. The per-minute costs are 60 while normal costs are 1 + type='int', + ), ), ), 'CraftingBenchOptions.dat': File( @@ -12312,6 +12288,14 @@ type='ulong', key='ItemStances.dat', ), + Field( + name="CanScourge", + type='bool', + ), + Field( + name="Flag1", + type='bool', + ), ), ), 'ItemCostPerLevel.dat': File( diff --git a/PyPoE/poe/file/specification/fields.py b/PyPoE/poe/file/specification/fields.py index 503054e9..e3fe8e65 100644 --- a/PyPoE/poe/file/specification/fields.py +++ b/PyPoE/poe/file/specification/fields.py @@ -448,7 +448,8 @@ def __init__(self, pass for item in delete_zip: - del self.columns_zip[item] + if item in self.columns_zip.keys(): #This may just be hiding errors. + del self.columns_zip[item] def __getitem__(self, item): return getattr(self, item) From b34b707bbb1be167977f534c71b4b611a0cfdef9 Mon Sep 17 00:00:00 2001 From: angelic_knight Date: Thu, 28 Oct 2021 00:35:22 -0500 Subject: [PATCH 02/13] Enough spec fixes to export all 26k mods without crashing. Many mods don't have proper translations from internal descriptions to normal text. --- PyPoE/poe/file/specification/data/stable.py | 139 +++++++++++++++++--- PyPoE/poe/file/translations.py | 26 +++- 2 files changed, 146 insertions(+), 19 deletions(-) diff --git a/PyPoE/poe/file/specification/data/stable.py b/PyPoE/poe/file/specification/data/stable.py index 7ffde5f9..49d36a47 100644 --- a/PyPoE/poe/file/specification/data/stable.py +++ b/PyPoE/poe/file/specification/data/stable.py @@ -1476,6 +1476,20 @@ file_path=True, file_ext='.ogg', ), + Field( + name='PassiveTreeImage', + type='ref|string', + file_path=True, + file_ext='.dds', + ), + Field( + name='Unknown0', + type='int', + ), + Field( + name='Unknown1', + type='int', + ), ), ), 'AtlasAwakeningStats.dat': File( @@ -4233,6 +4247,10 @@ name='Flag18', type='bool', ), + Field( + name='Key2', + type='ulong', + ), ), ), 'BuffTemplates.dat': File( @@ -4292,6 +4310,10 @@ type='ref|list|ulong', key='Stats.dat', ), + Field( + name='Unknown8', + type='int', + ), ), ), 'BuffVisualOrbArt.dat': File( @@ -4870,19 +4892,15 @@ key='SkillGems.dat', ), Field( - name='Unknown0', - type='int', - ), - Field( - name='Unknown1', - type='int', + name='Key0', + type='ulong', ), Field( - name='Unknown2', + name='Unknown0', type='int', ), Field( - name='Unknown3', + name='Unknown1', type='int', ), Field( @@ -4901,17 +4919,13 @@ key='BaseItemTypes.dat', ), Field( - name='Unknown4', - type='int', + name='Gender', + type='ref|string', ), Field( name='TraitDescription', type='ref|string', ), - Field( - name='Key0', - type='ulong', - ), Field( name='Key1', type='ulong', @@ -4925,13 +4939,31 @@ type='ulong', ), Field( - name='Unknown5', + name='Key4', + type='ulong', + ), + Field( + name='Unknown2', type='int', ), Field( name='Keys1', type='ref|list|ulong', ), + Field( + name='PassiveTreeImage', + type='ref|string', + file_path=True, + file_ext='.dds', + ), + Field( + name='Unknown3', + type='int', + ), + Field( + name='Unknown4', + type='int', + ), ), ), 'ChestClusters.dat': File( @@ -12802,6 +12834,10 @@ ), Field( name='Unknown15', + type='ref|string', + ), + Field( + name='Unknown16', type='int', ), ), @@ -18454,6 +18490,68 @@ ), ), ), + 'PassiveSkillMasteryEffects.dat': File( + fields=( + Field( + name='ID', + type='ref|string', + ), + Field( + name='hash', + type='int', + ), + Field( + name='Stats', + type='ref|list|ulong', + key='Stats.dat', + ), + Field( + name='Stat1Value', + type='int', + ), + Field( + name='Stat2Value', + type='int', + ), + Field( + name='Stat3Value', + type='int', + ), + ), + ), + 'PassiveSkillMasteryGroups.dat': File( + fields=( + Field( + name='ID', + type='ref|string', + ), + Field( + name='MasteryEffects', + type='ref|list|ulong', + key='PassiveSkillMasteryEffects.dat', + ), + Field( + name='InactiveIcon', + type='ref|string', + ), + Field( + name='ActiveIcon', + type='ref|string', + ), + Field( + name='ActiveEffectImage', + type='ref|string', + ), + Field( + name='Flag0', + type='bool', + ), + Field( + name='Key0', + type='ulong', + ), + ), + ), 'PassiveSkillStatCategories.dat': File( fields=( Field( @@ -18627,7 +18725,7 @@ key='GrantedEffectsPerLevel.dat', ), Field( - name='Flag0', + name='IsAnnointmentOnly', type='bool', ), Field( @@ -18635,7 +18733,7 @@ type='int', ), Field( - name='Flag1', + name='IsExpansion', type='bool', ), Field( @@ -18643,9 +18741,14 @@ type='bool', ), Field( - name='Unknown1', + name='SkillType', type='int', ), + Field( + name='MasteryGroup', + type='ulong', + key='PassiveSkillMasteryGroups.dat', + ), ), virtual_fields=( VirtualField( diff --git a/PyPoE/poe/file/translations.py b/PyPoE/poe/file/translations.py index 4916394f..d251cf92 100644 --- a/PyPoE/poe/file/translations.py +++ b/PyPoE/poe/file/translations.py @@ -147,7 +147,7 @@ r'^' r'[\s]*' r'(?P(?:[0-9\-\|#!]+[ \t]+)+)' - r'"(?P.*)"' + r'"(?P.*\s*)"' r'(?P(?:[ \t]*[\w%]+)*)' r'[ \t]*[\r\n]*' r'$', @@ -2342,6 +2342,30 @@ def _get_from_value(value): reverse_handler=lambda v: int(v)//20, ) +TranslationQuantifier( + id='times_one_point_five', + handler=lambda v: v*1.5, + reverse_handler=lambda v: int(v/1.5), +) + +TranslationQuantifier( + id='double', + handler=lambda v: v*2, + reverse_handler=lambda v: int(v)//2, +) + +TranslationQuantifier( + id='negate_and_double', + handler=lambda v: -v * 2, + reverse_handler=lambda v: int(-v) // 2, +) + +TranslationQuantifier( + id='divide_by_four', + handler=lambda v: v / 4, + reverse_handler=lambda v: v * 4, +) + TranslationQuantifier( id='canonical_line', type=TranslationQuantifier.QuantifierTypes.STRING, From 557a214908d9ec4746fa1ca3b105c8c04486d2b4 Mon Sep 17 00:00:00 2001 From: angelic_knight Date: Thu, 28 Oct 2021 00:40:32 -0500 Subject: [PATCH 03/13] Fixing #24, where our PyPoE was using Fandom's table name. --- PyPoE/cli/exporter/wiki/parsers/lua.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyPoE/cli/exporter/wiki/parsers/lua.py b/PyPoE/cli/exporter/wiki/parsers/lua.py index 29ee4dc5..f59f6dbb 100644 --- a/PyPoE/cli/exporter/wiki/parsers/lua.py +++ b/PyPoE/cli/exporter/wiki/parsers/lua.py @@ -1438,7 +1438,7 @@ def SortCategoryHelper(d): # 'default': '', # }), ('CraftingItemClassCategoriesKeys', { - 'key': 'crafting_item_class_categories', + 'key': 'item_class_categories', 'value': lambda v: [k['Text'] for k in v], }), # ('CraftingBenchUnlockCategoriesKeys', { From f702e9a6a37fe3d79a30a52e10a037641b8a0b9b Mon Sep 17 00:00:00 2001 From: angelic_knight Date: Mon, 1 Nov 2021 20:56:59 -0500 Subject: [PATCH 04/13] Remove redundant quote sanitizing. Merged commit 5c4db74cc51d518f5f761af601649012c0bf6069 should have fixed this elsewhere. --- PyPoE/cli/exporter/util.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/PyPoE/cli/exporter/util.py b/PyPoE/cli/exporter/util.py index 9a74c712..622543a1 100644 --- a/PyPoE/cli/exporter/util.py +++ b/PyPoE/cli/exporter/util.py @@ -75,8 +75,6 @@ def get_content_path(): def fix_path(path: str) -> str: - #First, replace any double quotes with HTML-encoded quotes - path = path.replace('\"','"') if re.search('[a-zA-Z]:.*', path) is not None: return path[:2] + re.sub(r':', '_', path[2:]) else: From 78214dc2f69b75c0f31c5a0c32967f1db57307ad Mon Sep 17 00:00:00 2001 From: angelic_knight Date: Mon, 1 Nov 2021 23:47:23 -0500 Subject: [PATCH 05/13] Supporting armour base types with ranges for their defence values Adding printing to indicate current rows being processed (up to 200 row IDs and names will print) during exporting. Re-evaluated and updated the MAP_FRAGMENTS_FAMILIES enum Fixed some specs that were erroring out during item exporting. --- PyPoE/cli/exporter/wiki/parsers/item.py | 104 ++++++++++++++------ PyPoE/poe/constants.py | 36 ++++--- PyPoE/poe/file/specification/data/stable.py | 42 +++++++- 3 files changed, 129 insertions(+), 53 deletions(-) diff --git a/PyPoE/cli/exporter/wiki/parsers/item.py b/PyPoE/cli/exporter/wiki/parsers/item.py index 54e325b5..4b7064a8 100644 --- a/PyPoE/cli/exporter/wiki/parsers/item.py +++ b/PyPoE/cli/exporter/wiki/parsers/item.py @@ -56,6 +56,7 @@ def _apply_column_map(infobox, column_map, list_object): + for k, data in column_map: value = list_object[k] if data.get('condition') and not data['condition'](value): @@ -2200,24 +2201,40 @@ def _type_amulet(self, infobox, base_item_type): _type_armour = _type_factory( data_file='ArmourTypes.dat', data_mapping=( - ('Armour', { - 'template': 'armour', + ('ArmourMin', { + 'template': 'armour_min', 'condition': lambda v: v > 0, }), - ('Evasion', { - 'template': 'evasion', + ('ArmourMax', { + 'template': 'armour_max', 'condition': lambda v: v > 0, }), - ('EnergyShield', { - 'template': 'energy_shield', + ('EvasionMin', { + 'template': 'evasion_min', + 'condition': lambda v: v > 0, + }), + ('EvasionMin', { + 'template': 'evasion_max', + 'condition': lambda v: v > 0, + }), + ('EnergyShieldMin', { + 'template': 'energy_shield_min', + 'condition': lambda v: v > 0, + }), + ('EnergyShieldMax', { + 'template': 'energy_shield_max', 'condition': lambda v: v > 0, }), ('IncreasedMovementSpeed', { 'template': 'movement_speed', 'condition': lambda v: v != 0, }), - ('Ward', { - 'template': 'ward', + ('WardMin', { + 'template': 'ward_min', + 'condition': lambda v: v != 0, + }), + ('WardMax', { + 'template': 'ward_max', 'condition': lambda v: v != 0, }), ), @@ -2362,17 +2379,17 @@ def _currency_extra(self, infobox, base_item_type, currency): ) _master_hideout_doodad_map = ( - ('HideoutNPCsKey', { - 'template': 'master', - 'format': lambda v: v['Hideout_NPCsKey']['Name'], - 'condition': lambda v: v is not None, - }), - ('MasterLevel', { - 'template': 'master_level_requirement', - }), - ('FavourCost', { - 'template': 'master_favour_cost', - }), + # ('HideoutNPCsKey', { + # 'template': 'master', + # 'format': lambda v: v['Hideout_NPCsKey']['Name'], + # 'condition': lambda v: v is not None, + # }), + # ('MasterLevel', { + # 'template': 'master_level_requirement', + # }), + # ('FavourCost', { + # 'template': 'master_favour_cost', + # }), ) def _apply_master_map(self, infobox, base_item_type, hideout): @@ -2387,19 +2404,19 @@ def _apply_master_map(self, infobox, base_item_type, hideout): 'template': 'is_master_doodad', 'format': lambda v: not v, }), - ('HideoutNPCsKey', { - 'template': 'master', - 'format': lambda v: v['Hideout_NPCsKey']['Name'], - 'condition': lambda v: v, - }), - ('FavourCost', { - 'template': 'master_favour_cost', - #'condition': lambda v: v, - }), - ('MasterLevel', { - 'template': 'master_level_requirement', - #'condition': lambda v: v, - }), + # ('HideoutNPCsKey', { + # 'template': 'master', + # 'format': lambda v: v['Hideout_NPCsKey']['Name'], + # 'condition': lambda v: v, + # }), + # ('FavourCost', { + # 'template': 'master_favour_cost', + # #'condition': lambda v: v, + # }), + # ('MasterLevel', { + # 'template': 'master_level_requirement', + # #'condition': lambda v: v, + # }), ('Variation_AOFiles', { 'template': 'variation_count', 'format': lambda v: len(v), @@ -2781,10 +2798,14 @@ def _harvest_plant_booster_extra(self, infobox, base_item_type, row_index=True, ) + ''' + This defines the expected data elements for an item class. + ''' _cls_map = { # Jewellery 'Amulet': (_type_amulet, ), # Armour types + 'Armour': (_type_level, _type_attribute, _type_armour, ), 'Gloves': (_type_level, _type_attribute, _type_armour, ), 'Boots': (_type_level, _type_attribute, _type_armour, ), 'Body Armour': (_type_level, _type_attribute, _type_armour, ), @@ -3196,6 +3217,8 @@ def _export(self, parsed_args, items): cls_id = base_item_type['ItemClassesKey']['Id'] m_id = base_item_type['Id'] + self._print_item_rowid(parsed_args, base_item_type) + infobox = OrderedDict() self._process_base_item_type(base_item_type, infobox) self._process_purchase_costs(base_item_type, infobox) @@ -3283,6 +3306,23 @@ def _export(self, parsed_args, items): return r + def _print_item_rowid(self, parsed_args, base_item_type): + #Don't print anything if not running in the rowid mode. + if (parsed_args.start is None) or (parsed_args.end is None): + return + + export_row_count = parsed_args.end - parsed_args.start + #If we're printing less than 100 rows, print every rowid + if export_row_count <= 100: + print_granularity = 1 + else: + print_granularity = export_row_count//100 + + item_offset = base_item_type.rowid - parsed_args.start + if (item_offset == 0) or item_offset % print_granularity == 0: + console('Processing item with rowid {}: {}'.format(base_item_type.rowid, base_item_type['Name'])) + return + def _format_map_name(self, base_item_type, map_series, language=None): if language is None: language = self._language diff --git a/PyPoE/poe/constants.py b/PyPoE/poe/constants.py index 985a6e33..c2d7cba5 100644 --- a/PyPoE/poe/constants.py +++ b/PyPoE/poe/constants.py @@ -605,23 +605,27 @@ class MAP_FRAGMENT_FAMILIES(IntEnumOverride): """ Representation of map fragment families (MapFragmentFamilies.dat) """ - BESTIARY = 0 + BESTIARY_AND_SULPHITE = 0 #Maybe just master-related? BREACH = 1 - CARTOGRAPHY = 2 - RELIQUARY = 3 - SHAPER = 4 - ELDER = 5 - DIVINATION = 6 - TORMENT = 7 - AMBUSH = 8 - HARBINGER = 9 - PERANDUS = 10 - LEGION = 11 - METAMORPH = 12 - REGULAR = 13 - RITUAL = 14 - EXPEDITION = 15 - SCOURGE = 16 + CARTOGRAPHY_SCARAB = 2 + RELIQUARY_SCARAB = 3 + SHAPER_SCARAB = 4 + ELDER_SCARAB = 5 + DIVINATION_SCARAB = 6 + TORMENT_SCARAB = 7 + AMBUSH_SCARAB = 8 + HARBINGER_SCARAB = 9 + EXPEDITION_SCARAB = 10 + LEGION_SCARAB = 11 + METAMORPH_SCARAB = 12 + BLIGHT_SCARAB = 13 + ABYSS_SCARAB = 14 + KARUI = 15 + MARAKETH = 16 + ETERNAL = 17 + TEMPLAR = 18 + VAAL = 19 + REGULAR = 20 DEFAULT = REGULAR STANDARD = REGULAR diff --git a/PyPoE/poe/file/specification/data/stable.py b/PyPoE/poe/file/specification/data/stable.py index f34c585f..74506779 100644 --- a/PyPoE/poe/file/specification/data/stable.py +++ b/PyPoE/poe/file/specification/data/stable.py @@ -147,6 +147,22 @@ name='Message', type='ref|string', ), + Field( + name='Unknown0', + type='byte', + ), + Field( + name='Unknown1', + type='byte', + ), + Field( + name='Unknown2', + type='byte', + ), + Field( + name='Unknown3', + type='byte', + ), ), ), 'AchievementItems.dat': File( @@ -230,6 +246,22 @@ name='HideoutName', type='ref|string', ), + Field( + name='Unknown0', + type='byte', + ), + Field( + name='Unknown2', + type='byte', + ), + Field( + name='Unknown3', + type='byte', + ), + Field( + name='Unknown4', + type='byte', + ), ), ), 'AchievementSetsDisplay.dat': File( @@ -19266,7 +19298,7 @@ ), Field( name='Stuck_AOFile', - type='ref|string', + type='ref|list|ref|string', file_path=True, file_ext='.ao', ), @@ -19324,10 +19356,6 @@ name='Keys0', type='ref|list|ulong', ), - Field( - name='Key3', - type='ulong', - ), Field( name='Flag5', type='bool', @@ -19336,6 +19364,10 @@ name='Unknown12', type='int', ), + Field( + name='Unknown13', + type='int', + ), ), ), 'Prophecies.dat': File( From 860ef96c9e1633e7917d9f66e35a3fee0fa7f9c7 Mon Sep 17 00:00:00 2001 From: angelic_knight Date: Tue, 2 Nov 2021 00:18:13 -0500 Subject: [PATCH 06/13] Stop using broken hideout doodad conflict resolver (this introduces issues where conflicts aren't resolved) Fix the HeistJobs.dat spec to include the new columns for 3.16 --- PyPoE/cli/exporter/wiki/parsers/item.py | 3 ++- PyPoE/poe/file/specification/data/stable.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/PyPoE/cli/exporter/wiki/parsers/item.py b/PyPoE/cli/exporter/wiki/parsers/item.py index 4b7064a8..bd58f3d1 100644 --- a/PyPoE/cli/exporter/wiki/parsers/item.py +++ b/PyPoE/cli/exporter/wiki/parsers/item.py @@ -3013,7 +3013,8 @@ def _conflict_atlas_region_upgrade( _conflict_resolver_map = { 'Active Skill Gem': _conflict_active_skill_gems, 'QuestItem': _conflict_quest_items, - 'HideoutDoodad': _conflict_hideout_doodad, + #TODO: Make a new doodad resolver that doesn't rely on 'HideoutNPCsKey' + #'HideoutDoodad': _conflict_hideout_doodad, 'Map': _conflict_maps, 'MapFragment': _conflict_map_fragments, 'DivinationCard': _conflict_divination_card, diff --git a/PyPoE/poe/file/specification/data/stable.py b/PyPoE/poe/file/specification/data/stable.py index 74506779..9abe4ae5 100644 --- a/PyPoE/poe/file/specification/data/stable.py +++ b/PyPoE/poe/file/specification/data/stable.py @@ -10779,6 +10779,14 @@ type='ulong', key='Stats.dat', ), + Field( + name='ConsoleBlueprintLegend', + type='ref|string', + ), + Field( + name='Description', + type='ref|string', + ), ), ), 'HeistJobsExperiencePerLevel.dat': File( From f1283ae08e5691a883f8b7862ac6d99cfa0eb988 Mon Sep 17 00:00:00 2001 From: angelic_knight Date: Wed, 3 Nov 2021 01:49:54 -0500 Subject: [PATCH 07/13] Update PassiveSkillGraph.psg parsing to work for 3.16. Supporting exporting passives that don't have icon paths. Revert some spec column name changes that were causing issues. I need to run the spec autogeneration sometime and then update PyPoE hardcoded references to use the new values instead of manually editing the spec. --- PyPoE/cli/exporter/wiki/parsers/passives.py | 5 +++++ PyPoE/poe/file/psg.py | 10 +++++++++- PyPoE/poe/file/specification/data/stable.py | 18 +++++++++--------- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/PyPoE/cli/exporter/wiki/parsers/passives.py b/PyPoE/cli/exporter/wiki/parsers/passives.py index 6771ebd6..eda32841 100644 --- a/PyPoE/cli/exporter/wiki/parsers/passives.py +++ b/PyPoE/cli/exporter/wiki/parsers/passives.py @@ -39,6 +39,7 @@ # Python import re import os.path +import warnings from functools import partialmethod from collections import OrderedDict @@ -289,6 +290,10 @@ def export(self, parsed_args, passives): data['icon'] = '%s (%s)' % (icon[-1], icon[-2]) else: data['icon'] = icon[-1] + #atlas_start_node doesn't have an icon path + else: + data['icon'] = '' + warnings.warn('Icon path file not found for {}: {}'.format(passive['Id'], passive['Name'])) data['icon'] = data['icon'].replace('.dds', '') diff --git a/PyPoE/poe/file/psg.py b/PyPoE/poe/file/psg.py index 27ef4b12..b4f24b9c 100644 --- a/PyPoE/poe/file/psg.py +++ b/PyPoE/poe/file/psg.py @@ -270,15 +270,23 @@ def _read(self, buffer, *args, **kwargs): version = struct.unpack_from(' 1000): + raise ValueError( + 'root_length is unrealistically large at {}.\nStopping to prevent allocating too much memory'.format(root_length) + ) offset += 4 self.root_passives = list(struct.unpack_from( diff --git a/PyPoE/poe/file/specification/data/stable.py b/PyPoE/poe/file/specification/data/stable.py index 9abe4ae5..5ac6bb0b 100644 --- a/PyPoE/poe/file/specification/data/stable.py +++ b/PyPoE/poe/file/specification/data/stable.py @@ -4047,7 +4047,7 @@ key='BuffDefinitions.dat', ), Field( - name='Data0', + name='Buff_StatValues', type='ref|list|int', ), Field( @@ -18777,7 +18777,7 @@ file_ext='.dds', ), Field( - name='Stats', + name='StatsKeys', type='ref|list|ulong', key='Stats.dat', ), @@ -18808,7 +18808,7 @@ type='ref|string', ), Field( - name='Characters', + name='CharactersKeys', type='ref|list|ulong', key='Characters.dat', ), @@ -18829,7 +18829,7 @@ type='bool', ), Field( - name='AchievementItem', + name='AchievementItemsKey', type='ulong', key='AchievementItems.dat', ), @@ -18847,7 +18847,7 @@ type='bool', ), Field( - name='ReminderStrings', + name='Reminder_ClientStringsKeys', type='ref|list|ulong', key='ClientStrings.dat', ), @@ -18868,12 +18868,12 @@ type='int', ), Field( - name='PassiveSkillBuffs', + name='PassiveSkillBuffsKeys', type='ref|list|ulong', key='BuffTemplates.dat', ), Field( - name='GrantedEffectsPerLevel', + name='GrantedEffectsPerLevelKey', type='ulong', key='GrantedEffectsPerLevel.dat', ), @@ -18909,8 +18909,8 @@ fields=('Stat1Value', 'Stat2Value', 'Stat3Value', 'Stat4Value', 'Stat5Value'), ), VirtualField( - name='StatsZip', - fields=('Stats', 'StatValues'), + name='Stats', + fields=('StatsKeys', 'StatValues'), zip=True, ), ), From dcd5e3c8b38b99ffee92cafd0097a82c57f9b046 Mon Sep 17 00:00:00 2001 From: angelic_knight Date: Wed, 3 Nov 2021 21:56:29 -0500 Subject: [PATCH 08/13] Energy Blade & Cleanup Adding disambiguation suffixes for Energy Blade skill gem and Energy Blade swords (that you get from using the skill gem) Removed old fix to an issue since it was fixed a better way (in fields.py). Added extra printing to skill export. --- PyPoE/cli/exporter/wiki/parsers/item.py | 14 ++++++++++++++ PyPoE/cli/exporter/wiki/parsers/skill.py | 4 ++-- PyPoE/poe/file/specification/fields.py | 3 +-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/PyPoE/cli/exporter/wiki/parsers/item.py b/PyPoE/cli/exporter/wiki/parsers/item.py index bd58f3d1..36213259 100644 --- a/PyPoE/cli/exporter/wiki/parsers/item.py +++ b/PyPoE/cli/exporter/wiki/parsers/item.py @@ -766,12 +766,26 @@ class ItemsParser(SkillParserShared): 'Metadata/Items/Gems/SupportGemConcentratedEffect': '', 'Metadata/Items/Gems/SupportGemIncreasedCriticalStrikes': '', 'Metadata/Items/Gems/SupportGemMeleeSplash': '', + 'Metadata/Items/Gems/SkillGemEnergyBlade': '', # ================================================================= # One Hand Axes # ================================================================= 'Metadata/Items/Weapons/OneHandWeapons/OneHandAxes/OneHandAxe22': '', + + # ================================================================= + # One Hand Swords + # ================================================================= + 'Metadata/Items/Weapons/OneHandWeapons/OneHandSwords/StormBladeOneHand': + ' (One Handed Sword)', + + # ================================================================= + # Two Hand Swords + # ================================================================= + 'Metadata/Items/Weapons/TwoHandWeapons/TwoHandSwords/StormBladeTwoHand': + ' (Two Handed Sword)', + # ================================================================= # Boots # ================================================================= diff --git a/PyPoE/cli/exporter/wiki/parsers/skill.py b/PyPoE/cli/exporter/wiki/parsers/skill.py index 9db32252..09ae6895 100644 --- a/PyPoE/cli/exporter/wiki/parsers/skill.py +++ b/PyPoE/cli/exporter/wiki/parsers/skill.py @@ -742,7 +742,7 @@ def export(self, parsed_args, skills): if not parsed_args.allow_skill_gems and skill in \ self.rr['SkillGems.dat'].index['GrantedEffectsKey']: console( - 'Skipping skill gem skill "%s"' % skill['Id'], + 'Skipping skill gem skill "{}" at row {}'.format(skill['Id'], skill.rowid), msg=Msg.warning) continue data = OrderedDict() @@ -751,7 +751,7 @@ def export(self, parsed_args, skills): self._skill(ge=skill, infobox=data, parsed_args=parsed_args) except Exception as e: console( - 'Error when parsing skill "%s":' % skill['Id'], + 'Error when parsing skill "{}" at {}:'.format(skill['Id'], skill.rowid), msg=Msg.error) console(traceback.format_exc(), msg=Msg.error) diff --git a/PyPoE/poe/file/specification/fields.py b/PyPoE/poe/file/specification/fields.py index e3fe8e65..503054e9 100644 --- a/PyPoE/poe/file/specification/fields.py +++ b/PyPoE/poe/file/specification/fields.py @@ -448,8 +448,7 @@ def __init__(self, pass for item in delete_zip: - if item in self.columns_zip.keys(): #This may just be hiding errors. - del self.columns_zip[item] + del self.columns_zip[item] def __getitem__(self, item): return getattr(self, item) From 4fcc59fa5f7aef8a88ff23d250bc69bd573b899d Mon Sep 17 00:00:00 2001 From: angelic_knight Date: Wed, 3 Nov 2021 22:26:43 -0500 Subject: [PATCH 09/13] Fix printing item rowids so it won't crash when not in rowid mode. --- PyPoE/cli/exporter/wiki/parsers/item.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PyPoE/cli/exporter/wiki/parsers/item.py b/PyPoE/cli/exporter/wiki/parsers/item.py index 36213259..fab8adf8 100644 --- a/PyPoE/cli/exporter/wiki/parsers/item.py +++ b/PyPoE/cli/exporter/wiki/parsers/item.py @@ -3323,7 +3323,7 @@ def _export(self, parsed_args, items): def _print_item_rowid(self, parsed_args, base_item_type): #Don't print anything if not running in the rowid mode. - if (parsed_args.start is None) or (parsed_args.end is None): + if (not 'start' in vars(parsed_args).keys()) or (not 'end' in vars(parsed_args).keys()): return export_row_count = parsed_args.end - parsed_args.start From 59bcc8aebeb1746b2f090bfd144ad9d71e01bd74 Mon Sep 17 00:00:00 2001 From: angelic_knight Date: Sat, 6 Nov 2021 12:33:22 -0500 Subject: [PATCH 10/13] Fixing PR issues from PR #37 --- PyPoE/cli/exporter/wiki/parsers/item.py | 4 ++-- PyPoE/cli/exporter/wiki/parsers/passives.py | 2 +- PyPoE/cli/exporter/wiki/parsers/skill.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/PyPoE/cli/exporter/wiki/parsers/item.py b/PyPoE/cli/exporter/wiki/parsers/item.py index fab8adf8..676e2972 100644 --- a/PyPoE/cli/exporter/wiki/parsers/item.py +++ b/PyPoE/cli/exporter/wiki/parsers/item.py @@ -3323,7 +3323,7 @@ def _export(self, parsed_args, items): def _print_item_rowid(self, parsed_args, base_item_type): #Don't print anything if not running in the rowid mode. - if (not 'start' in vars(parsed_args).keys()) or (not 'end' in vars(parsed_args).keys()): + if not set(['start', 'end']).issubset(vars(parsed_args).keys()): return export_row_count = parsed_args.end - parsed_args.start @@ -3335,7 +3335,7 @@ def _print_item_rowid(self, parsed_args, base_item_type): item_offset = base_item_type.rowid - parsed_args.start if (item_offset == 0) or item_offset % print_granularity == 0: - console('Processing item with rowid {}: {}'.format(base_item_type.rowid, base_item_type['Name'])) + console(f"Processing item with rowid {base_item_type.rowid}: {base_item_type['Name']}") return def _format_map_name(self, base_item_type, map_series, language=None): diff --git a/PyPoE/cli/exporter/wiki/parsers/passives.py b/PyPoE/cli/exporter/wiki/parsers/passives.py index eda32841..952015cd 100644 --- a/PyPoE/cli/exporter/wiki/parsers/passives.py +++ b/PyPoE/cli/exporter/wiki/parsers/passives.py @@ -293,7 +293,7 @@ def export(self, parsed_args, passives): #atlas_start_node doesn't have an icon path else: data['icon'] = '' - warnings.warn('Icon path file not found for {}: {}'.format(passive['Id'], passive['Name'])) + warnings.warn(f"Icon path file not found for {passive['Id']}: {passive['Name']}") data['icon'] = data['icon'].replace('.dds', '') diff --git a/PyPoE/cli/exporter/wiki/parsers/skill.py b/PyPoE/cli/exporter/wiki/parsers/skill.py index 09ae6895..f985bf6a 100644 --- a/PyPoE/cli/exporter/wiki/parsers/skill.py +++ b/PyPoE/cli/exporter/wiki/parsers/skill.py @@ -742,7 +742,7 @@ def export(self, parsed_args, skills): if not parsed_args.allow_skill_gems and skill in \ self.rr['SkillGems.dat'].index['GrantedEffectsKey']: console( - 'Skipping skill gem skill "{}" at row {}'.format(skill['Id'], skill.rowid), + f"Skipping skill gem skill \"{skill['Id']}\" at row {skill.rowid}", msg=Msg.warning) continue data = OrderedDict() @@ -751,7 +751,7 @@ def export(self, parsed_args, skills): self._skill(ge=skill, infobox=data, parsed_args=parsed_args) except Exception as e: console( - 'Error when parsing skill "{}" at {}:'.format(skill['Id'], skill.rowid), + f"Error when parsing skill \"{skill['Id']}\" at {skill.rowid}:", msg=Msg.error) console(traceback.format_exc(), msg=Msg.error) From acd3c82071d1a54cab0d4bd96acccb09486e8eda Mon Sep 17 00:00:00 2001 From: angelic_knight Date: Sat, 6 Nov 2021 13:51:15 -0500 Subject: [PATCH 11/13] Removing unnecessary general fixes from /patches branch --- PyPoE/cli/exporter/util.py | 2 ++ PyPoE/cli/exporter/wiki/parsers/item.py | 19 ------------------- PyPoE/cli/exporter/wiki/parsers/lua.py | 2 +- PyPoE/cli/exporter/wiki/parsers/skill.py | 9 +++++---- 4 files changed, 8 insertions(+), 24 deletions(-) diff --git a/PyPoE/cli/exporter/util.py b/PyPoE/cli/exporter/util.py index 622543a1..9a74c712 100644 --- a/PyPoE/cli/exporter/util.py +++ b/PyPoE/cli/exporter/util.py @@ -75,6 +75,8 @@ def get_content_path(): def fix_path(path: str) -> str: + #First, replace any double quotes with HTML-encoded quotes + path = path.replace('\"','"') if re.search('[a-zA-Z]:.*', path) is not None: return path[:2] + re.sub(r':', '_', path[2:]) else: diff --git a/PyPoE/cli/exporter/wiki/parsers/item.py b/PyPoE/cli/exporter/wiki/parsers/item.py index 676e2972..fc102f23 100644 --- a/PyPoE/cli/exporter/wiki/parsers/item.py +++ b/PyPoE/cli/exporter/wiki/parsers/item.py @@ -3232,8 +3232,6 @@ def _export(self, parsed_args, items): cls_id = base_item_type['ItemClassesKey']['Id'] m_id = base_item_type['Id'] - self._print_item_rowid(parsed_args, base_item_type) - infobox = OrderedDict() self._process_base_item_type(base_item_type, infobox) self._process_purchase_costs(base_item_type, infobox) @@ -3321,23 +3319,6 @@ def _export(self, parsed_args, items): return r - def _print_item_rowid(self, parsed_args, base_item_type): - #Don't print anything if not running in the rowid mode. - if not set(['start', 'end']).issubset(vars(parsed_args).keys()): - return - - export_row_count = parsed_args.end - parsed_args.start - #If we're printing less than 100 rows, print every rowid - if export_row_count <= 100: - print_granularity = 1 - else: - print_granularity = export_row_count//100 - - item_offset = base_item_type.rowid - parsed_args.start - if (item_offset == 0) or item_offset % print_granularity == 0: - console(f"Processing item with rowid {base_item_type.rowid}: {base_item_type['Name']}") - return - def _format_map_name(self, base_item_type, map_series, language=None): if language is None: language = self._language diff --git a/PyPoE/cli/exporter/wiki/parsers/lua.py b/PyPoE/cli/exporter/wiki/parsers/lua.py index 728fb758..d14074bd 100644 --- a/PyPoE/cli/exporter/wiki/parsers/lua.py +++ b/PyPoE/cli/exporter/wiki/parsers/lua.py @@ -1436,7 +1436,7 @@ def SortCategoryHelper(d): # 'default': '', # }), ('CraftingItemClassCategoriesKeys', { - 'key': 'item_class_categories', + 'key': 'crafting_item_class_categories', 'value': lambda v: [k['Text'] for k in v], }), # ('CraftingBenchUnlockCategoriesKeys', { diff --git a/PyPoE/cli/exporter/wiki/parsers/skill.py b/PyPoE/cli/exporter/wiki/parsers/skill.py index f985bf6a..589f2ab5 100644 --- a/PyPoE/cli/exporter/wiki/parsers/skill.py +++ b/PyPoE/cli/exporter/wiki/parsers/skill.py @@ -84,7 +84,7 @@ def __init__(self, sub_parser): # by row ID s_rid = skill_sub.add_parser( 'by_row', - help='Extract skills by rowid.' + help='Extract areas by rowid.' ) self.add_default_parsers( parser=s_rid, @@ -303,7 +303,7 @@ def skill_stat_filter(self): self._skill_stat_filters.read(self.file_system.get_file( 'Metadata/StatDescriptions/skillpopup_stat_filters.txt' )) - # TODO: remove once fixed + #TODO: remove once fixed #self._skill_stat_filters.skills['spirit_offering'] = SkillEntry(skill_id='spirit_offering', translation_file_path='Metadata/StatDescriptions/offering_skill_stat_descriptions.txt', stats=[]) return self._skill_stat_filters @@ -742,7 +742,7 @@ def export(self, parsed_args, skills): if not parsed_args.allow_skill_gems and skill in \ self.rr['SkillGems.dat'].index['GrantedEffectsKey']: console( - f"Skipping skill gem skill \"{skill['Id']}\" at row {skill.rowid}", + 'Skipping skill gem skill "%s"' % skill['Id'], msg=Msg.warning) continue data = OrderedDict() @@ -751,7 +751,7 @@ def export(self, parsed_args, skills): self._skill(ge=skill, infobox=data, parsed_args=parsed_args) except Exception as e: console( - f"Error when parsing skill \"{skill['Id']}\" at {skill.rowid}:", + 'Error when parsing skill "%s":' % skill['Id'], msg=Msg.error) console(traceback.format_exc(), msg=Msg.error) @@ -759,6 +759,7 @@ def export(self, parsed_args, skills): data=data, cmdargs=parsed_args, ) + r.add_result( text=cond, out_file='skill_%s.txt' % data['skill_id'], From 6176bddbe7c0148a325bdae4914693b20efb04dc Mon Sep 17 00:00:00 2001 From: angelic_knight Date: Sat, 6 Nov 2021 14:23:43 -0500 Subject: [PATCH 12/13] Fixing code style issues from PR review. --- PyPoE/cli/exporter/wiki/parsers/item.py | 3 ++- PyPoE/cli/exporter/wiki/parsers/lua.py | 3 ++- PyPoE/poe/file/psg.py | 2 +- PyPoE/poe/file/translations.py | 6 ++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/PyPoE/cli/exporter/wiki/parsers/item.py b/PyPoE/cli/exporter/wiki/parsers/item.py index fc102f23..b33211a2 100644 --- a/PyPoE/cli/exporter/wiki/parsers/item.py +++ b/PyPoE/cli/exporter/wiki/parsers/item.py @@ -2812,10 +2812,11 @@ def _harvest_plant_booster_extra(self, infobox, base_item_type, row_index=True, ) + _cls_map = dict() ''' This defines the expected data elements for an item class. ''' - _cls_map = { + _cls_map = { # Jewellery 'Amulet': (_type_amulet, ), # Armour types diff --git a/PyPoE/cli/exporter/wiki/parsers/lua.py b/PyPoE/cli/exporter/wiki/parsers/lua.py index d14074bd..48b72e19 100644 --- a/PyPoE/cli/exporter/wiki/parsers/lua.py +++ b/PyPoE/cli/exporter/wiki/parsers/lua.py @@ -507,7 +507,8 @@ def main(self, parsed_args): row, self._COPY_KEYS_BESTIARY_COMPONENTS, components ) if row['BeastRarity'] != RARITY.ANY: - components[-1]['rarity'] = self.rr['ClientStrings.dat'].index['Id']['ItemDisplayString' + row['BeastRarity'].name_upper]['Text'] + display_string = 'ItemDisplayString' + row['BeastRarity'].name_upper + components[-1]['rarity'] = self.rr['ClientStrings.dat'].index['Id'][display_string]['Text'] recipe_components = [] for recipe_id, data in recipe_components_temp.items(): diff --git a/PyPoE/poe/file/psg.py b/PyPoE/poe/file/psg.py index b4f24b9c..1f7e1153 100644 --- a/PyPoE/poe/file/psg.py +++ b/PyPoE/poe/file/psg.py @@ -285,7 +285,7 @@ def _read(self, buffer, *args, **kwargs): root_length = struct.unpack_from(' 1000): raise ValueError( - 'root_length is unrealistically large at {}.\nStopping to prevent allocating too much memory'.format(root_length) + f'root_length is unrealistically large at {root_length}.\nStopping to prevent allocating too much memory' ) offset += 4 diff --git a/PyPoE/poe/file/translations.py b/PyPoE/poe/file/translations.py index ff4cfe0d..54c22aa8 100644 --- a/PyPoE/poe/file/translations.py +++ b/PyPoE/poe/file/translations.py @@ -1052,8 +1052,7 @@ def register_from_string(self, string: str): try: self.index_handlers[handler.id].append(int(args[0])) except ValueError as e: - warnings.warn('Broken quantifier "%s" - Error: %s' % - (string, e.args[0]), TranslationWarning) + warnings.warn(f'Broken quantifier "{string}" - Error: {e.args[0]}', TranslationWarning) elif handler.type == TranslationQuantifier.QuantifierTypes.STRING: self.string_handlers[handler.id] = args else: @@ -1595,8 +1594,7 @@ def _add_translation_hashed(self, translation_id, translation): translation.diff(other) print('')''' - warnings.warn('Duplicate id "%s"' % - translation_id, DuplicateIdentifierWarning) + warnings.warn(f'Duplicate id "{translation_id}"', DuplicateIdentifierWarning) self.translations_hash[translation_id].append(translation) else: self.translations_hash[translation_id] = [translation, ] From 640a1fc3aa566e42dcb63cbcb2e244c63540f37b Mon Sep 17 00:00:00 2001 From: angelic_knight Date: Sun, 7 Nov 2021 08:50:27 -0600 Subject: [PATCH 13/13] Removing outdated hideout doodad code that assumed masters were associated with doodads. --- PyPoE/cli/exporter/wiki/parsers/item.py | 33 ------------------------- 1 file changed, 33 deletions(-) diff --git a/PyPoE/cli/exporter/wiki/parsers/item.py b/PyPoE/cli/exporter/wiki/parsers/item.py index b33211a2..8e711de5 100644 --- a/PyPoE/cli/exporter/wiki/parsers/item.py +++ b/PyPoE/cli/exporter/wiki/parsers/item.py @@ -2392,25 +2392,6 @@ def _currency_extra(self, infobox, base_item_type, currency): function=_currency_extra, ) - _master_hideout_doodad_map = ( - # ('HideoutNPCsKey', { - # 'template': 'master', - # 'format': lambda v: v['Hideout_NPCsKey']['Name'], - # 'condition': lambda v: v is not None, - # }), - # ('MasterLevel', { - # 'template': 'master_level_requirement', - # }), - # ('FavourCost', { - # 'template': 'master_favour_cost', - # }), - ) - - def _apply_master_map(self, infobox, base_item_type, hideout): - if not hideout['IsNonMasterDoodad']: - _apply_column_map(infobox, self._master_hideout_doodad_map, - hideout) - _type_hideout_doodad = _type_factory( data_file='HideoutDoodads.dat', data_mapping=( @@ -2418,26 +2399,12 @@ def _apply_master_map(self, infobox, base_item_type, hideout): 'template': 'is_master_doodad', 'format': lambda v: not v, }), - # ('HideoutNPCsKey', { - # 'template': 'master', - # 'format': lambda v: v['Hideout_NPCsKey']['Name'], - # 'condition': lambda v: v, - # }), - # ('FavourCost', { - # 'template': 'master_favour_cost', - # #'condition': lambda v: v, - # }), - # ('MasterLevel', { - # 'template': 'master_level_requirement', - # #'condition': lambda v: v, - # }), ('Variation_AOFiles', { 'template': 'variation_count', 'format': lambda v: len(v), }), ), row_index=True, - function=_apply_master_map, ) def _maps_extra(self, infobox, base_item_type, maps):