fix: erase_tokens_after not removing some tokens from the LUT

`inter::Interner::erase_tokens_after` did not remove some tokens
from its internal LUT in rare cases which could cause panics or
incorrect interning afterwards.

CHANGELOG.md

@@ -4,7 +4,14 @@ All notable changes to imara-diff will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## 0.1.4 - 2022-10-26
+
+## 0.1.5 - 2022-11-4
+
+### Fixed
+
+* `inter::Interner::erase_tokens_after` not removing tokens from the LUT in some cases.
+
+## 0.1.4 - 2022-11-4
 
 ### Fixed
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "imara-diff"
-version = "0.1.4"
+version = "0.1.5"
 edition = "2021"
 authors = ["pascalkuthe <[email protected]>"]
 rust-version = "1.61"

src/intern.rs

@@ -157,7 +157,7 @@ impl<T: Hash + Eq> Interner<T> {
             }
         } else {
             for (i, token) in self.tokens[retained..].iter().enumerate() {
-                self.table.erase_entry(self.hasher.hash_one(token), |token| token.0 == i as u32);
+                self.table.erase_entry(self.hasher.hash_one(token), |token| token.0  == (retained + i) as u32);
             }
         }
         self.tokens.truncate(first_erased_token.0 as usize);