Updated package versions to eliminate vulnerable and deprecated transitive dependencies

.build/dependencies.props

@@ -32,34 +32,31 @@
         https://github.com/apache/lucene-solr/tree/31d7ec7bbfdcd2c4cc61d9d35e962165410b65fe/lucene/analysis/icu/src/data/utr30
         Just make sure they are adjusted to the right version of ICU/Lucene.
     <ICU4NPackageVersion>[60.1,60.2)</ICU4NPackageVersion> -->
-    <ICU4NPackageVersion>60.1.0-alpha.356</ICU4NPackageVersion>
-    <ICU4NCollationPackageVersion>$(ICU4NPackageVersion)</ICU4NCollationPackageVersion>
-    <ICU4NCurrencyDataPackageVersion>$(ICU4NPackageVersion)</ICU4NCurrencyDataPackageVersion>
-    <ICU4NLanguageDataPackageVersion>$(ICU4NPackageVersion)</ICU4NLanguageDataPackageVersion>
-    <ICU4NRegionDataPackageVersion>$(ICU4NPackageVersion)</ICU4NRegionDataPackageVersion>
-    <ICU4NTransliteratorPackageVersion>$(ICU4NPackageVersion)</ICU4NTransliteratorPackageVersion>
+    <ICU4NPackageVersion>[60.1.0-alpha.436]</ICU4NPackageVersion>
     <IKVMPackageVersion>8.7.5</IKVMPackageVersion>
     <IKVMMavenSdkPackageVersion>1.6.7</IKVMMavenSdkPackageVersion>
-    <J2NPackageVersion>2.0.0</J2NPackageVersion>
+    <!-- J2N will break binary compatibility in 3.0.0 to fix the APIs of collection types -->
+    <J2NPackageVersion>[2.1.0, 3.0.0)</J2NPackageVersion>
     <LiquidTestReportsMarkdownPackageVersion>1.0.9</LiquidTestReportsMarkdownPackageVersion>
-    <MicrosoftAspNetCoreHttpAbstractionsPackageVersion>2.0.0</MicrosoftAspNetCoreHttpAbstractionsPackageVersion>
-    <MicrosoftAspNetCoreTestHostPackageVersion>5.0.0</MicrosoftAspNetCoreTestHostPackageVersion>
-    <MicrosoftAspNetCoreTestHostPackageVersion Condition=" $(TargetFramework.StartsWith('net4')) Or $(TargetFramework.StartsWith('netcoreapp')) ">2.0.0</MicrosoftAspNetCoreTestHostPackageVersion>
+    <MicrosoftAspNetCoreHttpAbstractionsPackageVersion>2.1.1</MicrosoftAspNetCoreHttpAbstractionsPackageVersion>
+    <MicrosoftAspNetCoreHttpPackageVersion>2.1.34</MicrosoftAspNetCoreHttpPackageVersion>
+    <MicrosoftAspNetCoreTestHostPackageVersion>6.0.0</MicrosoftAspNetCoreTestHostPackageVersion>
+    <MicrosoftAspNetCoreTestHostPackageVersion Condition=" $(TargetFramework.StartsWith('net4')) ">2.1.1</MicrosoftAspNetCoreTestHostPackageVersion>
+    <MicrosoftAspNetCoreTestHostPackageVersion Condition=" '$(TargetFramework)' == 'net5.0' ">3.1.32</MicrosoftAspNetCoreTestHostPackageVersion>
     <MicrosoftCodeAnalysisAnalyzersPackageVersion>2.9.8</MicrosoftCodeAnalysisAnalyzersPackageVersion>
     <MicrosoftCodeAnalysisCSharpPackageVersion>2.6.1</MicrosoftCodeAnalysisCSharpPackageVersion>
     <MicrosoftCodeAnalysisCSharpWorkspacesPackageVersion>$(MicrosoftCodeAnalysisCSharpPackageVersion)</MicrosoftCodeAnalysisCSharpWorkspacesPackageVersion>
     <MicrosoftCodeAnalysisVisualBasicPackageVersion>2.6.1</MicrosoftCodeAnalysisVisualBasicPackageVersion>
     <MicrosoftCodeAnalysisVisualBasicWorkspacesPackageVersion>$(MicrosoftCodeAnalysisVisualBasicPackageVersion)</MicrosoftCodeAnalysisVisualBasicWorkspacesPackageVersion>
-    <MicrosoftExtensionsConfigurationPackageVersion>3.1.6</MicrosoftExtensionsConfigurationPackageVersion>
+    <MicrosoftExtensionsConfigurationPackageVersion>8.0.0</MicrosoftExtensionsConfigurationPackageVersion>
     <MicrosoftExtensionsConfigurationAbstractionsPackageVersion>$(MicrosoftExtensionsConfigurationPackageVersion)</MicrosoftExtensionsConfigurationAbstractionsPackageVersion>
     <MicrosoftExtensionsConfigurationCommandLinePackageVersion>$(MicrosoftExtensionsConfigurationPackageVersion)</MicrosoftExtensionsConfigurationCommandLinePackageVersion>
     <MicrosoftExtensionsConfigurationEnvironmentVariablesPackageVersion>$(MicrosoftExtensionsConfigurationPackageVersion)</MicrosoftExtensionsConfigurationEnvironmentVariablesPackageVersion>
-    <MicrosoftExtensionsConfigurationJsonPackageVersion>$(MicrosoftExtensionsConfigurationPackageVersion)</MicrosoftExtensionsConfigurationJsonPackageVersion>
-    <MicrosoftExtensionsConfigurationXmlPackageVersion>$(MicrosoftExtensionsConfigurationPackageVersion)</MicrosoftExtensionsConfigurationXmlPackageVersion>
-    <MicrosoftExtensionsDependencyInjectionAbstractionsPackageVersion>2.0.0</MicrosoftExtensionsDependencyInjectionAbstractionsPackageVersion>
-    <MicrosoftExtensionsDependencyInjectionPackageVersion>$(MicrosoftExtensionsDependencyInjectionAbstractionsPackageVersion)</MicrosoftExtensionsDependencyInjectionPackageVersion>
-    <MicrosoftExtensionsOptionsPackageVersion>2.0.0</MicrosoftExtensionsOptionsPackageVersion>
-    <MicrosoftNETFrameworkReferenceAssembliesPackageReferenceVersion>1.0.2</MicrosoftNETFrameworkReferenceAssembliesPackageReferenceVersion>
+    <MicrosoftExtensionsConfigurationJsonPackageVersion>8.0.1</MicrosoftExtensionsConfigurationJsonPackageVersion>
+    <MicrosoftExtensionsConfigurationXmlPackageVersion>8.0.1</MicrosoftExtensionsConfigurationXmlPackageVersion>
+    <MicrosoftExtensionsDependencyInjectionAbstractionsPackageVersion>8.0.0</MicrosoftExtensionsDependencyInjectionAbstractionsPackageVersion>
+    <MicrosoftExtensionsDependencyInjectionPackageVersion>8.0.1</MicrosoftExtensionsDependencyInjectionPackageVersion>
+    <MicrosoftNETFrameworkReferenceAssembliesPackageReferenceVersion>1.0.3</MicrosoftNETFrameworkReferenceAssembliesPackageReferenceVersion>
     <MicrosoftNETTestSdkPackageVersion>17.9.0</MicrosoftNETTestSdkPackageVersion>
     <MicrosoftSourceLinkAzureReposGitPackageReferenceVersion>1.1.1</MicrosoftSourceLinkAzureReposGitPackageReferenceVersion>
     <MicrosoftSourceLinkGitHubPackageReferenceVersion>$(MicrosoftSourceLinkAzureReposGitPackageReferenceVersion)</MicrosoftSourceLinkGitHubPackageReferenceVersion>
@@ -73,16 +70,18 @@
     <RandomizedTestingGeneratorsPackageVersion>2.7.8</RandomizedTestingGeneratorsPackageVersion>
     <SharpZipLibPackageVersion>1.4.2</SharpZipLibPackageVersion>
     <Spatial4nPackageVersion>0.4.1.1</Spatial4nPackageVersion>
+    <SystemIOPipelinesPackageVersion>8.0.0</SystemIOPipelinesPackageVersion>
     <SystemMemoryPackageVersion>4.5.5</SystemMemoryPackageVersion>
+    <SystemNetHttpPackageVersion>4.3.4</SystemNetHttpPackageVersion>
     <SystemReflectionEmitPackageVersion>4.3.0</SystemReflectionEmitPackageVersion>
     <SystemReflectionEmitILGenerationPackageVersion>4.3.0</SystemReflectionEmitILGenerationPackageVersion>
     <SystemReflectionTypeExtensionsPackageVersion>4.3.0</SystemReflectionTypeExtensionsPackageVersion>
-    <SystemRuntimeCompilerServicesUnsafePackageVersion>6.0.0</SystemRuntimeCompilerServicesUnsafePackageVersion>
     <SystemRuntimeInteropServicesRuntimeInformationPackageVersion>4.3.0</SystemRuntimeInteropServicesRuntimeInformationPackageVersion>
-    <SystemSecurityCryptographyXmlPackageVersion>6.0.1</SystemSecurityCryptographyXmlPackageVersion>
     <SystemTextEncodingCodePagesPackageVersion>4.3.0</SystemTextEncodingCodePagesPackageVersion>
     <SystemTextEncodingCodePagesPackageVersion Condition=" '$(TargetFramework)' == 'net472' ">5.0.0</SystemTextEncodingCodePagesPackageVersion>
-    <SystemTextJsonPackageVersion>6.0.6</SystemTextJsonPackageVersion>
+    <SystemTextEncodingsWebPackageVersion>8.0.0</SystemTextEncodingsWebPackageVersion>
+    <SystemTextJsonPackageVersion>6.0.10</SystemTextJsonPackageVersion>
+    <SystemTextRegularExpressionsPackageVersion>4.3.1</SystemTextRegularExpressionsPackageVersion>
     <TimeZoneConverterPackageVersion>6.1.0</TimeZoneConverterPackageVersion>
   </PropertyGroup>
   <PropertyGroup Label="Maven Package Reference Versions">

Lucene.Net.sln

@@ -119,6 +119,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Lucene.Net", "src\Lucene.Net\Lucene.Net.csproj", "{3A0AA37E-2B7B-4416-B528-DA4E0E6A6706}"
+	ProjectSection(ProjectDependencies) = postProject
+		{441876AF-F691-408C-85EC-6A934E60F627} = {441876AF-F691-408C-85EC-6A934E60F627}
+		{5CD4D4E8-6132-4384-98FC-6AB1C97E0B80} = {5CD4D4E8-6132-4384-98FC-6AB1C97E0B80}
+	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Lucene.Net.Analysis.Common", "src\Lucene.Net.Analysis.Common\Lucene.Net.Analysis.Common.csproj", "{3D0366A8-515D-44F0-835F-4118853CFA14}"
 EndProject
@@ -284,7 +288,7 @@ Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "websites", "websites\", "{8
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "docs", "docs", "{42599646-275F-4970-BC60-A3349F6498CC}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LuceneDocsPlugins", "src\docs\LuceneDocsPlugins\LuceneDocsPlugins.csproj", "{FED4A824-1F32-4948-8D37-2B7610804DB5}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LuceneDocsPlugins", "src\docs\LuceneDocsPlugins\LuceneDocsPlugins.csproj", "{FED4A824-1F32-4948-8D37-2B7610804DB5}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

NuGet.config

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 
 <!--
 Licensed to the Apache Software Foundation (ASF) under one
@@ -21,6 +21,8 @@ under the License.
 
 <configuration>
   <packageSources>
+    <add key="ICU4N preview feed" value="https://www.myget.org/F/icu4n/api/v3/index.json" />
+    <add key="J2N preview feed" value="https://www.myget.org/F/j2n-preview/api/v3/index.json" />
     <add key="NuGet official package source" value="https://api.nuget.org/v3/index.json" />
   </packageSources>
 </configuration>

src/Lucene.Net.Analysis.Common/Analysis/Util/OpenStringBuilder.cs

@@ -28,7 +28,7 @@ namespace Lucene.Net.Analysis.Util
     /// <summary>
     /// A StringBuilder that allows one to access the array.
     /// </summary>
-    public class OpenStringBuilder : IAppendable, ICharSequence
+    public class OpenStringBuilder : IAppendable, ISpanAppendable, ICharSequence // LUCENENET specific - implemented ISpanAppendable to support ReadOnlySpan<char>
     {
         protected char[] m_buf;
         protected int m_len;
@@ -93,6 +93,13 @@ public virtual OpenStringBuilder Append(ICharSequence csq, int startIndex, int c
             return this;
         }
 
+        public virtual OpenStringBuilder Append(ReadOnlySpan<char> value) // LUCENENET specific - added to support ReadOnlySpan<char>
+        {
+            EnsureCapacity(value.Length);
+            UnsafeWrite(value);
+            return this;
+        }
+
         // LUCENENET specific - overload for string (more common in .NET than ICharSequence)
         public virtual OpenStringBuilder Append(string csq)
         {
@@ -192,6 +199,13 @@ public virtual void UnsafeWrite(int b)
             UnsafeWrite((char)b);
         }
 
+        public virtual void UnsafeWrite(ReadOnlySpan<char> b) // LUCENENET specific - added to support ReadOnlySpan<char>
+        {
+            int len = b.Length;
+            b.CopyTo(m_buf.AsSpan(this.m_len, len));
+            this.m_len += len;
+        }
+
         public virtual void UnsafeWrite(char[] b, int off, int len)
         {
             Arrays.Copy(b, off, m_buf, this.m_len, len);
@@ -319,6 +333,23 @@ public override string ToString()
             return new string(m_buf, 0, Length);
         }
 
+        public virtual OpenStringBuilder Remove(int startIndex, int length) // LUCENENET specific - added missing remove method
+        {
+            if (m_len == length && startIndex == 0)
+            {
+                m_len = 0;
+                return this;
+            }
+
+            if (length > 0)
+            {
+                int endIndex = startIndex + length;
+                m_buf.AsSpan(endIndex).CopyTo(m_buf.AsSpan(startIndex));
+                m_len -= length;
+            }
+            return this;
+        }
+
         #region IAppendable Members
 
         IAppendable IAppendable.Append(char value)
@@ -366,6 +397,8 @@ IAppendable IAppendable.Append(ICharSequence value, int startIndex, int count)
             return Append(value, startIndex, count);
         }
 
+        ISpanAppendable ISpanAppendable.Append(ReadOnlySpan<char> value) => Append(value);
+
         #endregion IAppendable Members
     }
-}
+}

src/Lucene.Net.Analysis.ICU/Analysis/Icu/ICUNormalizer2CharFilter.cs

@@ -1,5 +1,6 @@
 // Lucene version compatibility level < 7.1.0
 using J2N;
+using J2N.Text;
 using ICU4N.Text;
 using Lucene.Net.Analysis.CharFilters;
 using Lucene.Net.Analysis.Util;
@@ -37,7 +38,7 @@ namespace Lucene.Net.Analysis.Icu
     public sealed class ICUNormalizer2CharFilter : BaseCharFilter
     {
         private readonly Normalizer2 normalizer;
-        private readonly StringBuilder inputBuffer = new StringBuilder();
+        private readonly OpenStringBuilder inputBuffer = new OpenStringBuilder(); // LUCENENET specific - Using OpenStringBuilder because StringBuilder cannot be directly converted to ReadOnlySpan<char>
         private readonly StringBuilder resultBuffer = new StringBuilder();
 
         private bool inputFinished;
@@ -162,10 +163,10 @@ private int ReadAndNormalizeFromInput()
 
         private int ReadFromInputWhileSpanQuickCheckYes()
         {
-            int end = normalizer.SpanQuickCheckYes(inputBuffer);
+            int end = normalizer.SpanQuickCheckYes(inputBuffer.Array.AsSpan(0, inputBuffer.Length));
             if (end > 0)
             {
-                resultBuffer.Append(inputBuffer.ToString(0, end));
+                resultBuffer.Append(inputBuffer.Array.AsSpan(0, end));
                 inputBuffer.Remove(0, end);
                 checkedInputBoundary = Math.Max(checkedInputBoundary - end, 0);
                 charCount += end;
@@ -212,7 +213,7 @@ private int ReadFromIoNormalizeUptoBoundary()
         private int NormalizeInputUpto(int length)
         {
             int destOrigLen = resultBuffer.Length;
-            normalizer.NormalizeSecondAndAppend(resultBuffer, inputBuffer.ToString(0, length));
+            normalizer.NormalizeSecondAndAppend(resultBuffer, inputBuffer.Array.AsSpan(0, length));
 
             inputBuffer.Remove(0, length);
             checkedInputBoundary = Math.Max(checkedInputBoundary - length, 0);

src/Lucene.Net.Analysis.ICU/Analysis/Icu/ICUNormalizer2Filter.cs

@@ -1,7 +1,9 @@
 // Lucene version compatibility level 7.1.0
 using ICU4N.Text;
 using Lucene.Net.Analysis.TokenAttributes;
+using Lucene.Net.Analysis.Util;
 using Lucene.Net.Support;
+using System;
 using System.Text;
 
 namespace Lucene.Net.Analysis.Icu
@@ -60,7 +62,7 @@ public class ICUNormalizer2Filter : TokenFilter
     {
         private readonly ICharTermAttribute termAtt;
         private readonly Normalizer2 normalizer;
-        private readonly StringBuilder buffer = new StringBuilder();
+        private readonly OpenStringBuilder buffer = new OpenStringBuilder(); // LUCENENET specific - Using OpenStringBuilder because StringBuilder cannot be directly converted to ReadOnlySpan<char>
 
         /// <summary>
         /// Create a new <see cref="ICUNormalizer2Filter"/> that combines NFKC normalization, Case
@@ -88,10 +90,10 @@ public override sealed bool IncrementToken()
         {
             if (m_input.IncrementToken())
             {
-                if (normalizer.QuickCheck(termAtt) != QuickCheckResult.Yes)
+                if (normalizer.QuickCheck(termAtt.Buffer.AsSpan(0, termAtt.Length)) != QuickCheckResult.Yes)
                 {
                     buffer.Length = 0;
-                    normalizer.Normalize(termAtt, buffer);
+                    normalizer.Normalize(termAtt.Buffer.AsSpan(0, termAtt.Length), buffer);
                     termAtt.SetEmpty().Append(buffer);
                 }
                 return true;

src/Lucene.Net.Analysis.ICU/Analysis/Icu/ICUTransformFilter.cs

@@ -2,6 +2,7 @@
 using ICU4N.Text;
 using Lucene.Net.Analysis.TokenAttributes;
 using Lucene.Net.Support;
+using System;
 
 namespace Lucene.Net.Analysis.Icu
 {
@@ -128,7 +129,7 @@ public void SetText(ICharTermAttribute token)
                 this.length = token.Length;
             }
 
-            public int Char32At(int pos) => UTF16.CharAt(buffer, 0, length, pos);
+            public int Char32At(int pos) => UTF16.CharAt(buffer.AsSpan(0, length), pos);
 
             public char this[int pos] => buffer[pos];
 
@@ -139,6 +140,11 @@ public void Copy(int startIndex, int length, int destinationIndex) // LUCENENET:
                 Replace(destinationIndex, destinationIndex - destinationIndex, text, 0, length); // LUCENENET: Corrected length & charsLen
             }
 
+            public void CopyTo(int sourceIndex, Span<char> destination, int count)
+            {
+                buffer.AsSpan(sourceIndex, count).CopyTo(destination);
+            }
+
             public void CopyTo(int sourceIndex, char[] destination, int destinationIndex, int count)
             {
                 Arrays.Copy(buffer, sourceIndex, destination, destinationIndex, count);
@@ -157,6 +163,15 @@ public void Replace(int start, int length, string text) // LUCENENET: Changed 2n
                 token.Length = (this.length = newLength);
             }
 
+            public void Replace(int start, int length, ReadOnlySpan<char> text) // LUCENENET: Changed 2nd parameter from limit to length
+            {
+                int charsLen = text.Length;
+                int newLength = ShiftForReplace(start, length + start, charsLen); // LUCENENET: Changed 2nd parameter to calculate limit
+                // insert the replacement text
+                text.CopyTo(buffer.AsSpan(start, charsLen));
+                token.Length = (this.length = newLength);
+            }
+
             public void Replace(int start, int length, char[] text, int charsStart,
                 int charsLen)
             {

src/Lucene.Net.Analysis.ICU/Analysis/Icu/Segmentation/BreakIteratorWrapper.cs

@@ -1,6 +1,7 @@
 // Lucene version compatibility level 8.6.1
 using ICU4N.Text;
 using J2N;
+using System;
 
 namespace Lucene.Net.Analysis.Icu.Segmentation
 {
@@ -79,7 +80,7 @@ private bool IsEmoji(int current, int next)
         {
             int begin = start + current;
             int end = start + next;
-            int codepoint = UTF16.CharAt(text, 0, end, begin);
+            int codepoint = UTF16.CharAt(text.AsSpan(0, end), begin); // LUCENENET: Checked 2nd argument
             if (EMOJI.Contains(codepoint))
             {
                 if (EMOJI_RK.Contains(codepoint))

src/Lucene.Net.Analysis.ICU/Analysis/Icu/Segmentation/ScriptIterator.cs

@@ -2,6 +2,7 @@
 using ICU4N;
 using ICU4N.Globalization;
 using ICU4N.Text;
+using System;
 
 namespace Lucene.Net.Analysis.Icu.Segmentation
 {
@@ -93,7 +94,7 @@ public bool Next()
 
             while (index < limit)
             {
-                int ch = UTF16.CharAt(text, start, limit, index - start);
+                int ch = UTF16.CharAt(text.AsSpan(start, limit), index - start);
                 int sc = GetScript(ch);
 
                 /*

src/Lucene.Net.Analysis.OpenNLP/Lucene.Net.Analysis.OpenNLP.csproj

@@ -52,6 +52,10 @@
     <PackageReference Include="ICU4N" Version="$(ICU4NPackageVersion)" />
     <PackageReference Include="IKVM" Version="$(IKVMPackageVersion)" />
     <PackageReference Include="IKVM.Maven.Sdk" Version="$(IKVMMavenSdkPackageVersion)" />
+
+    <!-- This is a transitive dependency of IKVM, but the version it references is vulnerable. We are currently blocked from upgrading
+        IKVM due to the Azure DevOps limitation of 10GB for a single build agent, so we have to add this reference instead. -->
+    <PackageReference Include="System.Text.Json" Version="$(SystemTextJsonPackageVersion)" />
   </ItemGroup>
 
   <ItemGroup>

src/Lucene.Net.Benchmark/Lucene.Net.Benchmark.csproj

@@ -53,7 +53,7 @@
 
   <ItemGroup>
     <PackageReference Include="J2N" Version="$(J2NPackageVersion)" />
-    <PackageReference Include="ICU4N.Collation" Version="$(ICU4NCollationPackageVersion)" />
+    <PackageReference Include="ICU4N" Version="$(ICU4NPackageVersion)" />
     <PackageReference Include="SharpZipLib" Version="$(SharpZipLibPackageVersion)" />
     <PackageReference Include="Spatial4n" Version="$(Spatial4nPackageVersion)" />
   </ItemGroup>

src/Lucene.Net.Expressions/Lucene.Net.Expressions.csproj

@@ -46,6 +46,10 @@
 
   <ItemGroup>
     <PackageReference Include="Antlr3.Runtime" Version="$(Antlr3RuntimePackageVersion)" />
+
+    <!-- These are transitive dependencies, but Antlr3.Runtime doesn't provide a secure alternative on netstandard2.0, so we need to include them. -->
+    <PackageReference Include="System.Net.Http" Version="$(SystemNetHttpPackageVersion)" />
+    <PackageReference Include="System.Text.RegularExpressions" Version="$(SystemTextRegularExpressionsPackageVersion)" />
   </ItemGroup>
 
   <ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.1' ">