diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 3dd46c084d..c1cd5da1e2 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -1,7 +1,7 @@ - 3.0.54013 - 3.0.54013.0 + 3.0.54016 + 3.0.54016.0 latest embedded diff --git a/src/Lucene.Net/Index/ArrayHolder.cs b/src/Lucene.Net/Index/ArrayHolder.cs index 392f21488b..a0c99d6d19 100644 --- a/src/Lucene.Net/Index/ArrayHolder.cs +++ b/src/Lucene.Net/Index/ArrayHolder.cs @@ -1,51 +1,40 @@ using System; -using System.Buffers; using System.Threading; using Lucene.Net.Store; +using Lucene.Net.Util; namespace Lucene.Net.Index { public class ArrayHolder : IDisposable { - private readonly int _size; private readonly Directory _directory; private readonly string _name; - private readonly long[] _longArray; - private readonly TermInfo[] _termInfoArray; + private readonly IArray _indexPointers; + private readonly IArray _termInfos; private readonly UnmanagedIndexTerms _unmanagedIndexTerms; private int _usages; private long _managedAllocations; - public Span LongArray => _longArray.AsSpan(0, _size); - public Span InfoArray => _termInfoArray.AsSpan(0, _size); + public IArray IndexPointers => _indexPointers; + public IArray TermInfos => _termInfos; public UnmanagedIndexTerms UnmanagedIndexTerms => _unmanagedIndexTerms; - public int ActualArraySize => _longArray.Length; public static Action OnArrayHolderCreated; public static Action OnArrayHolderDisposed; - public const int ArrayPoolThreshold = 128 * 1024; + public long TotalManagedAllocations => _indexPointers.TotalManagedAllocations + _termInfos.TotalManagedAllocations + _unmanagedIndexTerms.TotalManagedAllocations; - public ArrayHolder(int size, Directory directory, string name) + public ArrayHolder(int size, Directory directory, string name, FieldInfos fieldInfos) { - _size = size; _directory = directory; _name = name; - if (size > ArrayPoolThreshold) - { - _longArray = new long[size]; - _termInfoArray = new TermInfo[size]; - } - else - { - _longArray = ArrayPool.Shared.Rent(size); - _termInfoArray = ArrayPool.Shared.Rent(size); - } + _indexPointers = HybridArray.Create(size, UnmanagedStringArray.Type.TermCache, clear: false); + _termInfos = HybridArray.Create(size, UnmanagedStringArray.Type.TermCache, clear: false); - _unmanagedIndexTerms = new UnmanagedIndexTerms(size); + _unmanagedIndexTerms = new UnmanagedIndexTerms(size, fieldInfos); } public void AddRef() @@ -67,20 +56,20 @@ public static ArrayHolder GenerateArrayHolder(Directory directory, string name, { int indexSize = 1 + ((int)indexEnum.size - 1) / indexDivisor; // otherwise read index - var holder = new ArrayHolder(indexSize, directory, name); + var holder = new ArrayHolder(indexSize, directory, name, fieldInfos); for (int i = 0; indexEnum.Next(state); i++) { - holder.UnmanagedIndexTerms.Add(i, indexEnum.Field, indexEnum.TextAsSpan); - holder.InfoArray[i] = indexEnum.TermInfo(); - holder.LongArray[i] = indexEnum.indexPointer; + holder.UnmanagedIndexTerms.Add(i, indexEnum.FieldNumber, indexEnum.TextAsSpan); + holder.TermInfos[i] = indexEnum.TermInfo(); + holder.IndexPointers[i] = indexEnum.indexPointer; for (int j = 1; j < indexDivisor; j++) if (!indexEnum.Next(state)) break; } - holder._managedAllocations = (holder.ActualArraySize * (TermInfo.SizeOf + sizeof(long))); + holder._managedAllocations = holder.TotalManagedAllocations; OnArrayHolderCreated?.Invoke(holder._managedAllocations); @@ -98,18 +87,13 @@ public void Dispose() { GC.SuppressFinalize(this); - OnArrayHolderDisposed?.Invoke(_managedAllocations); - - _unmanagedIndexTerms?.Dispose(); - - if (_size > ArrayPoolThreshold) - return; - - if (_longArray != null) - ArrayPool.Shared.Return(_longArray); + using (_unmanagedIndexTerms) + using (_indexPointers) + using (_termInfos) + { + } - if (_termInfoArray != null) - ArrayPool.Shared.Return(_termInfoArray); + OnArrayHolderDisposed?.Invoke(_managedAllocations); } ~ArrayHolder() diff --git a/src/Lucene.Net/Index/SegmentTermEnum.cs b/src/Lucene.Net/Index/SegmentTermEnum.cs index d220d8fc78..0e993b5ca6 100644 --- a/src/Lucene.Net/Index/SegmentTermEnum.cs +++ b/src/Lucene.Net/Index/SegmentTermEnum.cs @@ -125,11 +125,11 @@ public System.Object Clone(IState state) return clone; } - internal void Seek(long pointer, long p, Term t, TermInfo ti, IState state) + internal void Seek(long pointer, long p, (string Field, UnmanagedStringArray.UnmanagedString Text) tuple, TermInfo ti, IState state) { input.Seek(pointer, state); position = p; - termBuffer.Set(t); + termBuffer.Set(tuple); prevBuffer.Reset(); termInfo.Set(ti); } @@ -202,6 +202,11 @@ public string Field get { return termBuffer.Field; } } + public int FieldNumber + { + get { return termBuffer.FieldNumber; } + } + public Span TextAsSpan { get { return termBuffer.TextAsSpan; } diff --git a/src/Lucene.Net/Index/Term.cs b/src/Lucene.Net/Index/Term.cs index 101f7e7cf9..1f8d9be345 100644 --- a/src/Lucene.Net/Index/Term.cs +++ b/src/Lucene.Net/Index/Term.cs @@ -17,6 +17,7 @@ using System; using System.Runtime.CompilerServices; +using Lucene.Net.Util; using StringHelper = Lucene.Net.Util.StringHelper; namespace Lucene.Net.Index @@ -141,12 +142,12 @@ public int CompareTo(Term other) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public int CompareTo(UnmanagedTerm other) + public int CompareTo((string Field, UnmanagedStringArray.UnmanagedString Text) tuple) { - if (ReferenceEquals(field, other.Field)) - return -other.Text.CompareTo(text); + if (ReferenceEquals(field, tuple.Field)) + return -tuple.Text.CompareTo(text); - return String.CompareOrdinal(field, other.Field); + return String.CompareOrdinal(field, tuple.Field); } ///// Resets the field and text of a Term. diff --git a/src/Lucene.Net/Index/TermBuffer.cs b/src/Lucene.Net/Index/TermBuffer.cs index 69b5b38244..9fcc21253c 100644 --- a/src/Lucene.Net/Index/TermBuffer.cs +++ b/src/Lucene.Net/Index/TermBuffer.cs @@ -15,9 +15,10 @@ * limitations under the License. */ -using System; using Lucene.Net.Store; using Lucene.Net.Support; +using Lucene.Net.Util; +using System; using IndexInput = Lucene.Net.Store.IndexInput; using UnicodeUtil = Lucene.Net.Util.UnicodeUtil; @@ -28,6 +29,7 @@ public sealed class TermBuffer : System.ICloneable { private System.String field; + private int fieldNumber = -1; private Term term; // cached private bool preUTF8Strings; // true if strings are stored in modified UTF8 encoding (LUCENE-510) private bool dirty; // true if text was set externally (ie not read via UTF8 bytes) @@ -37,6 +39,7 @@ public sealed class TermBuffer : System.ICloneable public Span TextAsSpan => new Span(text.result, 0, text.length); public string Field => field; + public int FieldNumber => fieldNumber; public int CompareTo(TermBuffer other) { @@ -71,7 +74,7 @@ internal void SetPreUTF8Strings() preUTF8Strings = true; } - public void Read(IndexInput input, FieldInfos fieldInfos, IState state) + public void Read(IndexInput input, FieldInfos fieldInfos, IState state) { this.term = null; // invalidate cache int start = input.ReadVInt(state); @@ -102,10 +105,12 @@ public void Read(IndexInput input, FieldInfos fieldInfos, IState state) UnicodeUtil.UTF8toUTF16(bytes.result, start, length, text); } } - this.field = fieldInfos.FieldName(input.ReadVInt(state)); - } + + this.fieldNumber = input.ReadVInt(state); + this.field = fieldInfos.FieldName(fieldNumber); + } - public void Set(Term term) + public void Set(Term term) { if (term == null) { @@ -121,17 +126,28 @@ public void Set(Term term) this.term = term; } - public void Set(TermBuffer other) + public void Set((string Field, UnmanagedStringArray.UnmanagedString Text) tuple) + { + text.SetLength(tuple.Text.Size); + TextSupport.GetCharsFromUnmanagedString(tuple.Text, text.result); + dirty = true; + field = tuple.Field; + term = null; + } + + public void Set(TermBuffer other) { text.CopyText(other.text); dirty = true; field = other.field; + fieldNumber = other.fieldNumber; term = other.term; } public void Reset() { field = null; + fieldNumber = -1; text.SetLength(0); term = null; dirty = true; diff --git a/src/Lucene.Net/Index/TermInfo.cs b/src/Lucene.Net/Index/TermInfo.cs index 1c9fa96fc6..8a87089df9 100644 --- a/src/Lucene.Net/Index/TermInfo.cs +++ b/src/Lucene.Net/Index/TermInfo.cs @@ -21,8 +21,6 @@ namespace Lucene.Net.Index public struct TermInfo { - public static int SizeOf = sizeof(int) + sizeof(long) + sizeof(long) + sizeof(int); - /// The number of documents which contain the term. internal int docFreq; diff --git a/src/Lucene.Net/Index/TermInfosReader.cs b/src/Lucene.Net/Index/TermInfosReader.cs index 6d60ba45f6..03f216f099 100644 --- a/src/Lucene.Net/Index/TermInfosReader.cs +++ b/src/Lucene.Net/Index/TermInfosReader.cs @@ -36,7 +36,6 @@ sealed class TermInfosReader : IDisposable { private readonly Directory directory; private readonly String segment; - private readonly FieldInfos fieldInfos; private bool isDisposed; @@ -44,10 +43,6 @@ sealed class TermInfosReader : IDisposable private readonly SegmentTermEnum origEnum; private readonly long size; - private UnmanagedIndexTerms unmanagedIndexTerms => _termsIndexCache.UnmanagedIndexTerms; - private Span indexInfos => _termsIndexCache.InfoArray; - private Span indexPointers =>_termsIndexCache.LongArray; - private readonly int totalIndexInterval; private const int DEFAULT_CACHE_SIZE = 1024; @@ -101,16 +96,15 @@ internal TermInfosReader(Directory dir, System.String seg, FieldInfos fis, int r { directory = dir; segment = seg; - fieldInfos = fis; - origEnum = new SegmentTermEnum(directory.OpenInput(segment + "." + IndexFileNames.TERMS_EXTENSION, readBufferSize, state), fieldInfos, false, state); + origEnum = new SegmentTermEnum(directory.OpenInput(segment + "." + IndexFileNames.TERMS_EXTENSION, readBufferSize, state), fis, false, state); size = origEnum.size; if (indexDivisor != - 1) { // Load terms index totalIndexInterval = origEnum.indexInterval * indexDivisor; - _termsIndexCache = directory.GetCache(segment + "." + IndexFileNames.TERMS_INDEX_EXTENSION, fieldInfos, readBufferSize, indexDivisor, state); + _termsIndexCache = directory.GetCache(segment + "." + IndexFileNames.TERMS_INDEX_EXTENSION, fis, readBufferSize, indexDivisor, state); _termsIndexCache.AddRef(); } else @@ -197,7 +191,7 @@ private ThreadResources GetThreadResources(IState state) private unsafe int GetIndexOffset(Term term) { int lo = 0; // binary search unmanagedIndexTerms[] - int hi = unmanagedIndexTerms.Length - 1; + int hi = _termsIndexCache.UnmanagedIndexTerms.Length - 1; byte[] arr = null; Span stringAsBytes; @@ -223,7 +217,7 @@ private unsafe int GetIndexOffset(Term term) while (hi >= lo) { int mid = Number.URShift((lo + hi), 1); - int delta = CompareTerms(term.Field, stringAsBytes, stringAsSpan, unmanagedIndexTerms[mid]); + int delta = CompareTerms(term.Field, stringAsBytes, stringAsSpan, _termsIndexCache.UnmanagedIndexTerms[mid]); if (delta < 0) hi = mid - 1; else if (delta > 0) @@ -241,12 +235,12 @@ private unsafe int GetIndexOffset(Term term) } } - private static int CompareTerms(string field, Span stringAsBytes, ReadOnlySpan stringAsChar, UnmanagedTerm unmanagedTerm) + private static int CompareTerms(string field, Span stringAsBytes, ReadOnlySpan stringAsChar, (string Field, UnmanagedString Text) tuple) { - if (ReferenceEquals(field, unmanagedTerm.Field)) - return UnmanagedString.CompareOrdinal(stringAsBytes, stringAsChar, unmanagedTerm.Text); + if (ReferenceEquals(field, tuple.Field)) + return UnmanagedString.CompareOrdinal(stringAsBytes, stringAsChar, tuple.Text); - return String.CompareOrdinal(field, unmanagedTerm.Field); + return String.CompareOrdinal(field, tuple.Field); } internal static Term DeepCopyOf(Term other) @@ -257,8 +251,8 @@ internal static Term DeepCopyOf(Term other) } private void SeekEnum(SegmentTermEnum enumerator, int indexOffset, IState state) - { - enumerator.Seek(indexPointers[indexOffset], ((long)indexOffset * totalIndexInterval) - 1, unmanagedIndexTerms[indexOffset].ToTerm(), indexInfos[indexOffset], state); + { + enumerator.Seek(_termsIndexCache.IndexPointers[indexOffset], ((long)indexOffset * totalIndexInterval) - 1, _termsIndexCache.UnmanagedIndexTerms[indexOffset], _termsIndexCache.TermInfos[indexOffset], state); } /// Returns the TermInfo for a Term in the set, or null. @@ -295,7 +289,7 @@ private TermInfo Get(Term term, bool useCache, IState state) if (enumerator.Term != null && ((enumerator.Prev() != null && term.CompareTo(enumerator.Prev()) > 0) || term.CompareTo(enumerator.Term) >= 0)) { int enumOffset = (int) (enumerator.position / totalIndexInterval) + 1; - if (unmanagedIndexTerms.Length == enumOffset || term.CompareTo(unmanagedIndexTerms[enumOffset]) < 0) + if (_termsIndexCache.UnmanagedIndexTerms.Length == enumOffset || term.CompareTo(_termsIndexCache.UnmanagedIndexTerms[enumOffset]) < 0) { // no need to seek @@ -340,9 +334,9 @@ private TermInfo Get(Term term, bool useCache, IState state) return ti; } - private void EnsureIndexIsRead() + private void EnsureIndexIsRead() { - if (unmanagedIndexTerms == null) + if (_termsIndexCache.UnmanagedIndexTerms == null) { throw new SystemException("terms index was not loaded when this reader was created"); } diff --git a/src/Lucene.Net/Index/UnmanagedIndexTerms.cs b/src/Lucene.Net/Index/UnmanagedIndexTerms.cs index 91dff7a631..16e2880fd6 100644 --- a/src/Lucene.Net/Index/UnmanagedIndexTerms.cs +++ b/src/Lucene.Net/Index/UnmanagedIndexTerms.cs @@ -1,43 +1,42 @@ -using System; -using System.Buffers; using Lucene.Net.Util; +using System; namespace Lucene.Net.Index { public class UnmanagedIndexTerms : IDisposable { - private readonly int _size; - private readonly string[] _fields; // fields are interned + private readonly FieldInfos _fieldInfos; + private readonly IArray _fieldNumber; private readonly UnmanagedStringArray _text; public int Length => _text.Length; - public UnmanagedIndexTerms(int size) + public long TotalManagedAllocations => _text.TotalManagedAllocations + _fieldNumber.TotalManagedAllocations; + + public UnmanagedIndexTerms(int size, FieldInfos fieldInfos) { - _size = size; - _fields = size > ArrayHolder.ArrayPoolThreshold ? new string[size] : ArrayPool.Shared.Rent(size); - _text = new UnmanagedStringArray(size, 0, UnmanagedStringArray.Type.TermCache); + _fieldInfos = fieldInfos; + _fieldNumber = HybridArray.Create(size, UnmanagedStringArray.Type.TermCache, clear: false); + _text = new UnmanagedStringArray(size, 0, UnmanagedStringArray.Type.TermCache, clear: false); } - public void Add(int index, string field, Span textAsSpan) + public void Add(int index, int fieldNumber, Span textAsSpan) { - _fields[index] = field; + _fieldNumber[index] = fieldNumber; _text.Add(textAsSpan); } - public UnmanagedTerm this[int position] + public (string Field, UnmanagedStringArray.UnmanagedString Text) this[int position] { - get => new UnmanagedTerm(_fields[position], _text[position]); + get => (_fieldInfos.FieldName(_fieldNumber[position]), _text[position]); } public void Dispose() { - _text?.Dispose(); - - if (_size > ArrayHolder.ArrayPoolThreshold || _fields == null) - return; - - ArrayPool.Shared.Return(_fields, clearArray: true); + using (_fieldNumber) + using (_text) + { + } } } } diff --git a/src/Lucene.Net/Index/UnmanagedTerm.cs b/src/Lucene.Net/Index/UnmanagedTerm.cs deleted file mode 100644 index 30e4b70874..0000000000 --- a/src/Lucene.Net/Index/UnmanagedTerm.cs +++ /dev/null @@ -1,25 +0,0 @@ -using Lucene.Net.Util; - -namespace Lucene.Net.Index -{ - public class UnmanagedTerm - { - private readonly string field; - private readonly UnmanagedStringArray.UnmanagedString unmanagedString; - - public string Field => field; - - public UnmanagedStringArray.UnmanagedString Text => unmanagedString; - - public UnmanagedTerm(string field, UnmanagedStringArray.UnmanagedString unmanagedString) - { - this.field = field; - this.unmanagedString = unmanagedString; - } - - public Term ToTerm() - { - return new Term(field, unmanagedString.ToString(), false); - } - } -} \ No newline at end of file diff --git a/src/Lucene.Net/Search/FieldCache.cs b/src/Lucene.Net/Search/FieldCache.cs index 91f5d57144..de020e2790 100644 --- a/src/Lucene.Net/Search/FieldCache.cs +++ b/src/Lucene.Net/Search/FieldCache.cs @@ -90,10 +90,10 @@ public virtual unsafe int BinarySearchLookup(System.String key) public UnmanagedStringArray lookup; /// For each document, an index into the lookup array. - public int[] order; + public IArray order; /// Creates one of these objects - public StringIndex(int[] values, UnmanagedStringArray lookup) + public StringIndex(IArray values, UnmanagedStringArray lookup) { this.order = values; this.lookup = lookup; @@ -544,7 +544,7 @@ public interface FieldCache /// The values in the given field for each document. /// /// java.io.IOException If any error occurs. - long[] GetLongs(IndexReader reader, System.String field, IState state); + IArray GetLongs(IndexReader reader, System.String field, IState state); /// Checks the internal cache for an appropriate entry, and if none is found, /// reads the terms in field as longs and returns an array of @@ -561,7 +561,7 @@ public interface FieldCache /// The values in the given field for each document. /// /// IOException If any error occurs. - long[] GetLongs(IndexReader reader, System.String field, LongParser parser, IState state); + IArray GetLongs(IndexReader reader, System.String field, LongParser parser, IState state); /// Checks the internal cache for an appropriate entry, and if none is @@ -577,7 +577,7 @@ public interface FieldCache /// The values in the given field for each document. /// /// IOException If any error occurs. - double[] GetDoubles(IndexReader reader, System.String field, IState state); + IArray GetDoubles(IndexReader reader, System.String field, IState state); /// Checks the internal cache for an appropriate entry, and if none is found, /// reads the terms in field as doubles and returns an array of @@ -594,7 +594,7 @@ public interface FieldCache /// The values in the given field for each document. /// /// IOException If any error occurs. - double[] GetDoubles(IndexReader reader, System.String field, DoubleParser parser, IState state); + IArray GetDoubles(IndexReader reader, System.String field, DoubleParser parser, IState state); /// Checks the internal cache for an appropriate entry, and if none /// is found, reads the term values in field and returns an array diff --git a/src/Lucene.Net/Search/FieldCacheImpl.cs b/src/Lucene.Net/Search/FieldCacheImpl.cs index 222557e501..4b2d428e5f 100644 --- a/src/Lucene.Net/Search/FieldCacheImpl.cs +++ b/src/Lucene.Net/Search/FieldCacheImpl.cs @@ -17,22 +17,16 @@ using System; using System.Collections.Concurrent; -using System.Collections.Generic; using System.IO; -using System.Linq; using Lucene.Net.Index; using System.Runtime.CompilerServices; -using System.Threading; using Lucene.Net.Store; -using Lucene.Net.Support; using Lucene.Net.Util; -using NumericField = Lucene.Net.Documents.NumericField; using IndexReader = Lucene.Net.Index.IndexReader; using Term = Lucene.Net.Index.Term; using TermDocs = Lucene.Net.Index.TermDocs; using TermEnum = Lucene.Net.Index.TermEnum; using FieldCacheSanityChecker = Lucene.Net.Util.FieldCacheSanityChecker; -using Single = Lucene.Net.Support.Single; using StringHelper = Lucene.Net.Util.StringHelper; namespace Lucene.Net.Search @@ -82,9 +76,23 @@ public virtual IDisposable PurgeAllCaches() // When the GC will run, the finalizer of the Segments will be executed and release the unmanaged memory. // We'll return the old StringIndexCache and let the caller decide if he wants to dispose it sooner - var copy = _stringIndexCache; + var disposable = new DisposableAction(_stringIndexCache, _longCache, _doubleCache); + Init(); - return copy; + + return disposable; + } + } + + private sealed class DisposableAction(StringIndexCache stringIndexCache, LongCache longCache, DoubleCache doubleCache) : IDisposable + { + public void Dispose() + { + using (stringIndexCache) + using (longCache) + using (doubleCache) + { + } } } @@ -582,24 +590,24 @@ protected internal override float[] CreateValue(IndexReader reader, Entry entryK - public virtual long[] GetLongs(IndexReader reader, System.String field, IState state) + public virtual IArray GetLongs(IndexReader reader, System.String field, IState state) { return GetLongs(reader, field, null, state); } // inherit javadocs - public virtual long[] GetLongs(IndexReader reader, System.String field, Lucene.Net.Search.LongParser parser, IState state) + public virtual IArray GetLongs(IndexReader reader, System.String field, Lucene.Net.Search.LongParser parser, IState state) { return _longCache.Get(reader, new Entry(field, parser), state); } - internal sealed class LongCache:Cache + internal sealed class LongCache : Cache>, IDisposable { internal LongCache(FieldCache wrapper):base(wrapper) { } - protected internal override long[] CreateValue(IndexReader reader, Entry entryKey, IState state) + protected internal override IArray CreateValue(IndexReader reader, Entry entryKey, IState state) { Entry entry = entryKey; System.String field = entry.field; @@ -615,9 +623,11 @@ protected internal override long[] CreateValue(IndexReader reader, Entry entryKe return wrapper.GetLongs(reader, field, Lucene.Net.Search.FieldCache_Fields.NUMERIC_UTILS_LONG_PARSER, state); } } - long[] retArray = null; + + var retArray = HybridArray.Create(reader.MaxDoc, UnmanagedStringArray.Type.Sorting, clear: true); TermDocs termDocs = reader.TermDocs(state); TermEnum termEnum = reader.Terms(new Term(field), state); + try { do @@ -626,9 +636,6 @@ protected internal override long[] CreateValue(IndexReader reader, Entry entryKe if (term == null || (System.Object) term.Field != (System.Object) field) break; long termval = parser.ParseLong(term.Text); - if (retArray == null) - // late init - retArray = new long[reader.MaxDoc]; termDocs.Seek(termEnum, state); while (termDocs.Next(state)) { @@ -645,33 +652,45 @@ protected internal override long[] CreateValue(IndexReader reader, Entry entryKe termDocs.Close(); termEnum.Close(); } - if (retArray == null) - // no values - retArray = new long[reader.MaxDoc]; + return retArray; } + + public void Dispose() + { + foreach (var keyValue in readerCache) + { + foreach (var keyValuePair in keyValue.Value) + { + if (keyValuePair.Value is IArray array) + { + array.Dispose(); + } + } + } + } } // inherit javadocs - public virtual double[] GetDoubles(IndexReader reader, System.String field, IState state) + public virtual IArray GetDoubles(IndexReader reader, System.String field, IState state) { return GetDoubles(reader, field, null, state); } // inherit javadocs - public virtual double[] GetDoubles(IndexReader reader, System.String field, Lucene.Net.Search.DoubleParser parser, IState state) + public virtual IArray GetDoubles(IndexReader reader, System.String field, Lucene.Net.Search.DoubleParser parser, IState state) { return _doubleCache.Get(reader, new Entry(field, parser), state); } - internal sealed class DoubleCache:Cache + internal sealed class DoubleCache : Cache>, IDisposable { internal DoubleCache(FieldCache wrapper):base(wrapper) { } - protected internal override double[] CreateValue(IndexReader reader, Entry entryKey, IState state) + protected internal override IArray CreateValue(IndexReader reader, Entry entryKey, IState state) { Entry entry = entryKey; System.String field = entry.field; @@ -687,9 +706,11 @@ protected internal override double[] CreateValue(IndexReader reader, Entry entry return wrapper.GetDoubles(reader, field, Lucene.Net.Search.FieldCache_Fields.NUMERIC_UTILS_DOUBLE_PARSER, state); } } - double[] retArray = null; + + var retArray = HybridArray.Create(reader.MaxDoc, UnmanagedStringArray.Type.Sorting, clear: true); TermDocs termDocs = reader.TermDocs(state); TermEnum termEnum = reader.Terms(new Term(field), state); + try { do @@ -698,9 +719,6 @@ protected internal override double[] CreateValue(IndexReader reader, Entry entry if (term == null || (System.Object) term.Field != (System.Object) field) break; double termval = parser.ParseDouble(term.Text); - if (retArray == null) - // late init - retArray = new double[reader.MaxDoc]; termDocs.Seek(termEnum, state); while (termDocs.Next(state)) { @@ -717,11 +735,23 @@ protected internal override double[] CreateValue(IndexReader reader, Entry entry termDocs.Close(); termEnum.Close(); } - if (retArray == null) - // no values - retArray = new double[reader.MaxDoc]; + return retArray; } + + public void Dispose() + { + foreach (var keyValue in readerCache) + { + foreach (var keyValuePair in keyValue.Value) + { + if (keyValuePair.Value is IArray array) + { + array.Dispose(); + } + } + } + } } @@ -784,10 +814,12 @@ internal StringIndexCache(FieldCache wrapper):base(wrapper) protected internal override StringIndex CreateValue(IndexReader reader, Entry entryKey, IState state) { System.String field = StringHelper.Intern(entryKey.field); - int[] retArray = new int[reader.MaxDoc]; + var retArray = HybridArray.Create(reader.MaxDoc, UnmanagedStringArray.Type.Sorting, clear: true); var length = reader.MaxDoc + 1; - UnmanagedStringArray mterms = new UnmanagedStringArray(length, 1, UnmanagedStringArray.Type.Sorting); + UnmanagedStringArray mterms = new UnmanagedStringArray(length, 1, UnmanagedStringArray.Type.Sorting, clear: false); + mterms.SetAsNull(0); + TermDocs termDocs = reader.TermDocs(state); SegmentTermEnum termEnum = (SegmentTermEnum)reader.Terms(new Term(field), state); @@ -798,7 +830,7 @@ protected internal override StringIndex CreateValue(IndexReader reader, Entry en // this puts them at the top - if it is changed, FieldDocSortedHitQueue // needs to change as well. termNumber++; - + try { do @@ -848,6 +880,7 @@ public void Dispose() if (keyValuePair.Value is StringIndex si) { si.lookup.Dispose(); + si.order.Dispose(); } } } diff --git a/src/Lucene.Net/Search/FieldCacheRangeFilter.cs b/src/Lucene.Net/Search/FieldCacheRangeFilter.cs index cd9b7ace35..579644a491 100644 --- a/src/Lucene.Net/Search/FieldCacheRangeFilter.cs +++ b/src/Lucene.Net/Search/FieldCacheRangeFilter.cs @@ -19,6 +19,7 @@ using Lucene.Net.Index; using Lucene.Net.Store; using Lucene.Net.Support; +using Lucene.Net.Util; using NumericField = Lucene.Net.Documents.NumericField; using IndexReader = Lucene.Net.Index.IndexReader; using TermDocs = Lucene.Net.Index.TermDocs; @@ -387,14 +388,14 @@ private class AnonymousClassFieldCacheRangeFilter4 : FieldCacheRangeFilter enclosingInstance) + private void InitBlock(IArray values, long inclusiveLowerPoint, long inclusiveUpperPoint, FieldCacheRangeFilter enclosingInstance) { this.values = values; this.inclusiveLowerPoint = inclusiveLowerPoint; this.inclusiveUpperPoint = inclusiveUpperPoint; this.enclosingInstance = enclosingInstance; } - private long[] values; + private IArray values; private long inclusiveLowerPoint; private long inclusiveUpperPoint; private FieldCacheRangeFilter enclosingInstance; @@ -406,7 +407,7 @@ public FieldCacheRangeFilter Enclosing_Instance } } - internal AnonymousClassFieldCacheDocIdSet(long[] values, long inclusiveLowerPoint, long inclusiveUpperPoint, FieldCacheRangeFilter enclosingInstance, Lucene.Net.Index.IndexReader Param1, bool Param2) + internal AnonymousClassFieldCacheDocIdSet(IArray values, long inclusiveLowerPoint, long inclusiveUpperPoint, FieldCacheRangeFilter enclosingInstance, Lucene.Net.Index.IndexReader Param1, bool Param2) : base(Param1, Param2) { InitBlock(values, inclusiveLowerPoint, inclusiveUpperPoint, enclosingInstance); @@ -450,7 +451,7 @@ public override DocIdSet GetDocIdSet(IndexReader reader, IState state) if (inclusiveLowerPoint > inclusiveUpperPoint) return DocIdSet.EMPTY_DOCIDSET; - long[] values = Lucene.Net.Search.FieldCache_Fields.DEFAULT.GetLongs(reader, field, (Lucene.Net.Search.LongParser)parser, state); + IArray values = Lucene.Net.Search.FieldCache_Fields.DEFAULT.GetLongs(reader, field, (Lucene.Net.Search.LongParser)parser, state); // we only request the usage of termDocs, if the range contains 0 return new AnonymousClassFieldCacheDocIdSet(values, inclusiveLowerPoint, inclusiveUpperPoint, this, reader, (inclusiveLowerPoint <= 0L && inclusiveUpperPoint >= 0L)); } @@ -541,14 +542,14 @@ private class AnonymousClassFieldCacheRangeFilter6 : FieldCacheRangeFilter enclosingInstance) + private void InitBlock(IArray values, double inclusiveLowerPoint, double inclusiveUpperPoint, FieldCacheRangeFilter enclosingInstance) { this.values = values; this.inclusiveLowerPoint = inclusiveLowerPoint; this.inclusiveUpperPoint = inclusiveUpperPoint; this.enclosingInstance = enclosingInstance; } - private double[] values; + private IArray values; private double inclusiveLowerPoint; private double inclusiveUpperPoint; private FieldCacheRangeFilter enclosingInstance; @@ -560,7 +561,7 @@ public FieldCacheRangeFilter Enclosing_Instance } } - internal AnonymousClassFieldCacheDocIdSet(double[] values, double inclusiveLowerPoint, double inclusiveUpperPoint, FieldCacheRangeFilter enclosingInstance, Lucene.Net.Index.IndexReader Param1, bool Param2) + internal AnonymousClassFieldCacheDocIdSet(IArray values, double inclusiveLowerPoint, double inclusiveUpperPoint, FieldCacheRangeFilter enclosingInstance, Lucene.Net.Index.IndexReader Param1, bool Param2) : base(Param1, Param2) { InitBlock(values, inclusiveLowerPoint, inclusiveUpperPoint, enclosingInstance); @@ -608,7 +609,7 @@ public override DocIdSet GetDocIdSet(IndexReader reader, IState state) if (inclusiveLowerPoint > inclusiveUpperPoint) return DocIdSet.EMPTY_DOCIDSET; - double[] values = Lucene.Net.Search.FieldCache_Fields.DEFAULT.GetDoubles(reader, field, (Lucene.Net.Search.DoubleParser)parser, state); + IArray values = Lucene.Net.Search.FieldCache_Fields.DEFAULT.GetDoubles(reader, field, (Lucene.Net.Search.DoubleParser)parser, state); // we only request the usage of termDocs, if the range contains 0 return new AnonymousClassFieldCacheDocIdSet(values, inclusiveLowerPoint, inclusiveUpperPoint, this, reader, (inclusiveLowerPoint <= 0.0 && inclusiveUpperPoint >= 0.0)); } diff --git a/src/Lucene.Net/Search/FieldComparator.cs b/src/Lucene.Net/Search/FieldComparator.cs index e9a39578e0..c2f3bb84bb 100644 --- a/src/Lucene.Net/Search/FieldComparator.cs +++ b/src/Lucene.Net/Search/FieldComparator.cs @@ -275,7 +275,7 @@ public override IComparable this[int slot] public sealed class DoubleComparator:FieldComparator { private double[] values; - private double[] currentReaderValues; + private IArray currentReaderValues; private System.String field; private DoubleParser parser; private double bottom; @@ -509,7 +509,7 @@ public override IComparable this[int slot] public sealed class LongComparator:FieldComparator { private long[] values; - private long[] currentReaderValues; + private IArray currentReaderValues; private System.String field; private LongParser parser; private long bottom; @@ -784,7 +784,7 @@ public sealed class StringOrdValComparator:FieldComparator private int currentReaderGen = - 1; private UnmanagedStringArray lookup; - private int[] order; + private IArray order; private System.String field; private int bottomSlot = - 1; diff --git a/src/Lucene.Net/Search/Function/OrdFieldSource.cs b/src/Lucene.Net/Search/Function/OrdFieldSource.cs index e797db0508..cabccd9e77 100644 --- a/src/Lucene.Net/Search/Function/OrdFieldSource.cs +++ b/src/Lucene.Net/Search/Function/OrdFieldSource.cs @@ -15,10 +15,10 @@ * limitations under the License. */ -using System; using Lucene.Net.Store; +using Lucene.Net.Util; +using System; using IndexReader = Lucene.Net.Index.IndexReader; -using FieldCache = Lucene.Net.Search.FieldCache; namespace Lucene.Net.Search.Function { @@ -57,16 +57,18 @@ public class OrdFieldSource:ValueSource { private class AnonymousClassDocValues:DocValues { - public AnonymousClassDocValues(int[] arr, OrdFieldSource enclosingInstance) + public AnonymousClassDocValues(IArray arr, OrdFieldSource enclosingInstance) { InitBlock(arr, enclosingInstance); } - private void InitBlock(int[] arr, OrdFieldSource enclosingInstance) + + private void InitBlock(IArray arr, OrdFieldSource enclosingInstance) { this.arr = arr; this.enclosingInstance = enclosingInstance; } - private int[] arr; + + private IArray arr; private OrdFieldSource enclosingInstance; public OrdFieldSource Enclosing_Instance { @@ -118,7 +120,7 @@ public override System.String Description() /*(non-Javadoc) doubles = cache.GetDoubles(reader, "theDouble", null); float[] floats = cache.GetFloats(reader, "theDouble", null); writer.Flush(); bos.Position = 0; @@ -94,7 +95,7 @@ public virtual void TestInfoStream() public virtual void Test() { FieldCache cache = Lucene.Net.Search.FieldCache_Fields.DEFAULT; - double[] doubles = cache.GetDoubles(reader, "theDouble", null); + IArray doubles = cache.GetDoubles(reader, "theDouble", null); Assert.AreSame(doubles, cache.GetDoubles(reader, "theDouble", null), "Second request to cache return same array"); Assert.AreSame(doubles, cache.GetDoubles(reader, "theDouble", Lucene.Net.Search.FieldCache_Fields.DEFAULT_DOUBLE_PARSER, null), "Second request with explicit parser return same array"); Assert.IsTrue(doubles.Length == NUM_DOCS, "doubles Size: " + doubles.Length + " is not: " + NUM_DOCS); @@ -103,7 +104,7 @@ public virtual void Test() Assert.IsTrue(doubles[i] == (System.Double.MaxValue - i), doubles[i] + " does not equal: " + (System.Double.MaxValue - i)); } - long[] longs = cache.GetLongs(reader, (string) "theLong", (IState) null); + IArray longs = cache.GetLongs(reader, (string) "theLong", (IState) null); Assert.AreSame(longs, cache.GetLongs(reader, (string) "theLong", (IState) null), "Second request to cache return same array"); Assert.AreSame(longs, cache.GetLongs(reader, "theLong", Lucene.Net.Search.FieldCache_Fields.DEFAULT_LONG_PARSER, null), "Second request with explicit parser return same array"); Assert.IsTrue(longs.Length == NUM_DOCS, "longs Size: " + longs.Length + " is not: " + NUM_DOCS); diff --git a/test/Lucene.Net.Test/Search/UnmanagedArrayTests.cs b/test/Lucene.Net.Test/Search/UnmanagedArrayTests.cs new file mode 100644 index 0000000000..5b3976de5a --- /dev/null +++ b/test/Lucene.Net.Test/Search/UnmanagedArrayTests.cs @@ -0,0 +1,160 @@ +using Lucene.Net.Index; +using Lucene.Net.Util; +using NUnit.Framework; +using static Lucene.Net.Util.UnmanagedStringArray; + +namespace Lucene.Net.Search; + +public class UnmanagedArrayTests +{ + [TestCase(10)] + [TestCase(1000)] + public void ManagedArray_Int_BasicOperations(int length) + { + using var arr = new ManagedArray(length, clear: false); + Assert.AreEqual(length, arr.Length); + + for (int i = 0; i < arr.Length; i++) + arr[i] = i * 2; + + for (int i = 0; i < arr.Length; i++) + Assert.AreEqual(i * 2, arr[i]); + } + + [TestCase(10)] + [TestCase(1000)] + public void UnmanagedArray_Int_BasicOperations(int length) + { + using var arr = new UnmanagedArray(length, UnmanagedStringArray.Type.Sorting, clear: false); + Assert.AreEqual(length, arr.Length); + + for (int i = 0; i < arr.Length; i++) + arr[i] = i * 3; + + for (int i = 0; i < arr.Length; i++) + Assert.AreEqual(i * 3, arr[i]); + } + + [TestCase(10)] + public void ManagedArray_Long_BasicOperations(int length) + { + using var arr = new ManagedArray(length, clear: false); + for (int i = 0; i < arr.Length; i++) + arr[i] = 100L + i; + for (int i = 0; i < arr.Length; i++) + Assert.AreEqual(100L + i, arr[i]); + } + + [TestCase(10)] + public void UnmanagedArray_Long_BasicOperations(int length) + { + using var arr = new UnmanagedArray(length, UnmanagedStringArray.Type.Sorting, clear: false); + for (int i = 0; i < arr.Length; i++) + arr[i] = 200L + i; + for (int i = 0; i < arr.Length; i++) + Assert.AreEqual(200L + i, arr[i]); + } + + [TestCase(10)] + public void ManagedArray_Double_BasicOperations(int length) + { + using var arr = new ManagedArray(length, clear: false); + for (int i = 0; i < arr.Length; i++) + arr[i] = i * 0.5; + for (int i = 0; i < arr.Length; i++) + Assert.AreEqual(i * 0.5, arr[i]); + } + + [TestCase(10)] + public void UnmanagedArray_Double_BasicOperations(int length) + { + using var arr = new UnmanagedArray(length, UnmanagedStringArray.Type.Sorting, clear: false); + for (int i = 0; i < arr.Length; i++) + arr[i] = i * 1.5; + for (int i = 0; i < arr.Length; i++) + Assert.AreEqual(i * 1.5, arr[i]); + } + + [Test] + public void ManagedArray_TermInfo_BasicOperations() + { + using var arr = new ManagedArray(5, clear: false); + for (int i = 0; i < arr.Length; i++) + arr[i] = new TermInfo { docFreq = i, freqPointer = i * 10, proxPointer = i * 100, skipOffset = i * 1000 }; + for (int i = 0; i < arr.Length; i++) + { + Assert.AreEqual(i, arr[i].docFreq); + Assert.AreEqual(i * 10, arr[i].freqPointer); + Assert.AreEqual(i * 100, arr[i].proxPointer); + Assert.AreEqual(i * 1000, arr[i].skipOffset); + } + } + + [Test] + public void UnmanagedArray_TermInfo_BasicOperations() + { + using var arr = new UnmanagedArray(5, UnmanagedStringArray.Type.Sorting, clear: false); + for (int i = 0; i < arr.Length; i++) + arr[i] = new TermInfo { docFreq = i, freqPointer = i * 10, proxPointer = i * 100, skipOffset = i * 1000 }; + for (int i = 0; i < arr.Length; i++) + { + Assert.AreEqual(i, arr[i].docFreq); + Assert.AreEqual(i * 10, arr[i].freqPointer); + Assert.AreEqual(i * 100, arr[i].proxPointer); + Assert.AreEqual(i * 1000, arr[i].skipOffset); + } + } + + [Test] + public unsafe void ManagedArray_UnmanagedString_BasicOperations() + { + using var arr = new ManagedArray(3, clear: false); + for (int i = 0; i < arr.Length; i++) + arr[i] = new UnmanagedString { Start = (byte*)i }; + for (int i = 0; i < arr.Length; i++) + Assert.AreEqual((long)i, (long)arr[i].Start); + } + + [Test] + public unsafe void UnmanagedArray_UnmanagedString_BasicOperations() + { + using var arr = new UnmanagedArray(3, UnmanagedStringArray.Type.Sorting, clear: false); + + for (int i = 0; i < arr.Length; i++) + arr[i] = new UnmanagedString { Start = (byte*)i }; + for (int i = 0; i < arr.Length; i++) + Assert.AreEqual((long)i, (long)arr[i].Start); + } + + [Test] + public void ManagedArray_arr_And_ReadOnlyarr_Consistency() + { + using var arr = new ManagedArray(5, clear: false); + for (int i = 0; i < arr.Length; i++) + arr[i] = i * 7; + for (int i = 0; i < arr.Length; i++) + Assert.AreEqual(arr[i], arr[i]); + } + + [Test] + public void UnmanagedArray_arr_And_ReadOnlyarr_Consistency() + { + using var arr = new UnmanagedArray(5, UnmanagedStringArray.Type.Sorting, clear: false); + for (int i = 0; i < arr.Length; i++) + arr[i] = i * 9; + for (int i = 0; i < arr.Length; i++) + Assert.AreEqual(arr[i], arr[i]); + } + + [Test] + public void HybridArray_Creates_Correct_Type() + { + var small = HybridArray.Create(10, UnmanagedStringArray.Type.Sorting); + Assert.IsInstanceOf>(small); + small.Dispose(); + + var large = HybridArray.Create(100_000, UnmanagedStringArray.Type.Sorting); + Assert.IsInstanceOf>(large); + large.Dispose(); + } +} \ No newline at end of file diff --git a/test/Lucene.Net.Test/Search/UnmanagedStringArrayTests.cs b/test/Lucene.Net.Test/Search/UnmanagedStringArrayTests.cs index 5e4d5fe976..7236babead 100644 --- a/test/Lucene.Net.Test/Search/UnmanagedStringArrayTests.cs +++ b/test/Lucene.Net.Test/Search/UnmanagedStringArrayTests.cs @@ -15,6 +15,8 @@ public void Should_Find_All_Terms() { using (var terms = new UnmanagedStringArray(11, startIndex: 1, UnmanagedStringArray.Type.Sorting)) { + terms.SetAsNull(0); + for (var letter = 'a'; letter <= 'j'; letter++) { terms.Add(new Span(letter.ToString().ToCharArray())); @@ -34,6 +36,8 @@ public void Should_Find_All_Terms() public void Should_Find_With_Missing_Terms() { var terms = new UnmanagedStringArray(11, startIndex: 1, UnmanagedStringArray.Type.Sorting); + terms.SetAsNull(0); + var count = 0; for (var letter = 'a'; letter <= 'j'; letter++) { @@ -63,6 +67,7 @@ public void Should_Find_With_Missing_Terms() VerifyNonExistingTerms(terms); terms = new UnmanagedStringArray(11, startIndex: 1, UnmanagedStringArray.Type.Sorting); + terms.SetAsNull(0); for (var letter = 'a'; letter <= 'j'; letter++) { @@ -90,6 +95,7 @@ public void Should_Find_With_Missing_Terms() VerifyNonExistingTerms(terms); terms = new UnmanagedStringArray(11, startIndex: 1, UnmanagedStringArray.Type.Sorting); + terms.SetAsNull(0); for (var letter = 'a'; letter <= 'j'; letter++) { diff --git a/test/Lucene.Net.Test/Util/TestFieldCacheSanityChecker.cs b/test/Lucene.Net.Test/Util/TestFieldCacheSanityChecker.cs index 75ffd5b196..99be211088 100644 --- a/test/Lucene.Net.Test/Util/TestFieldCacheSanityChecker.cs +++ b/test/Lucene.Net.Test/Util/TestFieldCacheSanityChecker.cs @@ -101,7 +101,7 @@ public virtual void TestSanity() FieldCache cache = Lucene.Net.Search.FieldCache_Fields.DEFAULT; cache.PurgeAllCaches(); - double[] doubles; + IArray doubles; int[] ints; doubles = cache.GetDoubles(readerA, "theDouble", null);