Skip to content

Commit 491f0c1

Browse files
tianleiwuxiaomsftXiaoyan Huapsonawanekunal-vaishnavi
authored
cherry picks for 1.23.0 release (microsoft#25959)
### Description Cherry-pick the following PRs: microsoft#25943 microsoft#25937 microsoft#25917 microsoft#25909 microsoft#25898 microsoft#25897 microsoft#25888 microsoft#25881 microsoft#25830 microsoft#25619 microsoft#25575 microsoft#25572 microsoft#25558 microsoft#25530 microsoft#25474 microsoft#25455 microsoft#25110 Also two dependent PRs for qMoE cpu: microsoft#25877 microsoft#25822 --------- Co-authored-by: xiaomsft <[email protected]> Co-authored-by: Xiaoyan Hu <[email protected]> Co-authored-by: Akshay Sonawane <[email protected]> Co-authored-by: Kunal Vaishnavi <[email protected]> Co-authored-by: Pradeep Sakhamoori <[email protected]> Co-authored-by: mingyue <[email protected]> Co-authored-by: Maximilian Müller <[email protected]> Co-authored-by: Adrian Lizarraga <[email protected]> Co-authored-by: Dmitri Smirnov <[email protected]> Co-authored-by: Emmanuel <[email protected]> Co-authored-by: Emmanuel Assumang <[email protected]> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: praneshgo <[email protected]> Co-authored-by: Hariharan Seshadri <[email protected]> Co-authored-by: Jing Fang <[email protected]> Co-authored-by: Ishwar Raut <[email protected]>
1 parent 30612fb commit 491f0c1

File tree

126 files changed

+11549
-2657
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

126 files changed

+11549
-2657
lines changed

‎cmake/external/onnxruntime_external_deps.cmake‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -603,10 +603,6 @@ if(NOT (onnx_FOUND OR ONNX_FOUND)) # building ONNX from source
603603
endif()
604604
endif()
605605

606-
if (onnxruntime_RUN_ONNX_TESTS)
607-
add_definitions(-DORT_RUN_EXTERNAL_ONNX_TESTS)
608-
endif()
609-
610606
if(onnxruntime_ENABLE_DLPACK)
611607
message(STATUS "dlpack is enabled.")
612608

‎cmake/onnxruntime_mlas.cmake‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@ function(setup_mlas_source_for_windows)
108108
${MLAS_SRC_DIR}/eltwise_kernel_neon.h
109109
${MLAS_SRC_DIR}/eltwise_kernel_neon.cpp
110110
${MLAS_SRC_DIR}/eltwise_kernel_neon_fp16.cpp
111+
${MLAS_SRC_DIR}/sqnbitgemm_kernel_neon_int8_i8mm.cpp
111112
)
112113

113114
set(mlas_platform_preprocess_srcs
@@ -429,12 +430,16 @@ else()
429430
${MLAS_SRC_DIR}/softmax_kernel_neon.cpp
430431
${MLAS_SRC_DIR}/eltwise_kernel_neon.h
431432
${MLAS_SRC_DIR}/eltwise_kernel_neon.cpp
433+
${MLAS_SRC_DIR}/sqnbitgemm_kernel_neon_int8_i8mm.cpp
432434
)
433435
if (onnxruntime_USE_KLEIDIAI)
434436
setup_kleidiai()
435437
endif()
436438
set_source_files_properties(${MLAS_SRC_DIR}/sqnbitgemm_kernel_neon_int8.cpp
437439
PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+dotprod")
440+
set_source_files_properties(${MLAS_SRC_DIR}/sqnbitgemm_kernel_neon_int8_i8mm.cpp
441+
PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+i8mm ")
442+
438443
if (NOT APPLE)
439444
set(mlas_platform_srcs
440445
${mlas_platform_srcs}

‎csharp/src/Microsoft.ML.OnnxRuntime/CompileModel.shared.cs‎

Lines changed: 376 additions & 21 deletions
Large diffs are not rendered by default.

‎csharp/src/Microsoft.ML.OnnxRuntime/NativeCompileApiMethods.shared.cs‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@ public struct OrtCompileApi
2121
public IntPtr ModelCompilationOptions_SetEpContextEmbedMode;
2222
public IntPtr CompileModel;
2323
public IntPtr ModelCompilationOptions_SetFlags;
24+
public IntPtr ModelCompilationOptions_SetEpContextBinaryInformation;
25+
public IntPtr ModelCompilationOptions_SetGraphOptimizationLevel;
26+
public IntPtr ModelCompilationOptions_SetOutputModelWriteFunc;
27+
public IntPtr ModelCompilationOptions_SetOutputModelGetInitializerLocationFunc;
2428
}
2529

2630
internal class NativeMethods
@@ -101,6 +105,37 @@ public DOrtModelCompilationOptions_SetOutputModelExternalInitializersFile
101105
uint flags);
102106
public DOrtModelCompilationOptions_SetFlags OrtModelCompilationOptions_SetFlags;
103107

108+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
109+
public delegate IntPtr /* OrtStatus* */ DOrtModelCompilationOptions_SetEpContextBinaryInformation(
110+
IntPtr /* OrtModelCompilationOptions* */ options,
111+
byte[] /* const ORTCHAR_T* */ outputDirectory,
112+
byte[] /* const ORTCHAR_T* */ modelName);
113+
public DOrtModelCompilationOptions_SetEpContextBinaryInformation
114+
OrtModelCompilationOptions_SetEpContextBinaryInformation;
115+
116+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
117+
public delegate IntPtr /* OrtStatus* */ DOrtModelCompilationOptions_SetGraphOptimizationLevel(
118+
IntPtr /* OrtModelCompilationOptions* */ options,
119+
GraphOptimizationLevel graphOptimizationLevel);
120+
public DOrtModelCompilationOptions_SetGraphOptimizationLevel
121+
OrtModelCompilationOptions_SetGraphOptimizationLevel;
122+
123+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
124+
public delegate IntPtr /* OrtStatus* */ DOrtModelCompilationOptions_SetOutputModelWriteFunc(
125+
IntPtr /* OrtModelCompilationOptions* */ options,
126+
IntPtr /* DOrtWriteBufferDelegate */ writeFunc,
127+
IntPtr /* void* */ state);
128+
public DOrtModelCompilationOptions_SetOutputModelWriteFunc
129+
OrtModelCompilationOptions_SetOutputModelWriteFunc;
130+
131+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
132+
public delegate IntPtr /* OrtStatus* */ DOrtModelCompilationOptions_SetOutputModelGetInitializerLocationFunc(
133+
IntPtr /* OrtModelCompilationOptions* */ options,
134+
IntPtr /* DOrtHandleInitializerDataDelegate */ handleInitializerFunc,
135+
IntPtr /* void* */ state);
136+
public DOrtModelCompilationOptions_SetOutputModelGetInitializerLocationFunc
137+
OrtModelCompilationOptions_SetOutputModelGetInitializerLocationFunc;
138+
104139
internal NativeMethods(OnnxRuntime.NativeMethods.DOrtGetCompileApi getCompileApi)
105140
{
106141

@@ -161,6 +196,27 @@ internal NativeMethods(OnnxRuntime.NativeMethods.DOrtGetCompileApi getCompileApi
161196
_compileApi.ModelCompilationOptions_SetFlags,
162197
typeof(DOrtModelCompilationOptions_SetFlags));
163198

199+
OrtModelCompilationOptions_SetEpContextBinaryInformation =
200+
(DOrtModelCompilationOptions_SetEpContextBinaryInformation)Marshal.GetDelegateForFunctionPointer(
201+
_compileApi.ModelCompilationOptions_SetEpContextBinaryInformation,
202+
typeof(DOrtModelCompilationOptions_SetEpContextBinaryInformation));
203+
204+
OrtModelCompilationOptions_SetGraphOptimizationLevel =
205+
(DOrtModelCompilationOptions_SetGraphOptimizationLevel)Marshal.GetDelegateForFunctionPointer(
206+
_compileApi.ModelCompilationOptions_SetGraphOptimizationLevel,
207+
typeof(DOrtModelCompilationOptions_SetGraphOptimizationLevel));
208+
209+
OrtModelCompilationOptions_SetOutputModelWriteFunc =
210+
(DOrtModelCompilationOptions_SetOutputModelWriteFunc)Marshal.GetDelegateForFunctionPointer(
211+
_compileApi.ModelCompilationOptions_SetOutputModelWriteFunc,
212+
typeof(DOrtModelCompilationOptions_SetOutputModelWriteFunc));
213+
214+
OrtModelCompilationOptions_SetOutputModelGetInitializerLocationFunc =
215+
(DOrtModelCompilationOptions_SetOutputModelGetInitializerLocationFunc)Marshal.
216+
GetDelegateForFunctionPointer(
217+
_compileApi.ModelCompilationOptions_SetOutputModelGetInitializerLocationFunc,
218+
typeof(DOrtModelCompilationOptions_SetOutputModelGetInitializerLocationFunc));
219+
164220
}
165221
}
166222
}

‎csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,6 +450,7 @@ public struct OrtApi
450450

451451
public IntPtr Graph_GetModelMetadata;
452452
public IntPtr GetModelCompatibilityForEpDevices;
453+
public IntPtr CreateExternalInitializerInfo;
453454
}
454455

455456
internal static class NativeMethods
@@ -787,9 +788,35 @@ static NativeMethods()
787788
api_.SessionOptionsSetEpSelectionPolicyDelegate,
788789
typeof(DSessionOptionsSetEpSelectionPolicyDelegate));
789790

791+
OrtReleaseExternalInitializerInfo =
792+
(DOrtReleaseExternalInitializerInfo)Marshal.GetDelegateForFunctionPointer(
793+
api_.ReleaseExternalInitializerInfo,
794+
typeof(DOrtReleaseExternalInitializerInfo));
795+
796+
OrtExternalInitializerInfo_GetFilePath =
797+
(DOrtExternalInitializerInfo_GetFilePath)Marshal.GetDelegateForFunctionPointer(
798+
api_.ExternalInitializerInfo_GetFilePath,
799+
typeof(DOrtExternalInitializerInfo_GetFilePath));
800+
801+
OrtExternalInitializerInfo_GetFileOffset =
802+
(DOrtExternalInitializerInfo_GetFileOffset)Marshal.GetDelegateForFunctionPointer(
803+
api_.ExternalInitializerInfo_GetFileOffset,
804+
typeof(DOrtExternalInitializerInfo_GetFileOffset));
805+
806+
OrtExternalInitializerInfo_GetByteSize =
807+
(DOrtExternalInitializerInfo_GetByteSize)Marshal.GetDelegateForFunctionPointer(
808+
api_.ExternalInitializerInfo_GetByteSize,
809+
typeof(DOrtExternalInitializerInfo_GetByteSize));
810+
790811
OrtGetModelCompatibilityForEpDevices = (DOrtGetModelCompatibilityForEpDevices)Marshal.GetDelegateForFunctionPointer(
791812
api_.GetModelCompatibilityForEpDevices,
792813
typeof(DOrtGetModelCompatibilityForEpDevices));
814+
815+
OrtCreateExternalInitializerInfo =
816+
(DOrtCreateExternalInitializerInfo)Marshal.GetDelegateForFunctionPointer(
817+
api_.CreateExternalInitializerInfo,
818+
typeof(DOrtCreateExternalInitializerInfo));
819+
793820
}
794821

795822
internal class NativeLib
@@ -2382,6 +2409,70 @@ out IntPtr lora_adapter
23822409
public delegate ref CompileApi.OrtCompileApi DOrtGetCompileApi();
23832410
#endif
23842411
public static DOrtGetCompileApi OrtGetCompileApi;
2412+
2413+
/// <summary>
2414+
/// Delegate called by ORT to write a buffer (ONNX model bytes) to a custom destination (e.g., file or stream).
2415+
/// </summary>
2416+
/// <param name="state">State that was provided in when the delegate was registered.</param>
2417+
/// <param name="buffer">The buffer to write.</param>
2418+
/// <param name="bufferNumBytes">The size of the buffer in bytes.</param>
2419+
/// <returns>OrtStatus*</returns>
2420+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
2421+
public delegate IntPtr /* OrtStatus* */ DOrtWriteBufferToDestinationDelegate(
2422+
IntPtr /* void* */ state,
2423+
IntPtr /* const void* */ buffer,
2424+
UIntPtr /* size_t */ bufferNumBytes
2425+
);
2426+
2427+
/// <summary>
2428+
/// Function called by ORT to allow user to specify how an initializer should be saved while compiling
2429+
/// a model, that is, either written to an external file or stored within the model. ORT calls this function
2430+
/// for every initializer.
2431+
/// </summary>
2432+
/// <param name="state">State that was provided when the delegate was registered.</param>
2433+
/// <param name="initializerName">The initializer's name.</param>
2434+
/// <param name="initializerValue">The OrtValue containing the initializer's data, type, and shape</param>
2435+
/// <param name="externalInfo">The original initializer's location in an external file, or NULL.</param>
2436+
/// <param name="newExternalInfo">Output parameter set to a new OrtExternalInitializerInfo instance
2437+
/// indicating the location where the function implementation stored the initializer data. If the function
2438+
/// implementation sets `newExternalInfo` to NULL, ORT stores the initializer within the generated model.</param>
2439+
/// <returns></returns>
2440+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
2441+
public delegate IntPtr /* OrtStatus* */ DOrtGetInitializerLocationDelegate(
2442+
IntPtr /* void* */ state,
2443+
IntPtr /* const char* */ initializerName,
2444+
IntPtr /* const OrtValue* */ initializerValue,
2445+
IntPtr /* const OrtExternalInitializerInfo* */ externalInfo,
2446+
out IntPtr /* OrtExternalInitializerInfo** */ newExternalInfo
2447+
);
2448+
2449+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
2450+
public delegate void DOrtReleaseExternalInitializerInfo(IntPtr /* OrtExternalInitializerInfo* */ info);
2451+
2452+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
2453+
public delegate IntPtr /* OrtStatus* */ DOrtCreateExternalInitializerInfo(
2454+
byte[] /* const ORTCHAR_T* */ filePath,
2455+
long /* int64_t */ fileOffset,
2456+
UIntPtr /* size_t */ byteSize,
2457+
out IntPtr /* OrtExternalInitializerInfo** */ outInfo);
2458+
2459+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
2460+
public delegate IntPtr /* const ORTCHAR_T* */ DOrtExternalInitializerInfo_GetFilePath(
2461+
IntPtr /* const OrtExternalInitializerInfo* */ info);
2462+
2463+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
2464+
public delegate long /* int64_t */ DOrtExternalInitializerInfo_GetFileOffset(
2465+
IntPtr /* const OrtExternalInitializerInfo* */ info);
2466+
2467+
[UnmanagedFunctionPointer(CallingConvention.Winapi)]
2468+
public delegate UIntPtr /* size_t */ DOrtExternalInitializerInfo_GetByteSize(
2469+
IntPtr /* const OrtExternalInitializerInfo* */ info);
2470+
2471+
public static DOrtReleaseExternalInitializerInfo OrtReleaseExternalInitializerInfo;
2472+
public static DOrtCreateExternalInitializerInfo OrtCreateExternalInitializerInfo;
2473+
public static DOrtExternalInitializerInfo_GetFilePath OrtExternalInitializerInfo_GetFilePath;
2474+
public static DOrtExternalInitializerInfo_GetFileOffset OrtExternalInitializerInfo_GetFileOffset;
2475+
public static DOrtExternalInitializerInfo_GetByteSize OrtExternalInitializerInfo_GetByteSize;
23852476
#endregion
23862477

23872478
#region Auto EP API related

‎csharp/src/Microsoft.ML.OnnxRuntime/NativeOnnxValueHelper.shared.cs‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,45 @@ internal static byte[] GetPlatformSerializedString(string str)
150150
else
151151
return StringToZeroTerminatedUtf8(str);
152152
}
153+
154+
/// <summary>
155+
/// Converts a null-terminated path string that is pointed to by the given IntPtr handle into
156+
/// a C# UTF-16 string.
157+
/// </summary>
158+
/// <remarks>A path string on Windows is utf-16, but utf-8 on other operating systems.</remarks>
159+
/// <param name="strPtr"></param>
160+
/// <returns></returns>
161+
internal static string StringFromNativePathString(IntPtr strPtr)
162+
{
163+
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
164+
{
165+
if (strPtr == IntPtr.Zero)
166+
{
167+
return string.Empty;
168+
}
169+
170+
// Get length of utf16 string by checking for two 0 bytes in a row.
171+
int length = 0;
172+
while (Marshal.ReadInt16(strPtr, length * 2) != 0)
173+
{
174+
length += 1;
175+
}
176+
177+
if (length == 0)
178+
{
179+
return string.Empty;
180+
}
181+
182+
unsafe
183+
{
184+
return System.Text.Encoding.Unicode.GetString((byte*)strPtr, length * 2);
185+
}
186+
}
187+
else
188+
{
189+
return StringFromNativeUtf8(strPtr);
190+
}
191+
}
153192
}
154193

155194
// Guards an array of disposable objects on stack and disposes them in reverse order

0 commit comments

Comments
 (0)