feat: naive benchmarks

Author: xLPMG

benchmarks/naive_matmul_benchmarks.txt

@@ -0,0 +1,12 @@
+Running NaiveMatmulMNKBench 2048x2048x2048 benchmark
+Total time (s):                       7.9635
+Total reps:                           1
+Total floating point operations:      17179869184
+Estimated GFLOPS/sec:                 2.15733
+--------------------------------------------------
+Running NaiveMatmulBrMNKBench 1024x1024x1024 br=16 benchmark
+Total time (s):                       45.542
+Total reps:                           1
+Total floating point operations:      34359738368
+Estimated GFLOPS/sec:                 0.754462
+--------------------------------------------------

benchmarks/naive_sigmoid_benchmark.txt

@@ -0,0 +1,28 @@
+Running NaiveSigmoidPrimitiveBench 50x50 benchmark
+Total time (s):                       3
+Total reps:                           968308
+Total number of elements:             2420770000
+Total amount of processed data (GiB): 18.0361
+Bandwidth (GiB/s)                     6.01205
+--------------------------------------------------
+Running NaiveSigmoidPrimitiveBench 64x64 benchmark
+Total time (s):                       3
+Total reps:                           593757
+Total number of elements:             2432028672
+Total amount of processed data (GiB): 18.12
+Bandwidth (GiB/s)                     6.04001
+--------------------------------------------------
+Running NaiveSigmoidPrimitiveBench 512x512 benchmark
+Total time (s):                       3.00025
+Total reps:                           9206
+Total number of elements:             2413297664
+Total amount of processed data (GiB): 17.9805
+Bandwidth (GiB/s)                     5.99299
+--------------------------------------------------
+Running NaiveSigmoidPrimitiveBench 2048x2048 benchmark
+Total time (s):                       3.00451
+Total reps:                           567
+Total number of elements:             2378170368
+Total amount of processed data (GiB): 17.7188
+Bandwidth (GiB/s)                     5.89738
+--------------------------------------------------

include/mlc/benchmarks/all_benchmarks.h

@@ -5,9 +5,12 @@
 #include <mlc/benchmarks/TensorOperation.bench.h>
 #include <mlc/benchmarks/matmul/Matmul_br_m_n_k.bench.h>
 #include <mlc/benchmarks/matmul/Matmul_m_n_k.bench.h>
+#include <mlc/benchmarks/matmul/naive_matmul_br_m_n_k.bench.h>
+#include <mlc/benchmarks/matmul/naive_matmul_m_n_k.bench.h>
 #include <mlc/benchmarks/unary/fast_sigmoid_primitive.bench.h>
 #include <mlc/benchmarks/unary/identity_primitive.bench.h>
 #include <mlc/benchmarks/unary/identity_trans_primitive.bench.h>
+#include <mlc/benchmarks/unary/naive_sigmoid_primitive.bench.h>
 #include <mlc/benchmarks/unary/reciprocal_primitive.bench.h>
 #include <mlc/benchmarks/unary/relu_primitive.bench.h>
 #include <mlc/benchmarks/unary/relu_trans_primitive.bench.h>

include/mlc/benchmarks/matmul/naive_matmul_br_m_n_k.bench.h

@@ -0,0 +1,47 @@
+#ifndef NAIVE_MATMUL_BR_M_N_K_BENCH_H
+#define NAIVE_MATMUL_BR_M_N_K_BENCH_H
+#include <mlc/benchmarks/Benchmark.h>
+
+namespace mini_jit
+{
+    namespace benchmarks
+    {
+        /**
+         * @brief Benchmark for naive matrix multiplication using BRGEMM.
+         */
+        class NaiveMatmulBrMNKBench : public Benchmark
+        {
+        public:
+            /**
+             * @brief Constructor for the benchmark for naive matrix multiplication using BRGEMM.
+             * @param run_time The time to run the benchmark in seconds.
+             * @param m number of rows in A and C.
+             * @param n number of columns in B and C.
+             * @param k number of columns in A and rows in B.
+             * @param br_size The size of the batch-reduce.
+             */
+            NaiveMatmulBrMNKBench(double run_time,
+                                  int    m,
+                                  int    n,
+                                  int    k,
+                                  int    br_size);
+            //! Destructor
+            ~NaiveMatmulBrMNKBench() override = default;
+            //! Runs the benchmark.
+            void run() override;
+
+        private:
+            int    m_M;
+            int    m_N;
+            int    m_K;
+            int    m_br_size;
+            double m_run_time;
+            float* m_A;
+            float* m_B;
+            float* m_C;
+        };
+
+    } // namespace benchmarks
+} // namespace mini_jit
+
+#endif // NAIVE_MATMUL_BR_M_N_K_BENCH_H

include/mlc/benchmarks/matmul/naive_matmul_m_n_k.bench.h

@@ -0,0 +1,41 @@
+#ifndef NAIVE_MATMUL_M_N_K_BENCH_H
+#define NAIVE_MATMUL_M_N_K_BENCH_H
+#include <mlc/benchmarks/Benchmark.h>
+
+namespace mini_jit
+{
+    namespace benchmarks
+    {
+        class NaiveMatmulMNKBench : public Benchmark
+        {
+        public:
+            /**
+             * @brief Constructor for the benchmark for naive matrix multiplication using GEMM.
+             * @param run_time The time to run the benchmark in seconds.
+             * @param m number of rows in A and C.
+             * @param n number of columns in B and C.
+             * @param k number of columns in A and rows in B.
+             */
+            NaiveMatmulMNKBench(double run_time,
+                                int    m,
+                                int    n,
+                                int    k);
+            //! Destructor
+            ~NaiveMatmulMNKBench() override = default;
+            //! Runs the benchmark.
+            void run() override;
+
+        private:
+            int    m_M;
+            int    m_N;
+            int    m_K;
+            double m_run_time;
+            float* m_A;
+            float* m_B;
+            float* m_C;
+        };
+
+    } // namespace benchmarks
+} // namespace mini_jit
+
+#endif // NAIVE_MATMUL_M_N_K_BENCH_H

include/mlc/benchmarks/unary/naive_sigmoid_primitive.bench.h

@@ -0,0 +1,38 @@
+#ifndef NAIVE_SIGMOID_PRIMITIVE_BENCH_H
+#define NAIVE_SIGMOID_PRIMITIVE_BENCH_H
+#include <cstdint>
+#include <mlc/benchmarks/Benchmark.h>
+
+namespace mini_jit
+{
+    namespace benchmarks
+    {
+        class NaiveSigmoidPrimitiveBench : public Benchmark
+        {
+        public:
+            /**
+             * @brief Constructor for the benchmark for the naive sigmoid primitive.
+             * @param runTime The time to run the benchmark in seconds.
+             * @param m number of rows in A and B.
+             * @param n number of columns in A and B.
+             */
+            NaiveSigmoidPrimitiveBench(double   runTime,
+                                      uint32_t m,
+                                      uint32_t n);
+            //! Destructor
+            ~NaiveSigmoidPrimitiveBench() override = default;
+            //! Runs the benchmark.
+            void run() override;
+
+        private:
+            uint32_t m_M;
+            uint32_t m_N;
+            double   m_runTime;
+            float*   m_A;
+            float*   m_B;
+        };
+
+    } // namespace benchmarks
+} // namespace mini_jit
+
+#endif // NAIVE_SIGMOID_PRIMITIVE_BENCH_H

src/benchmarks.cpp

@@ -621,6 +621,7 @@ int main(int argc, char* argv[])
     bool has_gemm                     = false;
     bool has_brgemm                   = false;
     bool has_matmul                   = false;
+    bool has_naive_matmul             = false;
     bool has_unary                    = false;
     bool has_tensor_operations        = false;
     bool has_shared_tensor_operations = false;
@@ -629,6 +630,7 @@ int main(int argc, char* argv[])
     bool has_opt_einsum_benchmark     = false;
     bool has_reciprocal               = false;
     bool has_sigmoid                  = false;
+    bool has_naive_sigmoid            = false;
     for (int i = 1; i < argc; ++i)
     {
         if (strcmp(argv[i], "gemm") == 0)
@@ -637,6 +639,8 @@ int main(int argc, char* argv[])
             has_brgemm = true;
         else if (strcmp(argv[i], "matmul") == 0)
             has_matmul = true;
+        else if (strcmp(argv[i], "naive-matmul") == 0)
+            has_naive_matmul = true;
         else if (strcmp(argv[i], "unary") == 0)
             has_unary = true;
         else if (strcmp(argv[i], "top") == 0)
@@ -653,6 +657,8 @@ int main(int argc, char* argv[])
             has_reciprocal = true;
         else if (strcmp(argv[i], "sigmoid") == 0)
             has_sigmoid = true;
+        else if (strcmp(argv[i], "naive-sigmoid") == 0)
+            has_naive_sigmoid = true;
         else if (strcmp(argv[i], "help") == 0)
             std::cout << "Usage: " << argv[0] << " [gemm|brgemm|matmul|unary|top|top-shared|top-opt|einsum|opt-einsum|reciprocal|sigmoid]" << std::endl;
         else
@@ -683,6 +689,16 @@ int main(int argc, char* argv[])
         matmul_bm.close();
     }
 
+    if (has_naive_matmul)
+    {
+        mini_jit::benchmarks::NaiveMatmulMNKBench   bench_mnk(3.0, 2048, 2048, 2048);
+        mini_jit::benchmarks::NaiveMatmulBrMNKBench bench_brmnk(3.0, 1024, 1024, 1024, 16);
+        std::ofstream                               matmul_bm("benchmarks/naive_matmul_benchmarks.txt");
+        print_throughput(bench_mnk, matmul_bm, "NaiveMatmulMNKBench 2048x2048x2048");
+        print_throughput(bench_brmnk, matmul_bm, "NaiveMatmulBrMNKBench 1024x1024x1024 br=16");
+        matmul_bm.close();
+    }
+
     if (has_unary)
     {
         const double  RUN_TIME = 3.0;
@@ -1024,5 +1040,22 @@ int main(int argc, char* argv[])
         sigmoid_bm.close();
     }
 
+    if (has_naive_sigmoid)
+    {
+        const double  RUN_TIME = 3.0;
+        std::ofstream sigmoid_bm("benchmarks/naive_sigmoid_benchmark.txt");
+
+        mini_jit::benchmarks::NaiveSigmoidPrimitiveBench bench_naive_sigmoid_50_50(RUN_TIME, 50, 50);
+        mini_jit::benchmarks::NaiveSigmoidPrimitiveBench bench_naive_sigmoid_64_64(RUN_TIME, 64, 64);
+        mini_jit::benchmarks::NaiveSigmoidPrimitiveBench bench_naive_sigmoid_512_512(RUN_TIME, 512, 512);
+        mini_jit::benchmarks::NaiveSigmoidPrimitiveBench bench_naive_sigmoid_2048_2048(RUN_TIME, 2048, 2048);
+        print_bandwidth(bench_naive_sigmoid_50_50, sigmoid_bm, "NaiveSigmoidPrimitiveBench 50x50");
+        print_bandwidth(bench_naive_sigmoid_64_64, sigmoid_bm, "NaiveSigmoidPrimitiveBench 64x64");
+        print_bandwidth(bench_naive_sigmoid_512_512, sigmoid_bm, "NaiveSigmoidPrimitiveBench 512x512");
+        print_bandwidth(bench_naive_sigmoid_2048_2048, sigmoid_bm, "NaiveSigmoidPrimitiveBench 2048x2048");
+
+        sigmoid_bm.close();
+    }
+
     return 0;
 }

src/benchmarks/matmul/naive_matmul_br_m_n_k.bench.cpp

@@ -0,0 +1,92 @@
+#include <chrono>
+#include <mlc/Brgemm.h>
+#include <mlc/Kernel.h>
+#include <mlc/benchmarks/Benchmark.h>
+#include <mlc/benchmarks/matmul/naive_matmul_br_m_n_k.bench.h>
+#include <random>
+
+mini_jit::benchmarks::NaiveMatmulBrMNKBench::NaiveMatmulBrMNKBench(double run_time,
+                                                                   int    m,
+                                                                   int    n,
+                                                                   int    k,
+                                                                   int    br_size)
+    : Benchmark()
+{
+    m_M        = m;
+    m_N        = n;
+    m_K        = k;
+    m_br_size  = br_size;
+    m_run_time = run_time;
+}
+
+void mini_jit::benchmarks::NaiveMatmulBrMNKBench::run()
+{
+    m_A = new float[m_M * m_K * m_br_size];
+    m_B = new float[m_K * m_N * m_br_size];
+    m_C = new float[m_M * m_N];
+
+    // Initialize matrices A and B with random values
+    std::random_device                    rd;
+    std::mt19937                          gen(rd());
+    std::uniform_real_distribution<float> dist(-10.0f, 10.0f);
+
+    for (int i = 0; i < m_M * m_K * m_br_size; i++)
+    {
+        m_A[i] = dist(gen);
+    }
+    for (int i = 0; i < m_K * m_N * m_br_size; i++)
+    {
+        m_B[i] = dist(gen);
+    }
+    // Initialize matrix C with zeros
+    for (int i = 0; i < m_M * m_N; ++i)
+    {
+        m_C[i] = 0.0f;
+    }
+
+    // RUN
+    long   l_num_reps   = 0;
+    auto   l_start_time = std::chrono::high_resolution_clock::now();
+    double l_elapsed    = 0.0;
+    double l_runTimeMs  = m_run_time * 1e6;
+    do
+    {
+        for (int col = 0; col < m_N; ++col)
+        {
+            for (int row = 0; row < m_M; ++row)
+            {
+                float sum = 0.0f;
+                for (int br = 0; br < m_br_size; ++br)
+                {
+                    for (int k = 0; k < m_K; ++k)
+                    {
+                        sum += m_A[br * m_M * m_K + row + k * m_M] * m_B[br * m_K * m_N + k + col * m_K];
+                    }
+                }
+                m_C[row + col * m_M] += sum;
+            }
+        }
+        ++l_num_reps;
+        auto l_now = std::chrono::high_resolution_clock::now();
+        l_elapsed  = std::chrono::duration_cast<std::chrono::microseconds>(
+                        l_now - l_start_time)
+                        .count();
+    } while (l_elapsed < l_runTimeMs);
+    l_elapsed /= 1e6; // Convert to seconds
+    // END RUN
+
+    // Calculate metrics
+    long   l_totalOperations = 2.0 * m_M * m_N * m_K * l_num_reps * m_br_size;
+    double l_gflops          = ((double)l_totalOperations) / (l_elapsed * 1e9);
+
+    // Store the results
+    m_benchmarkResult.numReps             = l_num_reps;
+    m_benchmarkResult.elapsedSeconds      = l_elapsed;
+    m_benchmarkResult.totalNumberElements = m_M * m_N * m_K * l_num_reps;
+    m_benchmarkResult.totalOperations     = l_totalOperations;
+    m_benchmarkResult.gflops              = l_gflops;
+
+    delete[] m_A;
+    delete[] m_B;
+    delete[] m_C;
+}