(WIP) MPI RMA support

CMakeLists.txt

@@ -20,11 +20,11 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/cmake)
 
 project(Grappa)
 
-set(CMAKE_C_FLAGS_RELWITHDEBINFO "-O3 -g -DNDEBUG")
-set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O3 -g -DNDEBUG")
+set(CMAKE_C_FLAGS_RELWITHDEBINFO "-DGRAPPA -O3 -g -DNDEBUG")
+set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-DGRAPPA -O3 -g -DNDEBUG")
 
-set(CMAKE_C_FLAGS_DEBUG "-O1 -DDEBUG -g -DGUARD_PAGES_ON_STACK")
-set(CMAKE_CXX_FLAGS_DEBUG "-O1 -DDEBUG -g -DGUARD_PAGES_ON_STACK")
+set(CMAKE_C_FLAGS_DEBUG "-DGRAPPA -O1 -DDEBUG -g -DGUARD_PAGES_ON_STACK")
+set(CMAKE_CXX_FLAGS_DEBUG "-DGRAPPA -O1 -DDEBUG -g -DGUARD_PAGES_ON_STACK")
 
 set_property(GLOBAL PROPERTY USE_FOLDERS ON)
 enable_testing()
@@ -44,32 +44,75 @@ else()
   message(WARNING "You are using an unsupported compiler! Compilation has only been tested with Clang and GCC.")
 endif()
 
-###########################
-# Use RUNPATH if available
-# use, i.e. don't skip the full RPATH for the build tree
-set(CMAKE_SKIP_BUILD_RPATH  FALSE)
-# use final INSTALL_RPATH even in build tree (this lets us manually add things to CMAKE_INSTALL_RPATH)
-set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib:${CMAKE_INSTALL_RPATH}")
-# add the automatically determined parts of the RPATH
-# which point to directories outside the build tree to the install RPATH
-# set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
-set(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE)
-
-# the RPATH to be used when installing, but only if it's not a system directory
-list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" isSystemDir)
-if("${isSystemDir}" STREQUAL "-1") # not a system directory
-  set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${CMAKE_INSTALL_PREFIX}/lib")
-endif("${isSystemDir}" STREQUAL "-1")
-
-## Not setting runpath because having MPI libs in LD_LIBRARY_PATH was messing up VampirTrace's ability to find its own libs. Maybe there's another way to fix this, but it just seems more robust (for now) to not allow LD_LIBRARY_PATH to affect our libs (after configure uses it to find them).
-# set runpath, too
-# if(NOT APPLE)
-#   set(CMAKE_EXE_LINKER_FLAGS "-Wl,--enable-new-dtags")
-# endif()
+# ###########################
+# # Use RUNPATH if available
+# # use, i.e. don't skip the full RPATH for the build tree
+# set(CMAKE_SKIP_BUILD_RPATH  FALSE)
+# # use final INSTALL_RPATH even in build tree (this lets us manually add things to CMAKE_INSTALL_RPATH)
+# set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+# set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib:${CMAKE_INSTALL_RPATH}")
+# # add the automatically determined parts of the RPATH
+# # which point to directories outside the build tree to the install RPATH
+# # set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+# set(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE)
+
+# # the RPATH to be used when installing, but only if it's not a system directory
+# list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" isSystemDir)
+# if("${isSystemDir}" STREQUAL "-1") # not a system directory
+#   set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}:${CMAKE_INSTALL_PREFIX}/lib")
+# endif("${isSystemDir}" STREQUAL "-1")
+
+# ## Not setting runpath because having MPI libs in LD_LIBRARY_PATH was messing up VampirTrace's ability to find its own libs. Maybe there's another way to fix this, but it just seems more robust (for now) to not allow LD_LIBRARY_PATH to affect our libs (after configure uses it to find them).
+# # set runpath, too
+# # if(NOT APPLE)
+# #   set(CMAKE_EXE_LINKER_FLAGS "-Wl,--enable-new-dtags")
+# # endif()
+
+# ##### </RUNPATH> ########
 
-##### </RUNPATH> ########
+#############################################################################
+# capture information about build from git
+#############################################################################
 
+# find git
+find_package(Git)
+if(GIT_FOUND)
+
+  # Get the current "revision"
+  execute_process(
+    COMMAND bash -c "${GIT_EXECUTABLE} log --oneline | wc -l"
+    WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+    OUTPUT_VARIABLE GRAPPA_REVISION
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+
+  # Get the current working branch
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD
+    WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+    OUTPUT_VARIABLE GRAPPA_GIT_BRANCH
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+
+  # Get the latest abbreviated commit hash of the working branch
+  execute_process(
+    COMMAND ${GIT_EXECUTABLE} log -1 --format=%h
+    WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
+    OUTPUT_VARIABLE GRAPPA_GIT_COMMIT_HASH
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+
+else()
+  set(GRAPPA_REVISION        "0")
+  set(GRAPPA_GIT_BRANCH      "unknown")
+  set(GRAPPA_GIT_COMMIT_HASH "unknown")
+endif()
+
+# fill in configuration file
+configure_file(
+  "system/Version.hpp.in"
+  "system/Version.hpp"
+  )
 
 #############################################################################
 # configure profiling and tracing

configure

@@ -38,6 +38,8 @@ OptionParser.new{|p|
   p.on('--tracing',"Enable VampirTrace/gperftools-based sampled tracing. #{line_break}Looks for VampirTrace build in 'third-party' dir."){|b| opt.tracing = true }
   p.on('--profiling',"Enable gperftools-based sampled profiling."){|b| opt.profiling = true }
   p.on('--vampir=path/to/vampirtrace/root',"Specify path to VampirTrace build (enables tracing)."){|p| opt.vampir = p; opt.tracing = true }
+  p.on('--third-party-cc=path/to/c/compiler','Use alternative C compiler just for third-party builds.'){|p| opt.third_party_cc = p; }
+  p.on('--third-party-cxx=path/to/c++/compiler','Use alternative C++ compiler just for third-party builds.'){|p| opt.third_party_cxx = p; }
   p.on('--third-party=path/to/built/deps/root',"Can optionally pre-build third-party dependencies #{line_break}instead of re-building for each configuration."){|p| opt.third_party = p }
   p.on('--build-here', "Just configure into current directory rather than creating new build directories."){|b| opt.build_here = true }
   p.on('--third-party-tarfile=/path/to/file.tar', "Instead of downloading third-party dependences from the web, extract them from the specified tar file (available from Grappa website)."){|p| opt.third_party_tarfile = p }
@@ -65,6 +67,26 @@ if opt.used_cxx_flag
   end
 end
 
+if opt.third_party_cc != '' && opt.third_party_cxx == ''
+  # automatically determine cxx from --cc
+  case opt.third_party_cc
+  when /gcc$/
+    opt.third_party_cxx = opt.third_party_cc.gsub(/gcc$/,'g++')
+  when /clang$/
+    opt.third_party_cxx = opt.third_party_cc.gsub(/clang$/,'clang++')
+  end
+end
+
+if opt.third_party_cxx != '' && opt.third_party_cc == ''
+  # automatically determine cc from --cxx
+  case opt.third_party_cxx
+  when /g\+\+$/
+    opt.third_party_cc = opt.third_party_cxx.gsub(/g\+\+$/,'gcc')
+  when /clang\+\+$/
+    opt.third_party_cc = opt.third_party_cxx.gsub(/clang\+\+$/,'clang')
+  end
+end
+
 # Xcode generator must have CC set to a version of Clang, and must have a corresponding plugin
 # see github.com/bholt/Clang-Head.xcplugin for an example
 
@@ -140,6 +162,17 @@ def configure(generator, mode, opt)
 
   args << "-DTHIRD_PARTY_ROOT:PATH=#{opt.third_party}" if opt.third_party
 
+  if opt.third_party_cc == ''
+    opt.third_party_cc = opt.cc
+  end
+
+  if opt.third_party_cxx == ''
+    opt.third_party_cxx = opt.cxx
+  end
+
+  args << "-DTHIRD_PARTY_C_COMPILER=#{opt.third_party_cc}"
+  args << "-DTHIRD_PARTY_CXX_COMPILER=#{opt.third_party_cxx}"
+
   if mode
     # note: use 'RelWithDebInfo' because it adds '-g'...
     args << "-DCMAKE_BUILD_TYPE=" + {'Debug'=>'Debug', 'Release'=>'RelWithDebInfo'}[mode]

system/CMakeLists.txt

@@ -143,6 +143,8 @@ set(SYSTEM_SOURCES
   tasks/Task.cpp
   tasks/TaskingScheduler.cpp
   stack.S
+  RMA.cpp
+  SymmetricAllocator.cpp
 )
 
 ################
@@ -245,6 +247,9 @@ add_check( NTMessage_tests.cpp               1 1  pass NTMessage.cpp )
 add_check( NTBuffer_tests.cpp                1 1  pass NTBuffer.cpp )
 add_check( NTMessage_aggregator_tests.cpp    2 1  pass )
 
+add_check( RMA_tests.cpp                     2 2  pass )
+add_check( SymmetricAllocator_tests.cpp      2 2  pass )
+
 
 #
 # begin hack for dealing with communicator test

system/CollectiveOps.hpp

@@ -0,0 +1,68 @@
+#pragma once
+////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2010-2015, University of Washington and Battelle
+// Memorial Institute.  All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//     * Redistributions of source code must retain the above
+//       copyright notice, this list of conditions and the following
+//       disclaimer.
+//     * Redistributions in binary form must reproduce the above
+//       copyright notice, this list of conditions and the following
+//       disclaimer in the documentation and/or other materials
+//       provided with the distribution.
+//     * Neither the name of the University of Washington, Battelle
+//       Memorial Institute, or the names of their contributors may be
+//       used to endorse or promote products derived from this
+//       software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+// UNIVERSITY OF WASHINGTON OR BATTELLE MEMORIAL INSTITUTE BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+// OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+// BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+// USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+// DAMAGE.
+////////////////////////////////////////////////////////////////////////
+
+namespace Grappa {
+
+/// Operations used for raw communicator atomic operations and collectives.
+/// (compared to MPI, just missing maxloc, minloc)
+/// TODO: reconcile with Collective.hpp
+namespace op {
+
+template< typename T > struct max         { T operator()( const T & lhs, const T & rhs ) const { return std::max(lhs, rhs); } };
+template< typename T > struct min         { T operator()( const T & lhs, const T & rhs ) const { return std::min(lhs, rhs); } };
+
+template< typename T > struct plus        { T operator()( const T & lhs, const T & rhs ) const { return  lhs +   rhs;       } };
+template< typename T > struct sum         { T operator()( const T & lhs, const T & rhs ) const { return  lhs +   rhs;       } };
+
+template< typename T > struct prod        { T operator()( const T & lhs, const T & rhs ) const { return  lhs *   rhs;       } };
+
+template< typename T > struct bitwise_and { T operator()( const T & lhs, const T & rhs ) const { return  lhs &   rhs;       } };
+template< typename T > struct bitwise_or  { T operator()( const T & lhs, const T & rhs ) const { return  lhs |   rhs;       } };
+template< typename T > struct bitwise_xor { T operator()( const T & lhs, const T & rhs ) const { return  lhs ^   rhs;       } };
+
+template< typename T > struct logical_and { T operator()( const T & lhs, const T & rhs ) const { return  lhs &&  rhs;       } };
+template< typename T > struct logical_or  { T operator()( const T & lhs, const T & rhs ) const { return  lhs ||  rhs;       } };
+template< typename T > struct logical_xor { T operator()( const T & lhs, const T & rhs ) const { return !lhs != !rhs;       } };
+
+template< typename T > struct replace     { T operator()( const T & lhs, const T & rhs ) const { return          rhs;       } };
+
+template< typename T > struct no_op       { T operator()( const T & lhs, const T & rhs ) const { return  lhs        ;       } };
+
+// these need to be fixed to handle lhs.first == rhs.first properly
+//template< typename T > struct maxloc      { T operator()( const T & lhs, const T & rhs ) const { return lhs.first > rhs.first ? lhs : rhs; } };
+//template< typename T > struct minloc      { T operator()( const T & lhs, const T & rhs ) const { return lhs.first < rhs.first ? lhs : rhs; } };
+}
+
+} // namespace Grappa

system/Collective_tests.cpp

@@ -39,7 +39,7 @@
 #include "GlobalAllocator.hpp"
 #include "Addressing.hpp"
 
-// Tests the functions in Collective.hpp
+// Tests the functions in Collective.hpp, as well as raw MPI collectives from the communicator
 
 BOOST_AUTO_TEST_SUITE( Collective_tests );
 
@@ -116,7 +116,76 @@ BOOST_AUTO_TEST_CASE( test1 ) {
 
     auto total = Grappa::sum_all_cores([]{ return global_x; });
     CHECK_EQ(total, cores());
-
+
+    Grappa::on_all_cores([]{
+
+      { // test broadcast
+        int64_t other[ 8 ] = { 0 };
+        if( Grappa::mycore() == 0 ) {
+          for( int i = 0; i < 8; ++i ) {
+            other[i] = i;
+          }
+        } else {
+          for( int i = 0; i < 8; ++i ) {
+            BOOST_CHECK_EQUAL( other[i], 0 );
+          }
+        }
+
+        Grappa::spmd::blocking::broadcast( &other[0], 8, 0 );
+
+        for( int i = 0; i < 8; ++i ) {
+          BOOST_CHECK_EQUAL( other[i], i );
+        }
+      }
+
+      { // test reduce
+        int64_t other[ 8 ];
+        for( int i = 0; i < 8; ++i ) {
+          other[i] = Grappa::mycore() * 8 + i;
+        }
+
+        Grappa::spmd::blocking::reduce( &other[0], std::greater<int64_t>(), 8 );
+
+        if( Grappa::mycore() == 0 ) {
+          for( int i = 0; i < 8; ++i ) {
+            BOOST_CHECK_EQUAL( other[i], 8 * (Grappa::cores()-1) + i );
+          }
+        }
+      }
+
+      { // test allreduce
+        int64_t other[ 8 ];
+        for( int i = 0; i < 8; ++i ) {
+          other[i] = Grappa::mycore() * 8 + i;
+        }
+
+        Grappa::spmd::blocking::allreduce( &other[0], std::greater<int64_t>(), 8 );
+
+        for( int i = 0; i < 8; ++i ) {
+          BOOST_CHECK_EQUAL( other[i], 8 * (Grappa::cores()-1) + i );
+        }
+      }
+
+      { // test user-defined reduction
+        int64_t other[ 8 ];
+        for( int i = 0; i < 8; ++i ) {
+          other[i] = Grappa::mycore() * 8 + i;
+        }
+
+        auto op = [] (const int64_t & lhs, const int64_t & rhs) -> int64_t {
+          return lhs > rhs ? lhs : rhs;
+        };
+        Grappa::spmd::blocking::reduce( &other[0], op, 8 );
+
+        if( Grappa::mycore() == 0 ) {
+          for( int i = 0; i < 8; ++i ) {
+            BOOST_CHECK_EQUAL( other[i], 8 * (Grappa::cores()-1) + i );
+          }
+        }
+      }
+
+    });
+
   });
   Grappa::finalize();
 }

system/Communicator.cpp

@@ -215,6 +215,11 @@ void Communicator::init( int * argc_p, char ** argv_p[] ) {
   MPI_CHECK( MPI_Comm_set_errhandler( grappa_comm, MPI_ERRORS_RETURN ) );
 
 
+  MPI_CHECK( MPI_Barrier( grappa_comm ) );
+}
+
+
+void Communicator::activate() {
   MPI_CHECK( MPI_Barrier( grappa_comm ) );
 
   // initialize masks
@@ -226,10 +231,6 @@ void Communicator::init( int * argc_p, char ** argv_p[] ) {
   sends = new CommunicatorContext[ 1 << FLAGS_log2_concurrent_sends ];
 
   MPI_CHECK( MPI_Barrier( grappa_comm ) );
-}
-
-
-void Communicator::activate() {
 
   for( int i = 0; i < (1 << FLAGS_log2_concurrent_sends); ++i ) {
     char * buf;
@@ -341,6 +342,7 @@ void Communicator::post_external_send( CommunicatorContext * c,
 }
 
 void Communicator::post_receive( CommunicatorContext * c ) {
+  DCHECK_GT( c->size, 0 ) << "Posting buffer of size zero?";
   MPI_CHECK( MPI_Irecv( c->buf, c->size, MPI_BYTE, MPI_ANY_SOURCE, MPI_ANY_TAG, grappa_comm, &c->request ) );
   DVLOG(6) << "Posted receive " << c << " with buf " << c->buf << " callback " << (void*) c->callback;
 }