[ntuple] create streamer infos from TClass names

Get the streamer info records of class fields and streamer fields from
their meta (TClass) normalized name. Previously, the names came from
the descriptor, which has them already RNTuple normalized.

(cherry picked from commit 1f6a481)

Author: jblomer

tree/ntuple/inc/ROOT/RField.hxx

@@ -252,6 +252,7 @@ public:
    size_t GetAlignment() const final;
    std::uint32_t GetTypeVersion() const final;
    std::uint32_t GetTypeChecksum() const final;
+   TClass *GetClass() const { return fClass; }
    void AcceptVisitor(ROOT::Detail::RFieldVisitor &visitor) const final;
 };
 

tree/ntuple/inc/ROOT/RNTupleDescriptor.hxx

@@ -1599,9 +1599,6 @@ public:
    ///   - Logical columns of piece two
    ///   - ...
    void ShiftAliasColumns(std::uint32_t offset);
-
-   /// Get the streamer info records for custom classes. Currently requires the corresponding dictionaries to be loaded.
-   ROOT::Internal::RNTupleSerializer::StreamerInfoMap_t BuildStreamerInfos() const;
 };
 
 inline RNTupleDescriptor CloneDescriptorSchema(const RNTupleDescriptor &desc)

tree/ntuple/inc/ROOT/RPageStorage.hxx

@@ -521,7 +521,7 @@ public:
 
    /// Updates the descriptor and calls InitImpl() that handles the backend-specific details (file, DAOS, etc.)
    void InitImpl(RNTupleModel &model) final;
-   void UpdateSchema(const ROOT::Internal::RNTupleModelChangeset &changeset, ROOT::NTupleSize_t firstEntry) final;
+   void UpdateSchema(const ROOT::Internal::RNTupleModelChangeset &changeset, ROOT::NTupleSize_t firstEntry) override;
    void UpdateExtraTypeInfo(const ROOT::RExtraTypeInfoDescriptor &extraTypeInfo) final;
 
    /// Initialize sink based on an existing descriptor and fill into the descriptor builder, optionally copying over

tree/ntuple/inc/ROOT/RPageStorageFile.hxx

@@ -65,6 +65,10 @@ private:
    std::unique_ptr<ROOT::Internal::RNTupleFileWriter> fWriter;
    /// Number of bytes committed to storage in the current cluster
    std::uint64_t fNBytesCurrentCluster = 0;
+   /// On UpdateSchema(), the new class fields register the corresponding streamer info here so that the
+   /// streamer info records in the file can be properly updated on dataset commit
+   ROOT::Internal::RNTupleSerializer::StreamerInfoMap_t fInfosOfClassFields;
+
    RPageSinkFile(std::string_view ntupleName, const ROOT::RNTupleWriteOptions &options);
 
    /// We pass bytesPacked so that TFile::ls() reports a reasonable value for the compression ratio of the corresponding
@@ -98,6 +102,8 @@ public:
    RPageSinkFile(RPageSinkFile &&) = default;
    RPageSinkFile &operator=(RPageSinkFile &&) = default;
    ~RPageSinkFile() override;
+
+   void UpdateSchema(const ROOT::Internal::RNTupleModelChangeset &changeset, ROOT::NTupleSize_t firstEntry) final;
 }; // class RPageSinkFile
 
 // clang-format off

tree/ntuple/src/RNTupleDescriptor.cxx

@@ -23,7 +23,6 @@
 
 #include <RZip.h>
 #include <TError.h>
-#include <TVirtualStreamerInfo.h>
 
 #include <algorithm>
 #include <cstdint>
@@ -1359,48 +1358,6 @@ void ROOT::Internal::RNTupleDescriptorBuilder::ReplaceExtraTypeInfo(RExtraTypeIn
       fDescriptor.fExtraTypeInfoDescriptors.emplace_back(std::move(extraTypeInfoDesc));
 }
 
-RNTupleSerializer::StreamerInfoMap_t ROOT::Internal::RNTupleDescriptorBuilder::BuildStreamerInfos() const
-{
-   RNTupleSerializer::StreamerInfoMap_t streamerInfoMap;
-   const auto &desc = GetDescriptor();
-
-   std::function<void(const RFieldDescriptor &)> fnWalkFieldTree;
-   fnWalkFieldTree = [&desc, &streamerInfoMap, &fnWalkFieldTree](const RFieldDescriptor &fieldDesc) {
-      if (fieldDesc.IsCustomClass()) {
-         // Add streamer info for this class to streamerInfoMap
-         auto cl = TClass::GetClass(fieldDesc.GetTypeName().c_str());
-         if (!cl) {
-            throw RException(R__FAIL(std::string("cannot get TClass for ") + fieldDesc.GetTypeName()));
-         }
-         auto streamerInfo = cl->GetStreamerInfo(fieldDesc.GetTypeVersion());
-         if (!streamerInfo) {
-            throw RException(R__FAIL(std::string("cannot get streamerInfo for ") + fieldDesc.GetTypeName()));
-         }
-         streamerInfoMap[streamerInfo->GetNumber()] = streamerInfo;
-      }
-
-      // Recursively traverse sub fields
-      for (const auto &subFieldDesc : desc.GetFieldIterable(fieldDesc)) {
-         fnWalkFieldTree(subFieldDesc);
-      }
-   };
-
-   fnWalkFieldTree(desc.GetFieldZero());
-
-   // Add the streamer info records from streamer fields: because of runtime polymorphism we may need to add additional
-   // types not covered by the type names stored in the field headers
-   for (const auto &extraTypeInfo : desc.GetExtraTypeInfoIterable()) {
-      if (extraTypeInfo.GetContentId() != EExtraTypeInfoIds::kStreamerInfo)
-         continue;
-      // Ideally, we would avoid deserializing the streamer info records of the streamer fields that we just serialized.
-      // However, this happens only once at the end of writing and only when streamer fields are used, so the
-      // preference here is for code simplicity.
-      streamerInfoMap.merge(RNTupleSerializer::DeserializeStreamerInfos(extraTypeInfo.GetContent()).Unwrap());
-   }
-
-   return streamerInfoMap;
-}
-
 ROOT::RClusterDescriptor::RColumnRangeIterable ROOT::RClusterDescriptor::GetColumnRangeIterable() const
 {
    return RColumnRangeIterable(*this);

tree/ntuple/src/RPageStorageFile.cxx

@@ -29,6 +29,7 @@
 #include <RVersion.h>
 #include <TDirectory.h>
 #include <TError.h>
+#include <TVirtualStreamerInfo.h>
 
 #include <algorithm>
 #include <cstdio>
@@ -84,6 +85,37 @@ void ROOT::Internal::RPageSinkFile::InitImpl(unsigned char *serializedHeader, st
    fWriter->WriteNTupleHeader(zipBuffer.get(), szZipHeader, length);
 }
 
+void ROOT::Internal::RPageSinkFile::UpdateSchema(const ROOT::Internal::RNTupleModelChangeset &changeset,
+                                                 ROOT::NTupleSize_t firstEntry)
+{
+   RPagePersistentSink::UpdateSchema(changeset, firstEntry);
+
+   auto fnAddStreamerInfo = [this](const ROOT::RFieldBase *field) {
+      const TClass *cl = nullptr;
+      if (auto classField = dynamic_cast<const RClassField *>(field)) {
+         cl = classField->GetClass();
+      } else if (auto streamerField = dynamic_cast<const RStreamerField *>(field)) {
+         cl = streamerField->GetClass();
+      }
+      if (!cl)
+         return;
+
+      auto streamerInfo = cl->GetStreamerInfo(field->GetTypeVersion());
+      if (!streamerInfo) {
+         throw RException(R__FAIL(std::string("cannot get streamerInfo for ") + cl->GetName() + " [" +
+                                  std::to_string(field->GetTypeVersion()) + "]"));
+      }
+      fInfosOfClassFields[streamerInfo->GetNumber()] = streamerInfo;
+   };
+
+   for (const auto field : changeset.fAddedFields) {
+      fnAddStreamerInfo(field);
+      for (const auto &subField : *field) {
+         fnAddStreamerInfo(&subField);
+      }
+   }
+}
+
 inline ROOT::RNTupleLocator
 ROOT::Internal::RPageSinkFile::WriteSealedPage(const RPageStorage::RSealedPage &sealedPage, std::size_t bytesPacked)
 {
@@ -243,7 +275,18 @@ ROOT::Internal::RPageSinkFile::CommitClusterGroupImpl(unsigned char *serializedP
 
 void ROOT::Internal::RPageSinkFile::CommitDatasetImpl(unsigned char *serializedFooter, std::uint32_t length)
 {
-   fWriter->UpdateStreamerInfos(fDescriptorBuilder.BuildStreamerInfos());
+   // Add the streamer info records from streamer fields: because of runtime polymorphism we may need to add additional
+   // types not covered by the type names of the class fields
+   for (const auto &extraTypeInfo : fDescriptorBuilder.GetDescriptor().GetExtraTypeInfoIterable()) {
+      if (extraTypeInfo.GetContentId() != EExtraTypeInfoIds::kStreamerInfo)
+         continue;
+      // Ideally, we would avoid deserializing the streamer info records of the streamer fields that we just serialized.
+      // However, this happens only once at the end of writing and only when streamer fields are used, so the
+      // preference here is for code simplicity.
+      fInfosOfClassFields.merge(RNTupleSerializer::DeserializeStreamerInfos(extraTypeInfo.GetContent()).Unwrap());
+   }
+   fWriter->UpdateStreamerInfos(fInfosOfClassFields);
+
    auto bufFooterZip = MakeUninitArray<unsigned char>(length);
    auto szFooterZip =
       RNTupleCompressor::Zip(serializedFooter, length, GetWriteOptions().GetCompression(), bufFooterZip.get());

tree/ntuple/test/CMakeLists.txt

@@ -29,12 +29,8 @@ ROOT_GENERATE_DICTIONARY(RXTupleDict ${CMAKE_CURRENT_SOURCE_DIR}/RXTuple.hxx
                        MODULE ntuple_compat
                        LINKDEF RXTupleLinkDef.h
                        DEPENDENCIES RIO)
-ROOT_ADD_GTEST(ntuple_descriptor ntuple_descriptor.cxx LIBRARIES ROOTNTuple CustomStruct)
-ROOT_GENERATE_DICTIONARY(RNTupleDescriptorDict ${CMAKE_CURRENT_SOURCE_DIR}/RNTupleDescriptorDict.hxx
-                       MODULE ntuple_descriptor
-                       LINKDEF RNTupleDescriptorLinkDef.h
-                       OPTIONS -inlineInputHeader
-                       DEPENDENCIES RIO CustomStruct)
+ROOT_ADD_GTEST(ntuple_descriptor ntuple_descriptor.cxx LIBRARIES ROOTNTuple)
+
 ROOT_ADD_GTEST(ntuple_endian ntuple_endian.cxx LIBRARIES ROOTNTuple)
 if(NOT MSVC)
   # These unit tests rely on fork(), which is not available on Windows.

tree/ntuple/test/CustomStruct.hxx

@@ -11,6 +11,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <functional>
+#include <map>
 #include <random>
 #include <string>
 #include <variant>

tree/ntuple/test/CustomStructLinkDef.h

@@ -23,6 +23,9 @@
 #pragma link C++ class IOConstructor+;
 #pragma link C++ class LowPrecisionFloats+;
 
+#pragma link C++ class std::map<int, CustomStruct>+ ;
+#pragma link C++ class std::map<int, float>+ ;
+
 #pragma link C++ class EdmWrapper<CustomStruct> +;
 #pragma link C++ class EdmHash < 1> + ;
 #pragma link C++ class EdmWrapper<long long>+;