Struct array writing (JuliaIO#207)

Allow struct array writing in mat files via the `MatlabStructArray` type. 

Includes:
* a new `MatlabStructArray` type with all related functionality and docs
* a new `MatlabClassObject` type and compatibility with `MatlabStructArray`
* a fix for single `Char` writing

Author: matthijscox

Project.toml

@@ -1,6 +1,6 @@
 name = "MAT"
 uuid = "23992714-dd62-5051-b70f-ba57cb901cac"
-version = "0.10.7"
+version = "0.11.0"
 
 [deps]
 BufferedStreams = "e1450e63-4bb3-523b-b2a4-4ffa8c0fd77d"

docs/Project.toml

@@ -1,8 +1,7 @@
 [deps]
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
-Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
+MAT = "23992714-dd62-5051-b70f-ba57cb901cac"
 
 [compat]
 Documenter = "1"
-Literate = "2"

docs/make.jl

@@ -1,47 +1,13 @@
 execute = isempty(ARGS) || ARGS[1] == "run"
 
-org, reps = :JuliaIO, :MAT
-eval(:(using $reps))
+org, repo = :JuliaIO, :MAT
+eval(:(using $repo))
 using Documenter
-using Literate
 
 # https://juliadocs.github.io/Documenter.jl/stable/man/syntax/#@example-block
 ENV["GKSwstype"] = "100"
 ENV["GKS_ENCODING"] = "utf-8"
 
-# generate examples using Literate
-lit = joinpath(@__DIR__, "lit")
-src = joinpath(@__DIR__, "src")
-gen = joinpath(@__DIR__, "src/generated")
-
-base = "$org/$reps.jl"
-repo_root_url =
-    "https://github.com/$base/blob/main/docs/lit/examples"
-nbviewer_root_url =
-    "https://nbviewer.org/github/$base/tree/gh-pages/dev/generated/examples"
-binder_root_url =
-    "https://mybinder.org/v2/gh/$base/gh-pages?filepath=dev/generated/examples"
-
-
-repo = eval(:($reps))
-DocMeta.setdocmeta!(repo, :DocTestSetup, :(using $reps); recursive=true)
-
-for (root, _, files) in walkdir(lit), file in files
-    splitext(file)[2] == ".jl" || continue # process .jl files only
-    ipath = joinpath(root, file)
-    opath = splitdir(replace(ipath, lit => gen))[1]
-    Literate.markdown(ipath, opath; documenter = execute, # run examples
-        repo_root_url, nbviewer_root_url, binder_root_url)
-    Literate.notebook(ipath, opath; execute = false, # no-run notebooks
-        repo_root_url, nbviewer_root_url, binder_root_url)
-end
-
-
-# Documentation structure
-ismd(f) = splitext(f)[2] == ".md"
-pages(folder) =
-    [joinpath("generated/", folder, f) for f in readdir(joinpath(gen, folder)) if ismd(f)]
-
 isci = get(ENV, "CI", nothing) == "true"
 
 format = Documenter.HTML(;
@@ -52,14 +18,14 @@ format = Documenter.HTML(;
 )
 
 makedocs(;
-    modules = [repo],
+    modules = [MAT],
     authors = "Contributors",
     sitename = "$repo.jl",
     format,
     pages = [
         "Home" => "index.md",
+        "Object Arrays" => "object_arrays.md",
         "Methods" => "methods.md",
-#       "Examples" => pages("examples")
     ],
     warnonly = [:missing_docs,],
 )

docs/src/index.md

@@ -10,3 +10,65 @@ This Julia package
 [(MAT.jl)](https://github.com/JuliaIO/MAT.jl)
 provides tools
 for reading and writing MATLAB format data files in Julia.
+
+## Basic Usage
+
+To read a single variable from a MAT file (compressed files are detected and handled automatically):
+
+```julia
+using MAT
+file = matopen("matfile.mat")
+read(file, "varname") # note that this does NOT introduce a variable ``varname`` into scope
+close(file)
+```
+
+To write a variable to a MAT file:
+
+```julia
+file = matopen("matfile.mat", "w")
+write(file, "varname", variable)
+close(file)
+```
+
+To read all variables from a MAT file as a Dict:
+
+```julia
+vars = matread("matfile.mat")
+```
+
+To write a Dict to a MAT file, using its keys as variable names.
+The `compress` argument is optional, and compression is off by default:
+
+```julia
+matwrite("matfile.mat", Dict(
+	"myvar1" => 0,
+	"myvar2" => 1
+); compress = true)
+```
+
+To write in MATLAB v4 format:
+
+```julia
+matwrite("matfile.mat", Dict(
+	"myvar1" => 0,
+	"myvar2" => 1
+);version="v4")
+```
+
+To get a list of variable names in a MAT file:
+
+```julia
+file = matopen("matfile.mat")
+varnames = keys(file)
+close(file)
+```
+
+To check for the presence of a variable name in a MAT file:
+
+```julia
+file = matopen("matfile.mat")
+if haskey(file, "variable")
+    # something
+end
+close(file)
+```

docs/src/methods.md

@@ -6,5 +6,5 @@
 ## Methods usage
 
 ```@autodocs
-Modules = [MAT]
+Modules = [MAT, MAT.MAT_types]
 ```

docs/src/object_arrays.md

@@ -0,0 +1,117 @@
+# Objects and struct arrays
+
+To better handle special cases we have these types since MAT 0.11:
+* [`MatlabStructArray`](@ref MatlabStructArray)
+* [`MatlabClassObject`](@ref MatlabClassObject)
+
+## Struct arrays vs Cell arrays
+
+Cell arrays are written for `Array{Any}` or any other unsupported element type:
+
+```julia
+sarr = Any[
+    Dict("x"=>1.0, "y"=>2.0),
+    Dict("x"=>3.0, "y"=>4.0)
+]
+matwrite("matfile.mat", Dict("cell" => sarr))
+
+```
+
+Inside MATLAB you will find:
+
+```matlab
+>> load('matfile.mat')
+>> cell
+
+cell =
+
+  2×1 cell array
+
+    {1×1 struct}
+    {1×1 struct}
+```
+
+Read and write behavior for struct arrays is different. For struct arrays we use the `MatlabStructArray` type. You can also write with MAT.jl using Dict arrays `AbstractArray{<:AbstractDict}` if all the Dicts have equal keys, which will automatically convert internally to `MatlabStructArray`.
+
+```julia
+sarr = Dict{String, Any}[
+    Dict("x"=>1.0, "y"=>2.0),
+    Dict("x"=>3.0, "y"=>4.0)
+]
+matwrite("matfile.mat", Dict("s" => sarr))
+# which is the same as:
+matwrite("matfile.mat", Dict("s" => MatlabStructArray(sarr)))
+# which is the same as:
+matwrite("matfile.mat", Dict("s" => MatlabStructArray(["x", "y"], [[1.0, 3.0], [2.0, 4.0]])))
+```
+
+Now you'll find the following inside MATLAB:
+
+```matlab
+>> load('matfile.mat')
+>> s
+
+s =
+
+[2x1 struct, 576 bytes]
+x: 1
+y: 2
+```
+
+Note that when you read the file again, you'll find the `MatlabStructArray`, which you can convert back to the Dict array with `Array`:
+
+```julia
+julia> sarr = matread("matfile.mat")["struct_array"]
+MatlabStructArray{1} with 2 columns:
+ "x": Any[1.0, 3.0]
+ "y": Any[2.0, 4.0]
+
+julia> sarr["x"]
+2-element Vector{Any}:
+ 1.0
+ 3.0
+
+julia> Array(sarr)
+2-element Vector{Dict{String, Any}}:
+ Dict("x" => 1.0, "y" => 2.0)
+ Dict("x" => 3.0, "y" => 4.0)
+
+```
+
+Note that before v0.11 MAT.jl will read struct arrays as a Dict with concatenated arrays in the fields/keys, which is equal to `Dict(sarr)`.
+
+## Object Arrays
+
+You can write an old class object with the `MatlabClassObject` and arrays of objects with `MatlabStructArray` by providing the class name. These are also the types you obtain when you read files.
+
+Write a single class object:
+```julia
+d = Dict("foo" => 5.0)
+obj = MatlabClassObject(d, "TestClassOld")
+matwrite("matfile.mat", Dict("tc_old" => obj))
+```
+
+A class object array
+```julia
+class_array = MatlabStructArray(["foo"], [[5.0, "bar"]], "TestClassOld")
+matwrite("matfile.mat", Dict("class_array" => class_array))
+```
+
+Also a class object array, but will be converted to `MatlabStructArray` internally:
+```julia
+class_array = MatlabClassObject[
+    MatlabClassObject(Dict("foo" => 5.0), "TestClassOld"),
+    MatlabClassObject(Dict("foo" => "bar"), "TestClassOld")
+]
+matwrite("matfile.mat", Dict("class_array" => class_array))
+```
+
+A cell array:
+```julia
+cell_array = Any[
+    MatlabClassObject(Dict("foo" => 5.0), "TestClassOld"),
+    MatlabClassObject(Dict("a" => "bar"), "AnotherClass")
+]
+matwrite("matfile.mat", Dict("cell_array" => cell_array))
+```
+

src/MAT.jl

@@ -26,13 +26,17 @@ module MAT
 
 using HDF5, SparseArrays
 
+include("MAT_types.jl")
+using .MAT_types
+
 include("MAT_HDF5.jl")
 include("MAT_v5.jl")
 include("MAT_v4.jl")
 
 using .MAT_HDF5, .MAT_v5, .MAT_v4
 
 export matopen, matread, matwrite, @read, @write
+export MatlabStructArray, MatlabClassObject
 
 # Open a MATLAB file
 const HDF5_HEADER = UInt8[0x89, 0x48, 0x44, 0x46, 0x0d, 0x0a, 0x1a, 0x0a]
@@ -140,47 +144,37 @@ end
 
 # Write a dict to a MATLAB file
 """
-    matwrite(filename, d::Dict; compress::Bool = false, version::String)
+    matwrite(filename, d::Dict; compress::Bool = false, version::String = "v7.3")
 
 Write a dictionary containing variable names as keys and values as values
 to a Matlab file, opening and closing it automatically.
 """
-function matwrite(filename::AbstractString, dict::AbstractDict{S, T}; compress::Bool = false, version::String ="") where {S, T}
-
+function matwrite(filename::AbstractString, dict::AbstractDict{S, T}; compress::Bool = false, version::String ="v7.3") where {S, T}
     if version == "v4"
         file = open(filename, "w")
         m = MAT_v4.Matlabv4File(file, false)
-        try
-            for (k, v) in dict
-                local kstring
-                try
-                    kstring = ascii(convert(String, k))
-                catch x
-                    error("matwrite requires a Dict with ASCII keys")
-                end
-                write(m, kstring, v)
-            end
-        finally
-            close(file)
-        end
-
-    else
-        
+        _write_dict(m, dict)
+    elseif version == "v7.3"
         file = matopen(filename, "w"; compress = compress)
-        try
-            for (k, v) in dict
-                local kstring
-                try
-                    kstring = ascii(convert(String, k))
-                catch x
-                    error("matwrite requires a Dict with ASCII keys")
-                end
-                write(file, kstring, v)
+        _write_dict(file, dict)
+    else
+        error("writing for \"$(version)\" is not supported")
+    end
+end
+
+function _write_dict(fileio, dict::AbstractDict)
+    try
+        for (k, v) in dict
+            local kstring
+            try
+                kstring = ascii(convert(String, k))
+            catch x
+                error("matwrite requires a Dict with ASCII keys")
             end
-        finally
-            close(file)
+            write(fileio, kstring, v)
         end
-
+    finally
+        close(fileio)
     end
 end