diff --git a/.gitignore b/.gitignore index 86a93f47..349cc91d 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ docs/node_modules docs/.vitepress/cache docs/.vitepress/dist docs/.DS_Store -docs/package-lock.json \ No newline at end of file +docs/package-lock.json +*.tif \ No newline at end of file diff --git a/Project.toml b/Project.toml index a549797f..2b87892d 100644 --- a/Project.toml +++ b/Project.toml @@ -36,7 +36,7 @@ CFTime = "0.0, 0.1" DataStructures = "0.17, 0.18" DimensionalData = "0.27, 0.28" DiskArrayTools = "0.1" -DiskArrays = "0.3,0.4" +DiskArrays = "0.3, 0.4" DocStringExtensions = "0.8, 0.9" Glob = "1.3" Interpolations = "0.12, 0.13, 0.14, 0.15" diff --git a/docs/src/UserGuide/faq.md b/docs/src/UserGuide/faq.md index 96ea356c..4895303a 100644 --- a/docs/src/UserGuide/faq.md +++ b/docs/src/UserGuide/faq.md @@ -313,6 +313,12 @@ nothing # hide ds = YAXArrays.Dataset(; (keylist .=> varlist)...) ```` +::: warning + +You will not be able to save this dataset, first you will need to rename those `dimensions` with the `same name` but different values. + +::: + ## Ho do I construct a `Dataset` from a TimeArray In this section we will use `MarketData.jl` and `TimeSeries.jl` to simulate some stocks. diff --git a/docs/src/UserGuide/read.md b/docs/src/UserGuide/read.md index 9b621219..cf4024bc 100644 --- a/docs/src/UserGuide/read.md +++ b/docs/src/UserGuide/read.md @@ -54,5 +54,4 @@ using Downloads: download path = download("https://github.com/yeesian/ArchGDALDatasets/raw/307f8f0e584a39a050c042849004e6a2bd674f99/gdalworkshop/world.tif", "world.tif") ds = open_dataset(path) -nothing ```` \ No newline at end of file diff --git a/src/DAT/DAT.jl b/src/DAT/DAT.jl index e18a594a..706324fa 100644 --- a/src/DAT/DAT.jl +++ b/src/DAT/DAT.jl @@ -796,14 +796,19 @@ function getbackend(oc, ispar, max_cache) outsize = sizeof(elementtype) * (length(oc.allAxes) > 0 ? prod(map(length, oc.allAxes)) : 1) rt = oc.desc.backend - if rt == :auto - if ispar[] || outsize > max_cache - rt = :zarr + ispath = get(oc.desc.backendargs, :path, nothing) + + b = if rt == :auto && !isnothing(ispath) + YAXArrayBase.backendfrompath(ispath) + elseif rt == :auto + if ispar[] || outsize > max_cache + YAXArrayBase.backendlist[:zarr] + else + YAXArrayBase.backendlist[:array] + end else - rt = :array - end + YAXArrayBase.backendlist[Symbol(rt)] # Handle non-auto rt case end - b = YAXArrayBase.backendlist[Symbol(rt)] if !allow_parallel_write(b) ispar[] = false end @@ -849,7 +854,10 @@ function generateOutCube( newsize = map(length, oc.allAxes) outar = Array{elementtype}(undef, newsize...) fill!(outar,_zero(elementtype)) - oc.cube = YAXArray(tuple(oc.allAxes...), outar) + # @show oc.desc.backendargs[:properties] + properties = get(oc.desc.backendargs, :properties, Dict{String, Any}()) + # @show properties + oc.cube = YAXArray(tuple(oc.allAxes...), outar, properties) # properties ? include properties! oc.cube_unpermuted = oc.cube end _zero(T) = zero(T) diff --git a/src/DatasetAPI/Datasets.jl b/src/DatasetAPI/Datasets.jl index 4cbf06e8..766bc560 100644 --- a/src/DatasetAPI/Datasets.jl +++ b/src/DatasetAPI/Datasets.jl @@ -678,7 +678,7 @@ function savecube( if chunks !== nothing error("Setting chunks in savecube is not supported anymore. Rechunk using `setchunks` before saving. ") end - + ds = to_dataset(c; layername, datasetaxis) ds = savedataset(ds; path, max_cache, driver, overwrite, append,skeleton, writefac, kwargs...) Cube(ds, joinname = datasetaxis) @@ -718,7 +718,7 @@ function createdataset( properties = Dict{String,Any}(), globalproperties = Dict{String,Any}(), datasetaxis = "Variable", - layername = "layer", + layername = get(properties, "name", "layer"), kwargs..., ) if persist === nothing diff --git a/test/Datasets/datasets.jl b/test/Datasets/datasets.jl index e871c0f5..e7789cb0 100644 --- a/test/Datasets/datasets.jl +++ b/test/Datasets/datasets.jl @@ -410,6 +410,76 @@ end end +@testset "Saving, OutDims" begin + using YAXArrays, Zarr, NetCDF, ArchGDAL + using Dates + + flolat(lo, la, t) = (lo + la + Dates.dayofyear(t)) + flola(lo, la) = lo + la + 1 + + function g(xout, lo, la, t) + xout .= flolat.(lo, la, t) + end + + function g2d(xout, lo, la) + xout .= flola.(lo, la) + end + + lon = YAXArray(Dim{:lon}(range(1, 15))) + lat = YAXArray(Dim{:lat}(range(1, 10))) + tspan = Date("2022-01-01"):Day(1):Date("2022-01-30") + time = YAXArray(Dim{:time}(tspan)) + + properties = Dict{String, Any}("name" => "out_array") + + gen_cube = mapCube(g, (lon, lat, time); + indims = (InDims(), InDims(), InDims("time")), + outdims = OutDims("time"; properties, + outtype = Float32) + # max_cache=1e9 + ) + + gen_cube2d = mapCube(g2d, (lon, lat); + indims = (InDims(), InDims()), + outdims = OutDims(; outtype = Float32) + # max_cache=1e9 + ) + properties = Dict{String, Any}("name" => "out_zarr") + # test saves, zarr + mapCube(g, (lon, lat, time); + indims = (InDims(), InDims(), InDims("time")), + outdims = OutDims("time"; overwrite=true, path="my_gen_cube.zarr", + properties, + outtype = Float32) + # max_cache=1e9 + ) + ds_zarr = open_dataset("my_gen_cube.zarr") + # test saves, nc + properties = Dict{String, Any}("name" => "out_nc") + mapCube(g, (lon, lat, time); + indims = (InDims(), InDims(), InDims("time")), + outdims = OutDims("time"; overwrite=true, path="my_gen_cube.nc", + properties, + outtype = Float32) + # max_cache=1e9 + ) + mapCube(g, (lon, lat, time); + indims = (InDims(), InDims(), InDims("time")), + outdims = OutDims("time"; overwrite=true, path="my_gen_cube_no_p.nc", + outtype = Float32) + # max_cache=1e9 + ) + ds_nc = open_dataset("my_gen_cube.nc") + ds_nc_no_p = open_dataset("my_gen_cube_no_p.nc") + + # TODO: fix tif for general inputs, so that writing also works. + + @test gen_cube.properties["name"] == "out_array" + @test gen_cube.data[:,:,:] == ds_zarr["out_zarr"].data[:,:,:] + @test gen_cube.data[:,:,:] == ds_nc["out_nc"].data[:,:,:] + @test gen_cube.data[:,:,:] == ds_nc_no_p["layer"].data[:,:,:] +end + @testset "Caching" begin using YAXArrays.Cubes.DiskArrays.TestTypes using YAXArrays.Cubes: DiskArrays diff --git a/test/Project.toml b/test/Project.toml index a7486127..0810e4b4 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -1,4 +1,5 @@ [deps] +ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3" CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"