Skip to content

Commit

Permalink
fix LoD generation for part indices; add part index data to ~100 exis…
Browse files Browse the repository at this point in the history 
…ting tests
  • Loading branch information
@stefanmaierhofer
stefanmaierhofer committed Oct 6, 2023
1 parent 0352219 commit 6d9e5a9
Show file tree
Hide file tree
Showing 14 changed files with 211 additions and 33 deletions.
12 changes: 9 additions & 3 deletions src/Aardvark.Algodat.Tests/DeleteTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,9 @@ public static PointSet CreateRandomPointsInUnitCube(int n, int splitLimit)
.WithKey("test")
.WithOctreeSplitLimit(splitLimit)
;
return PointCloud.Chunks(new Chunk(ps), config);
var chunk = new Chunk(ps);
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
return PointCloud.Chunks(chunk, config);
}

public static PointSet CreateRegularPointsInUnitCube(int n, int splitLimit)
Expand All @@ -59,7 +61,9 @@ public static PointSet CreateRegularPointsInUnitCube(int n, int splitLimit)
.WithKey("test")
.WithOctreeSplitLimit(splitLimit)
;
return PointCloud.Chunks(new Chunk(ps), config);
var chunk = new Chunk(ps);
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
return PointCloud.Chunks(chunk, config);
}

public static PointSet CreateRandomClassifiedPoints(int n, int splitLimit)
Expand All @@ -77,7 +81,9 @@ public static PointSet CreateRandomClassifiedPoints(int n, int splitLimit)
.WithKey("testaa")
.WithOctreeSplitLimit(splitLimit)
;
return PointCloud.Chunks(new Chunk(ps, null, null, null, ks, null, null), config);
var chunk = new Chunk(ps, null, null, null, ks, null, null);
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
return PointCloud.Chunks(chunk, config);
}

[Test]
Expand Down
10 changes: 9 additions & 1 deletion src/Aardvark.Algodat.Tests/ImportTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ public void CanImportChunkWithoutColor()
.WithKey("test")
.WithOctreeSplitLimit(10)
;

if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
var pointcloud = PointCloud.Chunks(chunk, config);
Assert.IsTrue(pointcloud.PointCount == 100);
}
Expand All @@ -102,6 +102,9 @@ public void CanImportChunk_MinDist()
.WithOctreeSplitLimit(10)
.WithMinDist(0.5)
;
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
var pointcloud = PointCloud.Chunks(chunk, config);
Assert.IsTrue(pointcloud.PointCount < 100);
}
Expand All @@ -124,6 +127,7 @@ public void CanImportChunk_Reproject()
.WithOctreeSplitLimit(10)
.WithReproject(xs => xs.Select(x => x += V3d.OIO).ToArray())
;
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
var pointcloud = PointCloud.Chunks(chunk, config);
Assert.IsTrue(pointcloud.BoundingBox == bb + V3d.OIO);
}
Expand All @@ -145,6 +149,7 @@ public void CanImport_WithKey()
.WithMinDist(0.0)
.WithReproject(null)
;
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
var pointcloud = PointCloud.Chunks(chunk, config);
Assert.IsTrue(pointcloud.Id == "test");
}
Expand All @@ -170,6 +175,7 @@ public void CanImportWithKeyAndThenLoadAgainFromStore()
.WithMinDist(0.0)
.WithReproject(null)
;
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
var pointcloud = PointCloud.Chunks(chunk, config);
Assert.IsTrue(pointcloud.Id == key);

Expand All @@ -195,6 +201,7 @@ public void CanImport_WithoutKey()
.WithMinDist(0.0)
.WithReproject(null)
;
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
var pointcloud = PointCloud.Chunks(chunk, config);
Assert.IsTrue(pointcloud.Id != null);
}
Expand All @@ -217,6 +224,7 @@ public void CanImport_DuplicateKey()
.WithReproject(null)
;

if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);

var pointcloud = PointCloud.Chunks(Array.Empty<Chunk>(), config);
Assert.IsTrue(pointcloud.Id != null);
Expand Down
24 changes: 17 additions & 7 deletions src/Aardvark.Algodat.Tests/LodTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ public void LodCreationSetsLodPointCountCell()
var cs = ps.Map(_ => C4b.White);

var pointset = PointSet.Create(
storage, "test", ps.ToList(), cs.ToList(), null, null, null, null, 5000,
storage, "test", ps.ToList(), cs.ToList(), null, null, null, partIndices: 42u, 5000,
generateLod: false, isTemporaryImportNode: true, ct: default
);
pointset.Root.Value.ForEachNode(true, cell =>
Expand Down Expand Up @@ -64,7 +64,7 @@ public void LodPositions()
var cs = ps.Map(_ => C4b.White);

var pointset = PointSet.Create(
storage, "test", ps.ToList(), cs.ToList(), null, null, null, null, 5000,
storage, "test", ps.ToList(), cs.ToList(), null, null, null, partIndices: 42u, 5000,
generateLod: true, isTemporaryImportNode: true, default
);
pointset.Root.Value.ForEachNode(true, cell =>
Expand Down Expand Up @@ -192,7 +192,9 @@ public void HasCentroid()
};

var config = ImportConfig.Default.WithStorage(storage).WithRandomKey();
var n = PointCloud.Chunks(new Chunk(ps), config).Root.Value;
var chunk = new Chunk(ps);
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
var n = PointCloud.Chunks(chunk, config).Root.Value;

Assert.IsTrue(n.HasCentroidLocal);
Assert.IsTrue(n.HasCentroidLocalStdDev);
Expand All @@ -217,7 +219,9 @@ public void HasBoundingBoxExact()
};

var config = ImportConfig.Default.WithStorage(storage).WithRandomKey();
var n = PointCloud.Chunks(new Chunk(ps), config).Root.Value;
var chunk = new Chunk(ps);
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
var n = PointCloud.Chunks(chunk, config).Root.Value;

Assert.IsTrue(n.HasBoundingBoxExactLocal);
Assert.IsTrue(n.BoundingBoxExactLocal == new Box3f(new V3f(-0.4f), new V3f(0.4f)));
Expand All @@ -237,7 +241,9 @@ public void HasTreeDepth()
var ps = new V3d[10].SetByIndex(_ => new V3d(r.NextDouble(), r.NextDouble(), r.NextDouble()));

var config = ImportConfig.Default.WithStorage(storage).WithRandomKey();
var n = PointCloud.Chunks(new Chunk(ps), config).Root.Value;
var chunk = new Chunk(ps);
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
var n = PointCloud.Chunks(chunk, config).Root.Value;

Assert.IsTrue(n.HasMinTreeDepth);
Assert.IsTrue(n.HasMaxTreeDepth);
Expand All @@ -255,7 +261,9 @@ public void HasTreeDepth2()
var ps = new V3d[20000].SetByIndex(_ => new V3d(r.NextDouble(), r.NextDouble(), r.NextDouble()));

var config = ImportConfig.Default.WithStorage(storage).WithRandomKey();
var n = PointCloud.Chunks(new Chunk(ps), config).Root.Value;
var chunk = new Chunk(ps);
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
var n = PointCloud.Chunks(chunk, config).Root.Value;

Assert.IsTrue(n.HasMinTreeDepth);
Assert.IsTrue(n.HasMaxTreeDepth);
Expand Down Expand Up @@ -283,7 +291,9 @@ public void HasPointDistance()
};

var config = ImportConfig.Default.WithStorage(storage).WithRandomKey();
var n = PointCloud.Chunks(new Chunk(ps), config).Root.Value;
var chunk = new Chunk(ps);
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
var n = PointCloud.Chunks(chunk, config).Root.Value;

Assert.IsTrue(n.HasPointDistanceAverage);
Assert.IsTrue(n.HasPointDistanceStandardDeviation);
Expand Down
8 changes: 6 additions & 2 deletions src/Aardvark.Algodat.Tests/PointCloudTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ private static PointSet CreateRandomPointsInUnitCube(int n, int splitLimit)
.WithKey("test")
.WithOctreeSplitLimit(splitLimit)
;
return PointCloud.Chunks(new Chunk(ps), config);
var chunk = new Chunk(ps);
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
return PointCloud.Chunks(chunk, config);
}

private static PointSet CreateRegularPointsInUnitCube(int n, int splitLimit)
Expand All @@ -52,7 +54,9 @@ private static PointSet CreateRegularPointsInUnitCube(int n, int splitLimit)
.WithKey("test")
.WithOctreeSplitLimit(splitLimit)
;
return PointCloud.Chunks(new Chunk(ps), config);
var chunk = new Chunk(ps);
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
return PointCloud.Chunks(chunk, config);
}

[Test]
Expand Down
8 changes: 6 additions & 2 deletions src/Aardvark.Algodat.Tests/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1750,7 +1750,8 @@ internal static void Test_Import_Regression()
{
var filenames = new[]
{
@"W:\Datasets\Vgm\Data\structured_pointclouds\lowergetikum 20230321.e57",
@"W:\Datasets\Vgm\Data\E57\JBs_Haus.e57",
//@"W:\Datasets\Vgm\Data\structured_pointclouds\lowergetikum 20230321.e57",
//@"W:\Datasets\Vgm\Data\structured_pointclouds\JB_Haus_2022_KG.e57",
//@"W:\Datasets\Vgm\Data\2023-02-23_bugreport\KOE1 OG7.e57",
//@"W:\Datasets\unstruk\Christchurch.laz",
Expand Down Expand Up @@ -1806,6 +1807,7 @@ internal static void Test_Import_Regression()
.WithMinDist(0)
.WithNormalizePointDensityGlobal(false)
//.WithProgressCallback(p => { Report.Line($"{p:0.00}"); })
.WithEnabledPartIndices(true)
;

var pcl = PointCloud
Expand Down Expand Up @@ -2687,13 +2689,15 @@ static async Task Ranges_Test_20230802()

public static async Task Main(string[] _)

Check warning on line 2690 in src/Aardvark.Algodat.Tests/Program.cs

View workflow job for this annotation

GitHub Actions / build 

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.

Check warning on line 2690 in src/Aardvark.Algodat.Tests/Program.cs

View workflow job for this annotation

GitHub Actions / build 

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.

Check warning on line 2690 in src/Aardvark.Algodat.Tests/Program.cs

View workflow job for this annotation

GitHub Actions / build 

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.

Check warning on line 2690 in src/Aardvark.Algodat.Tests/Program.cs

View workflow job for this annotation

GitHub Actions / build 

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.

Check warning on line 2690 in src/Aardvark.Algodat.Tests/Program.cs

View workflow job for this annotation

GitHub Actions / build 

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.

Check warning on line 2690 in src/Aardvark.Algodat.Tests/Program.cs

View workflow job for this annotation

GitHub Actions / build 

This async method lacks 'await' operators and will run synchronously. Consider using the 'await' operator to await non-blocking API calls, or 'await Task.Run(...)' to do CPU-bound work on a background thread.
{
Test_Import_Regression();

//await CreateStore(
// @"T:\Kindergarten.pts",
// @"W:\Aardworx\pointshare2\testdata\2023-08-01_kindergarten.store",
// minDist: 0.0
// );

await Ranges_Test_20230802();
//await Ranges_Test_20230802();

//await ParseTest_20230730();

Expand Down
2 changes: 1 addition & 1 deletion src/Aardvark.Algodat.Tests/ProgressReportingTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ static IEnumerable<Chunk> GenerateChunks(int numberOfPointsPerChunk)
{
var _ps = new V3d[numberOfPointsPerChunk];
for (var i = 0; i < numberOfPointsPerChunk; i++) _ps[i] = new V3d(r.NextDouble(), r.NextDouble(), r.NextDouble());
yield return new Chunk(_ps);
yield return new Chunk(_ps).WithPartIndices(42u);
}
}
}
Expand Down
15 changes: 10 additions & 5 deletions src/Aardvark.Algodat.Tests/QueryTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ private static PointSet CreateRandomPointsInUnitCube(int n, int splitLimit)
.WithKey("test")
.WithOctreeSplitLimit(splitLimit)
;
return PointCloud.Chunks(new Chunk(ps), config);
var chunk = new Chunk(ps);
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
return PointCloud.Chunks(chunk, config);
}

private static PointSet CreateClusteredPointsInUnitCube(int n, int splitLimit)
Expand All @@ -51,7 +53,9 @@ private static PointSet CreateClusteredPointsInUnitCube(int n, int splitLimit)
.WithKey("test")
.WithOctreeSplitLimit(splitLimit)
;
return PointCloud.Chunks(new Chunk(ps), config);
var chunk = new Chunk(ps);
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
return PointCloud.Chunks(chunk, config);
}

private static PointSet CreateRegularPointsInUnitCube(int n, int splitLimit)
Expand All @@ -67,8 +71,9 @@ private static PointSet CreateRegularPointsInUnitCube(int n, int splitLimit)
.WithStorage(PointCloud.CreateInMemoryStore(cache: default))
.WithKey("test")
.WithOctreeSplitLimit(splitLimit)
;
var pc = PointCloud.Chunks(new Chunk(ps), config);
; var chunk = new Chunk(ps);
if (config.ParseConfig.EnabledProperties.PartIndices) chunk = chunk.WithPartIndices(42u);
var pc = PointCloud.Chunks(chunk, config);
return pc;
}

Expand Down Expand Up @@ -780,7 +785,7 @@ private static PointSet InternalCreateRandomPointSetForOctreeLevelTests()
var config = ImportConfig.Default.WithKey("Test").WithOctreeSplitLimit(1);
return PointSet
.Create(
storage, "test", ps.ToList(), cs.ToList(), null, null, null, null, 100,
storage, "test", ps.ToList(), cs.ToList(), null, null, null, partIndices: 42u, 100,
generateLod: false, isTemporaryImportNode: true, default
)
.GenerateLod(config)
Expand Down
47 changes: 45 additions & 2 deletions src/Aardvark.Data.Points.Base/PartIndexUtils.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ limitations under the License.
using Aardvark.Base;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics.CodeAnalysis;
using System.Linq;

#pragma warning disable CS1591
Expand All @@ -31,6 +29,51 @@ namespace Aardvark.Data.Points;
/// </summary>
public static class PartIndexUtils
{
/// <summary>
/// Compacts part indices.
/// If per-point indices are all identical, then return per-cell index.
/// If max per-point index fits in a smaller type (e.g. byte), then convert to array of smaller type.
/// </summary>
public static object? Compact(object? o)
{
switch (o)
{
case null: return null;
case uint x: return x;
case byte[] xs:
{
if (xs.Length == 0) throw new Exception("Invariant fa0e5cea-c04a-4649-9018-765606529e38.");
var range = new Range1b(xs);
if (range.Min < 0) throw new Exception("Invariant 46a46203-2525-40c5-95ab-ff6f05f71f55.");
return range.Min == range.Max ? (uint)range.Min : xs;
}
case short[] xs:
{
if (xs.Length == 0) throw new Exception("Invariant 9d18a39b-d19c-4084-95b0-eb30c6a3e38f.");
var range = new Range1s(xs);
if (range.Min < 0) throw new Exception("Invariant 5d7b3558-e235-4ccc-9b10-2d4217fb8459.");
if (range.Min == range.Max) return (uint)range.Min;
if (range.Max < 256) checked { return xs.Map(x => (byte)x); }
return xs;
}
case int[] xs:
{
if (xs.Length == 0) throw new Exception("Invariant f60565d1-6cea-47a0-95c2-30625bd16c1b.");
var range = new Range1i(xs);
if (range.Min < 0) throw new Exception("Invariant 2e002802-dd0b-402b-970b-a49a6decd987.");
if (range.Min == range.Max) return (uint)range.Min;
if (range.Max < 256) checked { return xs.Map(x => (byte)x); }
if (range.Max < 32768) checked { return xs.Map(x => (short)x); }
return xs;
}
default:
throw new Exception(
$"Unexpected type {o.GetType().FullName}. " +
$"Invariant 5b5857b3-b389-41d8-ae81-50f6ef3c133e."
);
}
}

public static Durable.Def GetDurableDefForPartIndices(object? partIndices) => partIndices switch
{
null => throw new Exception("Invariant 598ae146-211f-4cee-af57-985eb26ce961."),
Expand Down
9 changes: 8 additions & 1 deletion src/Aardvark.Geometry.PointSet/Import/MapReduce.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,14 @@ public static PointSet MapReduce(this IEnumerable<Chunk> chunks, ImportConfig co
var root = builder.ToPointSetNode(config.Storage, isTemporaryImportNode: true);
var id = $"Aardvark.Geometry.PointSet.{Guid.NewGuid()}.json";
var pointSet = new PointSet(config.Storage, id, root.Id, config.OctreeSplitLimit);
#if DEBUG
if (config.ParseConfig.EnabledProperties.PartIndices)
{
if (!chunk.HasPartIndices) throw new Exception("Invariant 04c62ea7-287b-473f-a3e9-6b5451222e27.");
}
#endif
return pointSet;
},
config.MaxDegreeOfParallelism, null, config.CancellationToken
Expand Down
8 changes: 7 additions & 1 deletion src/Aardvark.Geometry.PointSet/Octrees/InMemoryPointSet.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,13 @@ internal PointSetNode ToPointSetNode(Storage storage, bool isTemporaryImportNode
foreach (var kv in _octree.m_data)
{
if (kv.Key == Durable.Octree.PositionsGlobal3d) continue;
if (kv.Key == Durable.Octree.PerCellPartIndex1ui) continue;

if (kv.Key == Durable.Octree.PerCellPartIndex1ui)
{
attributes = attributes.Add(kv.Key, kv.Value);
continue;
}

var subset = kv.Value.Subset(_ia);
attributes = attributes.Add(kv.Key, subset);
}
Expand Down
Loading

0 comments on commit 6d9e5a9

Please sign in to comment.