Skip to content

fix: Point cloud component meta data group #89

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
janickm merged 1 commit into from
May 5, 2026
+95 −36
Merged

fix: Point cloud component meta data group #89

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ncore/impl/data/types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,9 @@
float,
bool,
None,
# special-case shouldn't be needed, but required to make mypy happy
# special-cases shouldn't be needed, but required to make mypy happy
List[int],
List[str],
]


Expand Down
14 changes: 6 additions & 8 deletions ncore/impl/data/v4/components.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1675,17 +1675,15 @@ def __init__(
self._attribute_schemas = attribute_schemas
self._pc_timestamps: List[int] = []

# Write source-level .zattrs
self._group.attrs.update(
# Pre-create the pcs group
self._pcs_group = self._group.require_group("pcs")
self._pcs_group.attrs.put(
{
"coordinate_unit": coordinate_unit.name,
"attribute_schemas": {name: s.to_dict() for name, s in self._attribute_schemas.items()},
}
)

# Pre-create the pcs group
self._pcs_group = self._group.require_group("pcs")

def store_pc(
self,
xyz: npt.NDArray[np.float32],
Expand Down Expand Up @@ -1799,7 +1797,7 @@ def supports_component_version(version: str) -> bool:

@property
def coordinate_unit(self) -> PointCloud.CoordinateUnit:
return PointCloud.CoordinateUnit[self._group.attrs["coordinate_unit"]]
return PointCloud.CoordinateUnit[self._group["pcs"].attrs["coordinate_unit"]]

@property
def pcs_count(self) -> int:
Expand All @@ -1811,12 +1809,12 @@ def pc_timestamps_us(self) -> "npt.NDArray[np.uint64]":

@property
def attribute_names(self) -> List[str]:
return list(self._group.attrs["attribute_schemas"].keys())
return list(self._group["pcs"].attrs["attribute_schemas"].keys())

# -- schema access -----------------------------------------------------

def get_attribute_schema(self, name: str) -> PointCloudsComponent.AttributeSchema:
schema_raw = self._group.attrs["attribute_schemas"]
schema_raw = self._group["pcs"].attrs["attribute_schemas"]
assert name in schema_raw, f"Unknown attribute: {name}"
return PointCloudsComponent.AttributeSchema.from_dict(schema_raw[name])

Expand Down
114 changes: 87 additions & 27 deletions ncore/impl/data/v4/components_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1482,7 +1482,7 @@ def setUp(self):
np.set_printoptions(floatmode="unique", linewidth=200, suppress=True)

def _make_writer_reader(
self, attribute_schemas={}, store_type: Literal["itar", "directory"] = "directory"
self, store_type: Literal["itar", "directory"], attribute_schemas={}
) -> Tuple[
PointCloudsComponent.Writer, SequenceComponentGroupsWriter, tempfile.TemporaryDirectory, HalfClosedInterval
]:
Expand Down Expand Up @@ -1516,7 +1516,13 @@ def _finalize_and_open_reader(self, store_writer: SequenceComponentGroupsWriter)
self.assertIn("test_pc", pc_readers)
return pc_readers["test_pc"]

def test_single_pc_with_attributes(self):
@parameterized.expand(
[
("itar",),
("directory",),
]
)
def test_single_pc_with_attributes(self, store_type: Literal["itar", "directory"]):
"""Write 1 PC with rgb (uint8, (N,3)) + normals (float32, (N,3)), read back, verify all fields."""
schemas = {
"rgb": PointCloudsComponent.AttributeSchema(
Expand All @@ -1530,12 +1536,12 @@ def test_single_pc_with_attributes(self):
shape_suffix=(3,),
),
}
pc_writer, store_writer, tmpdir, _ = self._make_writer_reader(attribute_schemas=schemas)
pc_writer, store_writer, tmpdir, _ = self._make_writer_reader(attribute_schemas=schemas, store_type=store_type)

N = 100
xyz = np.random.rand(N, 3).astype(np.float32)
rgb = np.random.randint(0, 256, size=(N, 3), dtype=np.uint8)
normals = np.random.rand(N, 3).astype(np.float32)
xyz = np.random.rand(N, 3).astype(np.float32) # type:ignore[attr-defined]
rgb = np.random.randint(0, 256, size=(N, 3), dtype=np.uint8) # type:ignore[attr-defined]
normals = np.random.rand(N, 3).astype(np.float32) # type:ignore[attr-defined]

pc_writer.store_pc(
xyz=xyz,
Expand Down Expand Up @@ -1576,9 +1582,15 @@ def test_single_pc_with_attributes(self):

tmpdir.cleanup()

def test_multiple_pcs_different_ref_frames(self):
@parameterized.expand(
[
("itar",),
("directory",),
]
)
def test_multiple_pcs_different_ref_frames(self, store_type: Literal["itar", "directory"]):
"""Write 2 PCs with different reference_frame_id, verify per-pc ref frames."""
pc_writer, store_writer, tmpdir, _ = self._make_writer_reader()
pc_writer, store_writer, tmpdir, _ = self._make_writer_reader(store_type=store_type)

xyz1 = np.array([[1.0, 2.0, 3.0]], dtype=np.float32)
xyz2 = np.array([[4.0, 5.0, 6.0], [7.0, 8.0, 9.0]], dtype=np.float32)
Expand Down Expand Up @@ -1642,7 +1654,13 @@ def test_attribute_schema_json_roundtrip(self):
rt = PointCloudsComponent.AttributeSchema.from_dict(scalar_schema.to_dict())
self.assertEqual(rt.shape_suffix, ())

def test_writer_rejects_undeclared_attribute(self):
@parameterized.expand(
[
("itar",),
("directory",),
]
)
def test_writer_rejects_undeclared_attribute(self, store_type: Literal["itar", "directory"]):
"""store_pc with attr not in schema -> AssertionError."""
schemas = {
"rgb": PointCloudsComponent.AttributeSchema(
Expand All @@ -1651,7 +1669,7 @@ def test_writer_rejects_undeclared_attribute(self):
shape_suffix=(3,),
),
}
pc_writer, _, tmpdir, _ = self._make_writer_reader(attribute_schemas=schemas)
pc_writer, _, tmpdir, _ = self._make_writer_reader(attribute_schemas=schemas, store_type=store_type)

xyz = np.array([[1.0, 2.0, 3.0]], dtype=np.float32)
rgb = np.array([[128, 64, 32]], dtype=np.uint8)
Expand All @@ -1667,7 +1685,13 @@ def test_writer_rejects_undeclared_attribute(self):

tmpdir.cleanup()

def test_writer_rejects_missing_schema_attribute(self):
@parameterized.expand(
[
("itar",),
("directory",),
]
)
def test_writer_rejects_missing_schema_attribute(self, store_type: Literal["itar", "directory"]):
"""store_pc missing a schema attr -> AssertionError."""
schemas = {
"rgb": PointCloudsComponent.AttributeSchema(
Expand All @@ -1681,7 +1705,7 @@ def test_writer_rejects_missing_schema_attribute(self):
shape_suffix=(3,),
),
}
pc_writer, _, tmpdir, _ = self._make_writer_reader(attribute_schemas=schemas)
pc_writer, _, tmpdir, _ = self._make_writer_reader(attribute_schemas=schemas, store_type=store_type)

xyz = np.array([[1.0, 2.0, 3.0]], dtype=np.float32)
rgb = np.array([[128, 64, 32]], dtype=np.uint8)
Expand All @@ -1696,7 +1720,13 @@ def test_writer_rejects_missing_schema_attribute(self):

tmpdir.cleanup()

def test_writer_rejects_wrong_shape(self):
@parameterized.expand(
[
("itar",),
("directory",),
]
)
def test_writer_rejects_wrong_shape(self, store_type: Literal["itar", "directory"]):
"""store_pc with wrong-shaped array -> AssertionError."""
schemas = {
"rgb": PointCloudsComponent.AttributeSchema(
Expand All @@ -1705,12 +1735,12 @@ def test_writer_rejects_wrong_shape(self):
shape_suffix=(3,),
),
}
pc_writer, _, tmpdir, _ = self._make_writer_reader(attribute_schemas=schemas)
pc_writer, _, tmpdir, _ = self._make_writer_reader(attribute_schemas=schemas, store_type=store_type)

N = 10
xyz = np.random.rand(N, 3).astype(np.float32)
xyz = np.random.rand(N, 3).astype(np.float32) # type:ignore[attr-defined]
# Wrong shape: (N, 4) instead of (N, 3)
rgb_wrong = np.random.randint(0, 256, size=(N, 4), dtype=np.uint8)
rgb_wrong = np.random.randint(0, 256, size=(N, 4), dtype=np.uint8) # type:ignore[attr-defined]

with self.assertRaises(AssertionError):
pc_writer.store_pc(
Expand All @@ -1722,9 +1752,15 @@ def test_writer_rejects_wrong_shape(self):

tmpdir.cleanup()

def test_writer_rejects_reference_frame_timestamp_out_of_range(self):
@parameterized.expand(
[
("itar",),
("directory",),
]
)
def test_writer_rejects_reference_frame_timestamp_out_of_range(self, store_type: Literal["itar", "directory"]):
"""store_pc with reference_frame_timestamp_us outside sequence range -> AssertionError."""
pc_writer, _, tmpdir, _ = self._make_writer_reader()
pc_writer, _, tmpdir, _ = self._make_writer_reader(store_type=store_type)

xyz = np.array([[1.0, 2.0, 3.0]], dtype=np.float32)
with self.assertRaises(AssertionError):
Expand All @@ -1736,23 +1772,35 @@ def test_writer_rejects_reference_frame_timestamp_out_of_range(self):

tmpdir.cleanup()

def test_writer_rejects_wrong_xyz_dtype(self):
@parameterized.expand(
[
("itar",),
("directory",),
]
)
def test_writer_rejects_wrong_xyz_dtype(self, store_type: Literal["itar", "directory"]):
"""store_pc with float64 xyz raises AssertionError (float32 required)."""
pc_writer, _, tmpdir, _ = self._make_writer_reader()
pc_writer, _, tmpdir, _ = self._make_writer_reader(store_type=store_type)

xyz_f64 = np.array([[1.0, 2.0, 3.0]], dtype=np.float64)
with self.assertRaises(AssertionError):
pc_writer.store_pc(
xyz=xyz_f64,
xyz=xyz_f64, # type: ignore
reference_frame_id="world",
reference_frame_timestamp_us=500_000,
)

tmpdir.cleanup()

def test_empty_writer_finalize(self):
@parameterized.expand(
[
("itar",),
("directory",),
]
)
def test_empty_writer_finalize(self, store_type: Literal["itar", "directory"]):
"""Finalizing a writer with zero store_pc calls produces a valid empty reader."""
pc_writer, store_writer, tmpdir, _ = self._make_writer_reader()
_, store_writer, tmpdir, _ = self._make_writer_reader(store_type=store_type)

# Finalize without any store_pc calls
reader = self._finalize_and_open_reader(store_writer)
Expand All @@ -1763,9 +1811,15 @@ def test_empty_writer_finalize(self):

tmpdir.cleanup()

def test_no_attributes_no_generic_data(self):
@parameterized.expand(
[
("itar",),
("directory",),
]
)
def test_no_attributes_no_generic_data(self, store_type: Literal["itar", "directory"]):
"""Write/read a PC with empty schema and no generic data."""
pc_writer, store_writer, tmpdir, _ = self._make_writer_reader()
pc_writer, store_writer, tmpdir, _ = self._make_writer_reader(store_type=store_type)

xyz = np.array([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], dtype=np.float32)
pc_writer.store_pc(
Expand All @@ -1786,9 +1840,15 @@ def test_no_attributes_no_generic_data(self):

tmpdir.cleanup()

def test_generic_data_and_metadata(self):
@parameterized.expand(
[
("itar",),
("directory",),
]
)
def test_generic_data_and_metadata(self, store_type: Literal["itar", "directory"]):
"""Verify generic_data arrays and generic_meta_data round-trip."""
pc_writer, store_writer, tmpdir, _ = self._make_writer_reader()
pc_writer, store_writer, tmpdir, _ = self._make_writer_reader(store_type=store_type)

xyz = np.array([[0.0, 0.0, 0.0]], dtype=np.float32)
gd_labels = np.array([42], dtype=np.int32)
Expand Down
Loading