diff --git a/devenv.nix b/devenv.nix index b3c1aa8..cb6adc6 100644 --- a/devenv.nix +++ b/devenv.nix @@ -8,8 +8,14 @@ packages = with pkgs; [ act git + ruff ]; + env.NIX_LD_LIBRARY_PATH = lib.makeLibraryPath (with pkgs; [ + stdenv.cc.cc + ]); + env.NIX_LD = lib.fileContents "${pkgs.stdenv.cc}/nix-support/dynamic-linker"; + languages = { python = { version = "3.12"; diff --git a/poetry.lock b/poetry.lock index d1e170b..d702070 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,44 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "loguru" +version = "0.7.2" +description = "Python logging made (stupidly) simple" +optional = false +python-versions = ">=3.5" +files = [ + {file = "loguru-0.7.2-py3-none-any.whl", hash = "sha256:003d71e3d3ed35f0f8984898359d65b79e5b21943f78af86aa5491210429b8eb"}, + {file = "loguru-0.7.2.tar.gz", hash = "sha256:e671a53522515f34fd406340ee968cb9ecafbc4b36c679da03c18fd8d0bd51ac"}, +] + +[package.dependencies] +colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} +win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} + +[package.extras] +dev = ["Sphinx (==7.2.5)", "colorama (==0.4.5)", "colorama (==0.4.6)", "exceptiongroup (==1.1.3)", "freezegun (==1.1.0)", "freezegun (==1.2.2)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v1.4.1)", "mypy (==v1.5.1)", "pre-commit (==3.4.0)", "pytest (==6.1.2)", "pytest (==7.4.0)", "pytest-cov (==2.12.1)", "pytest-cov (==4.1.0)", "pytest-mypy-plugins (==1.9.3)", "pytest-mypy-plugins (==3.0.0)", "sphinx-autobuild (==2021.3.14)", "sphinx-rtd-theme (==1.3.0)", "tox (==3.27.1)", "tox (==4.11.0)"] [[package]] name = "maturin" @@ -151,6 +191,52 @@ files = [ {file = "numpy-2.1.2.tar.gz", hash = "sha256:13532a088217fa624c99b843eeb54640de23b3414b14aa66d023805eb731066c"}, ] +[[package]] +name = "packaging" +version = "24.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, + {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, +] + +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pytest" +version = "8.3.3" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"}, + {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.5,<2" + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + [[package]] name = "ruff" version = "0.7.1" @@ -189,7 +275,21 @@ files = [ {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, ] +[[package]] +name = "win32-setctime" +version = "1.1.0" +description = "A small Python utility to set file creation time on Windows" +optional = false +python-versions = ">=3.5" +files = [ + {file = "win32_setctime-1.1.0-py3-none-any.whl", hash = "sha256:231db239e959c2fe7eb1d7dc129f11172354f98361c4fa2d6d2d7e278baa8aad"}, + {file = "win32_setctime-1.1.0.tar.gz", hash = "sha256:15cf5750465118d6929ae4de4eb46e8edae9a5634350c01ba582df868e932cb2"}, +] + +[package.extras] +dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] + [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "85b3c4320e42e10d502b97a6dddd344bf2e05d44ff2cffc196b7367b249157e5" +content-hash = "699c1dae24ec50cd9446264316617a08e2dfc19c8bec9b214662af177ead072b" diff --git a/pyproject.toml b/pyproject.toml index 192cddc..4a649ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,8 @@ numpy = ">=2.0.0,<3.0.0" [tool.poetry.group.dev.dependencies] ruff = "^0.7.1" mypy = "^1.13.0" +pytest = "^8.3.3" +loguru = "^0.7.2" [build-system] requires = ["maturin>=1.7,<2.0"] @@ -35,10 +37,8 @@ module-name = "read_aconity_layers" features = ["pyo3/extension-module"] [tool.ruff] -# Same as Black. line-length = 100 -# Assume Python 3.11 -target-version = "py311" +target-version = "py312" exclude = ["docs/", "tests/"] [tool.ruff.lint] @@ -201,6 +201,6 @@ check_untyped_defs = true ignore_missing_imports = true exclude = ["docs/", "tests/"] -[[tool.mypy.overrides]] -module = ["flet.*", "flet_core.*", "fsspec.*"] -ignore_missing_imports = true +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = [".venv/bin/python"] diff --git a/src/rust_fn/mod.rs b/src/rust_fn/mod.rs index 5497b80..646e40e 100644 --- a/src/rust_fn/mod.rs +++ b/src/rust_fn/mod.rs @@ -240,7 +240,7 @@ mod tests { use tempfile::{tempdir, TempDir}; use xz::read::XzDecoder; - const TEST_DATA_FILE: &str = "tests/data.tar.xz"; + const TEST_DATA_FILE: &str = "tests/correct.tar.xz"; const Z_SCALING: f64 = f64::MAX / 100.0; #[derive(Debug)] diff --git a/tests/data.tar.xz b/tests/correct.tar.xz similarity index 100% rename from tests/data.tar.xz rename to tests/correct.tar.xz diff --git a/tests/read_layers_regr.npz b/tests/read_layers_regr.npz new file mode 100644 index 0000000..4741101 Binary files /dev/null and b/tests/read_layers_regr.npz differ diff --git a/tests/test_read_layers.py b/tests/test_read_layers.py new file mode 100644 index 0000000..7c9bc1b --- /dev/null +++ b/tests/test_read_layers.py @@ -0,0 +1,101 @@ +from concurrent.futures import ThreadPoolExecutor +from enum import Enum +import numpy as np +from pathlib import Path +import pytest +import subprocess + + +TEST_ARRAY = np.arange( + np.iinfo(np.int32).min, + np.iinfo(np.int32).max, + 256, # Limited to ~256MB uncompressed for practicality +).reshape((-1, 4)) +N_FILES = 1024 +TEST_ZVALS = np.arange(0, 10_240, 10_240 // (N_FILES - 1)) + + +def build_module(): + subprocess.run(["maturin", "develop"]) + + +build_fixture = pytest.fixture(scope="module")(build_module) + + +def write_layers_to_dir(dir_path): + output_layers = TEST_ARRAY.reshape((1024, -1, 4)) + + def write_layer(ar, z): + np.savetxt(dir_path / f"{z}.pcd", ar, delimiter=" ", newline="\n", fmt="%i") + + with ThreadPoolExecutor() as p: + p.map(write_layer, output_layers, TEST_ZVALS) + + +# Sorts arrays without mixing datapoints +# This is needed because we dont need to guarantee read order +def sort_result(ar): + ar = ar[ar[:, 4].argsort()] + ar = ar[ar[:, 3].argsort()] + ar = ar[ar[:, 0].argsort()] + ar = ar[ar[:, 1].argsort()] + ar = ar[ar[:, 2].argsort()] + return ar + + +def regenerate_regr_outputs(): + from loguru import logger + import tempfile + + build_module() + from read_aconity_layers import read_layers + + dir_path = tempfile.mkdtemp() + logger.info("Writing temporary layer file...") + write_layers_to_dir(Path(dir_path)) + logger.info("Reading temporary layer file...") + output = read_layers(dir_path) + output_file_path = Path("tests/read_layers_regr.npz") + logger.info("Sorting outputs...") + output_sorted = sort_result(output) + logger.info("Saving outputs to compressed file...") + np.savez_compressed(output_file_path, output=output_sorted) + logger.info("Outputs regenerated!") + + +@pytest.fixture(scope="module") +def shared_dir(tmpdir_factory): + dir_path = tmpdir_factory.mktemp("read_layers_shared") + write_layers_to_dir(dir_path) + return dir_path + + +@pytest.fixture(scope="module") +def ground_truth(): + return np.load("tests/read_layers_regr.npz")["output"] + + +def test_read_layers(build_fixture, shared_dir, ground_truth): + from read_aconity_layers import read_layers + + result = sort_result(read_layers(str(shared_dir))) + assert np.all(np.isclose(np.argsort(result, 0), np.argsort(ground_truth, 0))) + + +def test_read_selected_layers(build_fixture, shared_dir, ground_truth): + from read_aconity_layers import read_selected_layers + + result = sort_result(read_selected_layers([str(x) for x in shared_dir.listdir()])) + assert np.all(np.isclose(result, ground_truth)) + + +def test_read_layer(build_fixture, shared_dir, ground_truth): + from read_aconity_layers import read_layer + + x = np.concat([read_layer(str(x)) for x in shared_dir.listdir()], axis=0) + result = sort_result(x) + assert np.all(np.isclose(result, ground_truth)) + + +if __name__ == "__main__": + regenerate_regr_outputs()