tests.test_scsv

  1"""> PyDRex: tests for the SCSV plain text file format."""
  2
  3import tempfile
  4
  5import numpy as np
  6import pytest
  7from numpy import testing as nt
  8from pydrex import exceptions as _err
  9from pydrex import io as _io
 10from pydrex import logger as _log
 11from pydrex import utils as _utils
 12
 13
 14def test_validate_schema(console_handler):
 15    """Test SCSV schema validation."""
 16    schema_nofill = {
 17        "delimiter": ",",
 18        "missing": "-",
 19        "fields": [{"name": "nofill", "type": "float"}],
 20    }
 21    schema_nomissing = {
 22        "delimiter": ",",
 23        "fields": [{"name": "nomissing", "type": "float", "fill": "NaN"}],
 24    }
 25    schema_nofields = {"delimiter": ",", "missing": "-"}
 26    schema_badfieldname = {
 27        "delimiter": ",",
 28        "missing": "-",
 29        "fields": [{"name": "bad name", "type": "float", "fill": "NaN"}],
 30    }
 31    schema_delimiter_eq_missing = {
 32        "delimiter": ",",
 33        "missing": ",",
 34        "fields": [{"name": "baddelim", "type": "float", "fill": "NaN"}],
 35    }
 36    schema_delimiter_in_missing = {
 37        "delimiter": ",",
 38        "missing": "-,",
 39        "fields": [{"name": "baddelim", "type": "float", "fill": "NaN"}],
 40    }
 41    schema_long_delimiter = {
 42        "delimiter": ",,",
 43        "missing": "-",
 44        "fields": [{"name": "baddelim", "type": "float", "fill": "NaN"}],
 45    }
 46
 47    # NOTE: NamedTemporaryFile() already opens the file.
 48    # Attempting to open it again will cause a crash on Windows so close the file first.
 49    with _log.handler_level("CRITICAL", console_handler):
 50        with pytest.raises(_err.SCSVError):
 51            temp = tempfile.NamedTemporaryFile()
 52            temp.close()
 53            _io.save_scsv(temp.name, schema_nofill, [[0.1]])
 54        with pytest.raises(_err.SCSVError):
 55            temp = tempfile.NamedTemporaryFile()
 56            temp.close()
 57            _io.save_scsv(temp.name, schema_nomissing, [[0.1]])
 58        with pytest.raises(_err.SCSVError):
 59            temp = tempfile.NamedTemporaryFile()
 60            temp.close()
 61            _io.save_scsv(temp.name, schema_nofields, [[0.1]])
 62        with pytest.raises(_err.SCSVError):
 63            temp = tempfile.NamedTemporaryFile()
 64            temp.close()
 65            _io.save_scsv(temp.name, schema_badfieldname, [[0.1]])
 66        with pytest.raises(_err.SCSVError):
 67            temp = tempfile.NamedTemporaryFile()
 68            temp.close()
 69            _io.save_scsv(temp.name, schema_delimiter_eq_missing, [[0.1]])
 70        with pytest.raises(_err.SCSVError):
 71            temp = tempfile.NamedTemporaryFile()
 72            temp.close()
 73            _io.save_scsv(temp.name, schema_delimiter_in_missing, [[0.1]])
 74        # CSV module already raises a TypeError on long delimiters.
 75        with pytest.raises(TypeError):
 76            temp = tempfile.NamedTemporaryFile()
 77            temp.close()
 78            _io.save_scsv(temp.name, schema_long_delimiter, [[0.1]])
 79
 80
 81def test_read_specfile():
 82    """Test SCSV spec file parsing."""
 83    data = _io.read_scsv(_io.data("specs") / "spec.scsv")
 84    assert data._fields == (
 85        "first_column",
 86        "second_column",
 87        "third_column",
 88        "float_column",
 89        "bool_column",
 90        "complex_column",
 91    )
 92    nt.assert_equal(data.first_column, ["s1", "MISSING", "s3"])
 93    nt.assert_equal(data.second_column, ["A", "B, b", ""])
 94    nt.assert_equal(data.third_column, [999999, 10, 1])
 95    nt.assert_equal(data.float_column, [1.1, np.nan, 1.0])
 96    nt.assert_equal(data.bool_column, [True, False, True])
 97    nt.assert_equal(data.complex_column, [0.1 + 0 * 1j, np.nan + 0 * 1j, 1.0 + 1 * 1j])
 98
 99
100@pytest.mark.skipif(_utils.in_ci("win32"), reason="Items are not equal")
101def test_save_specfile(outdir, named_tempfile_kwargs):
102    """Test SCSV spec file reproduction."""
103    schema = {
104        "delimiter": ",",
105        "missing": "-",
106        "fields": [
107            {
108                "name": "first_column",
109                "type": "string",
110                "fill": "MISSING",
111                "unit": "percent",
112            },
113            {"name": "second_column"},
114            {"name": "third_column", "type": "integer", "fill": "999999"},
115            {"name": "float_column", "type": "float", "fill": "NaN"},
116            {"name": "bool_column", "type": "boolean"},
117            {"name": "complex_column", "type": "complex", "fill": "NaN"},
118        ],
119    }
120    schema_alt = {
121        "delimiter": ",",
122        "missing": "-",
123        "fields": [
124            {
125                "name": "first_column",
126                "type": "string",
127                "unit": "percent",
128            },
129            {"name": "second_column"},
130            {"name": "third_column", "type": "integer", "fill": "999991"},
131            {"name": "float_column", "type": "float", "fill": "0.0"},
132            {"name": "bool_column", "type": "boolean"},
133            {"name": "complex_column", "type": "complex", "fill": "NaN"},
134        ],
135    }
136
137    data = [
138        ["s1", "MISSING", "s3"],
139        ["A", "B, b", ""],
140        [999999, 10, 1],
141        [1.1, np.nan, 1.0],
142        [True, False, True],
143        [0.1 + 0 * 1j, np.nan + 0 * 1j, 1.0 + 1 * 1j],
144    ]
145    data_alt = [
146        ["s1", "", "s3"],
147        ["A", "B, b", ""],
148        [999991, 10, 1],
149        [1.1, 0.0, 1.0],
150        [True, False, True],
151        [0.1 + 0 * 1j, np.nan + 0 * 1j, 1.0 + 1 * 1j],
152    ]
153
154    # The test writes two variants of the file, with identical CSV contents but
155    # different YAML header specs. Contents after the header must match.
156    if outdir is not None:
157        _io.save_scsv(f"{outdir}/spec_out.scsv", schema, data)
158        _io.save_scsv(f"{outdir}/spec_out_alt.scsv", schema_alt, data_alt)
159
160    # https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile
161    temp = tempfile.NamedTemporaryFile(**named_tempfile_kwargs)
162    temp_alt = tempfile.NamedTemporaryFile(**named_tempfile_kwargs)
163    _io.save_scsv(temp.name, schema, data)
164    _io.save_scsv(temp_alt.name, schema_alt, data_alt)
165    raw_read = []
166    raw_read_alt = []
167    with open(temp.name) as stream:
168        raw_read = stream.readlines()[23:]  # Extra spec for first column 'fill' value.
169    with open(temp_alt.name) as stream:
170        raw_read_alt = stream.readlines()[22:]
171    _log.debug("\n  first file: %s\n  second file: %s", raw_read, raw_read_alt)
172    nt.assert_equal(raw_read, raw_read_alt)
173
174
175def test_read_Kaminski2002():
176    data = _io.read_scsv(_io.data("thirdparty") / "Kaminski2002_ISAtime.scsv")
177    assert data._fields == ("time_ISA", "vorticity")
178    # fmt: off
179    nt.assert_equal(
180        data.time_ISA,
181        np.array(
182            [2.48, 2.50, 2.55, 2.78, 3.07, 3.58, 4.00, 4.88, 4.01, 3.79,
183             3.72, 3.66, 3.71, 4.22, 4.73, 3.45, 1.77, 0.51]
184        ),
185    )
186    nt.assert_equal(
187        data.vorticity,
188        np.array(
189            [0.05, 0.10, 0.20, 0.30, 0.35, 0.40, 0.45, 0.50, 0.55, 0.60,
190             0.65, 0.70, 0.75, 0.80, 0.85, 0.90, 0.95, 1.00]
191        ),
192    )
193    # fmt: on
194
195
196def test_save_scsv_errors(named_tempfile_kwargs):
197    """Check that we raise errors when attempting to write bad SCSV data."""
198    schema = {
199        "delimiter": ",",
200        "missing": "-",
201        "fields": [
202            {
203                "name": "foo",
204                "type": "integer",
205                "fill": 999999,
206            }
207        ],
208    }
209    # https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile
210    temp = tempfile.NamedTemporaryFile(**named_tempfile_kwargs)
211    with pytest.raises(_err.SCSVError):
212        foo = [1, 5, 0.2]
213        _io.save_scsv(temp.name, schema, [foo])
214        foo = [1, "foo"]
215        _io.save_scsv(temp.name, schema, [foo])
216        foo = ["foo"]
217        _io.save_scsv(temp.name, schema, [foo])
218        foo = [True]
219        _io.save_scsv(temp.name, schema, [foo])
220        foo = [1, 2, 3]
221        bar = [1, 2, 3]
222        _io.save_scsv(temp.name, schema, [foo, bar])
223
224
225def test_read_Kaminski2004():
226    data = _io.read_scsv(_io.data("thirdparty") / "Kaminski2004_AaxisDynamicShear.scsv")
227    assert data._fields == ("time", "meanA_X0", "meanA_X02", "meanA_X04")
228    # fmt: off
229    nt.assert_equal(
230        data.time,
231        np.array(
232            [-0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9,
233             1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 1.6, 1.7, 1.8, 1.9, 2.0,
234             2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 2.8, 2.9, 3.0, 3.1,
235             3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 4.0, 4.1, 4.2,
236             4.3, 4.4, 4.5, 4.6, 4.7, 4.8, 4.9, 5.0]
237        ),
238    )
239    nt.assert_equal(
240        data.meanA_X02,
241        np.array(
242            [-0.54, -0.54, -0.27, 0.13, 0.94, 2.82, 5.37, 9.53, 14.77, 20.40,
243             26.58, 32.89, 39.73, 47.25, 53.69, 58.66, 60.81, 60.81, 59.73, 58.52, 58.12,
244             56.64, 54.09, 53.69, 55.57, 57.05, 58.66, 60.54, 60.81, 61.21, 61.21, 61.61,
245             61.48, 61.61, 61.61, 61.61, 61.21, 61.21, 61.07, 60.81, 60.81, 60.54, 60.00,
246             59.60, 59.33, 58.52, 58.12, 57.85, 57.45, 57.05, 57.05]
247        ),
248    )
249    # fmt: on
250
251
252def test_read_Skemer2016():
253    data = _io.read_scsv(_io.data("thirdparty") / "Skemer2016_ShearStrainAngles.scsv")
254    assert data._fields == (
255        "study",
256        "sample_id",
257        "shear_strain",
258        "angle",
259        "fabric",
260        "M_index",
261    )
262    # fmt: off
263    nt.assert_equal(
264        data.study,
265        ["Z&K 1200 C", "Z&K 1200 C", "Z&K 1200 C", "Z&K 1200 C", "Z&K 1200 C",
266         "Z&K 1300 C", "Z&K 1300 C", "Z&K 1300 C", "Z&K 1300 C", "Z&K 1300 C",
267         "Z&K 1300 C", "Z&K 1300 C", "Bystricky 2000", "Bystricky 2000", "Bystricky 2000",
268         "Bystricky 2000", "Warren 2008", "Warren 2008", "Warren 2008", "Warren 2008",
269         "Warren 2008", "Warren 2008", "Warren 2008", "Warren 2008", "Warren 2008",
270         "Skemer 2011", "Skemer 2011", "Skemer 2011", "Webber 2010", "Webber 2010",
271         "Webber 2010", "Webber 2010", "Webber 2010", "Webber 2010", "Webber 2010",
272         "Katayama 2004", "Katayama 2004", "Katayama 2004", "Katayama 2004", "Katayama 2004",
273         "Katayama 2004", "Skemer 2010", "Skemer 2010", "Skemer 2010", "Skemer 2010",
274         "Skemer 2010", "Jung 2006", "Jung 2006", "Jung 2006", "Jung 2006",
275         "Jung 2006", "Jung 2006", "Hansen 2014", "Hansen 2014", "Hansen 2014",
276         "Hansen 2014", "Hansen 2014", "Hansen 2014", "Hansen 2014", "Hansen 2014",
277         "Hansen 2014", "Hansen 2014", "Hansen 2014", "Hansen 2014", "Hansen 2014",
278         "Hansen 2014", "Hansen 2014", "Hansen 2014", "Hansen 2014", "Hansen 2014",
279         "Hansen 2014", "Hansen 2014", "Hansen 2014", "Hansen 2014", "Hansen 2014",
280         "Hansen 2014", "Hansen 2014", "Hansen 2014", "H&W 2015", "H&W 2015",
281         "H&W 2015", "H&W 2015", "H&W 2015", "H&W 2015", "H&W 2015",
282         "H&W 2015", "H&W 2015", "H&W 2015", "H&W 2015", "H&W 2015",
283         "H&W 2015", "H&W 2015", "H&W 2015", "H&W 2015"],
284    )
285    nt.assert_equal(
286        data.sample_id,
287        ["PI-148", "PI-150", "PI-154", "PI-158", "PI-284", "MIT-5", "MIT-20", "MIT-6",
288         "MIT-21", "MIT-17", "MIT-18", "MIT-19", "", "", "", "", "3923J01",
289         "3923J11", "3923J13", "3923J14", "3924J06", "3924J09a", "3924J09b", "3924J08",
290         "3924J10", "PIP-20", "PIP-21", "PT-484", "", "", "", "", "", "", "", "GA10",
291         "GA38", "GA23", "GA12", "GA45", "GA25", "3925G08", "3925G05", "3925G02",
292         "3925G01", "PT-248_4", "JK43", "JK21B", "JK26", "JK11", "JK18", "JK23",
293         "PT0535", "PT0248_1", "", "PT0655", "PT0248_2", "PI-284", "PT0248_3", "PT0503_5",
294         "PT0248_4", "PT0248_5", "PT0503_4", "PT0640", "PT0503_3", "", "PT0494",
295         "PT0538", "PT0503_2", "PT0541", "PT0503_1", "PT0633", "PT0552", "PT0505",
296         "PT0651", "PT0499", "PT619", "PT0484", "3923J06", "3923J07", "3923J09",
297         "3923J08", "3923J13", "3923J12", "3924J06", "3924J05", "JP13-D07", "JP13-D06",
298         "3924J03a", "3924J03b", "3924J09a", "3924J09b", "3924J08", "3924J07"],
299    )
300    nt.assert_equal(
301        data.shear_strain,
302        np.array(
303            [
304                17, 30, 45, 65, 110, 11, 7, 65, 58, 100,
305                115, 150, 50, 200, 400, 500, 0, 65, 118, 131, 258, 386,
306                386, 525, 168, 120, 180, 350, 0, 25, 100, 130, 168, 330,
307                330, 60, 60, 80, 120, 260, 630, 60, 150, 1000, 1000, 290,
308                120, 400, 100, 110, 120, 400, 0, 30, 50, 100, 100, 110,
309                210, 270, 290, 390, 390, 400, 490, 500, 590, 680, 680, 760,
310                820, 870, 880, 1020, 1060, 1090, 1420, 1870, 32, 32, 81, 81,
311                118, 118, 258, 258, 286, 286, 337, 337, 386, 386, 525, 525
312            ]
313        ),
314    )
315    nt.assert_equal(
316        data.angle,
317        np.array(
318            [
319                43, 37, 38, 24, 20, 36, 28, 18, 10, 5,
320                10, 0, 0, 0, 0, 0, 62, 37, 49, 61, 4, 11, 0, 1, 33, 55,
321                53, 45, 55, 35, 47, 29, 37, 47, 45, -49, -26, -10, -35,
322                -10, -15, -52, -30, -14, -11, 0, 26, 15, 27, 25, 16, 36,
323                -71, -78, 71, 29, 39, 24, 12, 7.3, -3.7, -1.3, -6.1, 0.4,
324                -8.6, 5.4, 0.6, -5.5, -4.3, -8.4, -1.9, -0.9, -8.9, 2.8,
325                -2, -1.5, -4.1, -5.8, 48, 51, 44, 35, 35, 40, 1, 15, 25,
326                28, 28, 39, 1, 8, 4, 11
327            ]
328        ),
329    )
330    nt.assert_equal(
331        data.fabric,
332        [
333            "A", "A", "A", "A", "A", "A", "A", "A", "A", "A", "A", "A",
334            "D", "D", "D", "D", "A", "A", "A", "A", "A", "A", "A", "A",
335            "A", "A", "A", "A", "A", "A", "A", "A", "A", "A", "A", "E",
336            "E", "E", "E", "E", "E", "E", "E", "E", "E", "D", "A", "B",
337            "C", "C", "C", "C", "D", "D", "D", "D", "D", "D", "D", "D",
338            "D", "D", "D", "D", "D", "A", "A", "A", "A", "A", "A", "A",
339            "A", "A", "A", "A", "A", "A", "A", "A", "A", "A", "A", "A",
340            "A", "A", "A", "A", "A", "A", "A", "A", "A", "A",
341        ],
342    )
343    nt.assert_equal(
344        data.M_index,
345        np.array(
346            [
347                np.nan, np.nan, np.nan, np.nan, 0.09, np.nan, np.nan,
348                np.nan, np.nan, np.nan, np.nan, np.nan, 0.05, np.nan,
349                np.nan, 0.17, 0.08, 0.12, 0.16, 0.2, 0.17, 0.06, 0.13,
350                0.14, 0.16, np.nan, np.nan, np.nan, np.nan, np.nan,
351                np.nan, np.nan, np.nan, np.nan, np.nan, 0.17, np.nan,
352                np.nan, 0.28, np.nan, 0.44, 0.03, 0.1, 0.47, 0.37,
353                0.36, 0.06, 0.21, 0.21, 0.23, 0.27, 0.1, 0.01, 0.06,
354                0.05, 0.06, 0.06, 0.09, 0.2, 0.26, 0.36, 0.35, 0.34,
355                0.36, 0.45, 0.17, 0.4, 0.46, 0.46, 0.44, 0.35, 0.39,
356                0.4, 0.69, 0.55, 0.61, 0.51, 0.51, 0.08, 0.2, 0.16,
357                0.12, 0.12, 0.17, 0.18, 0.13, 0.15, 0.09, 0.19, 0.27,
358                0.12, 0.18, 0.14, 0.26,
359            ]
360        ),
361    )